Skip to content

Commit

Permalink
don't require pyessv-archive to run whole package
Browse files Browse the repository at this point in the history
  • Loading branch information
mishaschwartz committed Oct 7, 2024
1 parent 3bd6a40 commit c291918
Show file tree
Hide file tree
Showing 7 changed files with 57 additions and 14 deletions.
3 changes: 0 additions & 3 deletions .gitmodules

This file was deleted.

8 changes: 4 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@ STAC_HOST ?= http://localhost:8880/stac
# CATALOG = https://daccs.cs.toronto.edu/twitcher/ows/proxy/thredds/catalog/datasets/CMIP6/CMIP/NUIST/catalog.html
CATALOG = https://daccs.cs.toronto.edu/twitcher/ows/proxy/thredds/catalog/datasets/CMIP6/CMIP/MIROC/catalog.html

PYESSV_ARCHIVE_DIR ?= ~/.esdoc/pyessv-archive
PYESSV_ARCHIVE_HOME ?= ~/.esdoc/pyessv-archive
PYESSV_ARCHIVE_REF ?= https://github.com/ES-DOC/pyessv-archive

## -- Testing targets -------------------------------------------------------------------------------------------- ##

setup-pyessv-archive:
@echo "Updating pyessv archive [$(shell realpath $(PYESSV_ARCHIVE_DIR))]..."
@[ -d $(PYESSV_ARCHIVE_DIR) ] || git clone "$(PYESSV_ARCHIVE_REF)" $(PYESSV_ARCHIVE_DIR)
@cd $(PYESSV_ARCHIVE_DIR) && git pull
@echo "Updating pyessv archive [$(shell realpath $(PYESSV_ARCHIVE_HOME))]..."
@[ -d $(PYESSV_ARCHIVE_HOME) ] || git clone "$(PYESSV_ARCHIVE_REF)" $(PYESSV_ARCHIVE_HOME)
@cd $(PYESSV_ARCHIVE_HOME) && git pull

test-cmip6:
python $(IMP_DIR)/CMIP6_UofT/add_CMIP6.py $(STAC_HOST) $(CATALOG)
Expand Down
39 changes: 35 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,18 +49,49 @@ You should then be able to call the STAC populator CLI with following commands:

```shell
# obtain the installed version of the STAC populator
stac-popultaor --version
stac-populator --version

# obtain general help about available commands
stac-popultaor --help
stac-populator --help

# obtain general help about available STAC populator implementations
stac-popultaor run --help
stac-populator run --help

# obtain help specifically for the execution of a STAC populator implementation
stac-popultaor run [implementation] --help
stac-populator run [implementation] --help
```

### CMIP6 extension: extra requirements

The CMIP6 stac-populator extension requires that the [pyessv-archive](https://github.com/ES-DOC/pyessv-archive) data
files be installed. To install this package to the default location in your home directory at `~/.esdoc/pyessv-archive`:

```shell
git clone https://github.com/ES-DOC/pyessv-archive ~/.esdoc/pyessv-archive
# OR
make setup-pyessv-archive
```

You can also choose to install them to a location on disk other than the default:

```shell
git clone https://github.com/ES-DOC/pyessv-archive /some/other/place
# OR
PYESSV_ARCHIVE_HOME=/some/other/place make setup-pyessv-archive
```

*Note*: <br>
If you have installed the [pyessv-archive](https://github.com/ES-DOC/pyessv-archive) data files to a non-default
location, you need to specify that location with the `PYESSV_ARCHIVE_HOME` environment variable. For example,
if you've installed the pyessv-archive files to `/some/other/place` then run the following before executing
any of the example commands above:

```shell
export PYESSV_ARCHIVE_HOME=/some/other/place
```

### Docker

You can also employ the pre-built Docker, which can be called as follows,
where `[command]` corresponds to any of the above example operations.

Expand Down
8 changes: 7 additions & 1 deletion STACpopulator/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import logging
import os
import sys
import warnings
from datetime import datetime
from http import cookiejar
from typing import Callable, Optional
Expand All @@ -13,6 +14,7 @@
from requests.sessions import Session

from STACpopulator import __version__
from STACpopulator.exceptions import STACPopulatorError
from STACpopulator.logging import setup_logging

POPULATORS = {}
Expand Down Expand Up @@ -151,7 +153,11 @@ def make_run_command_parser(parent) -> argparse.ArgumentParser:
populator_name, pop_mod_file = populator_py_mod.rsplit(".", 1)
populator_root = f"STACpopulator.{populators_impl}.{populator_name}"
pop_mod_file_loc = f"{populator_root}.{pop_mod_file}"
populator_module = importlib.import_module(pop_mod_file_loc, populator_root)
try:
populator_module = importlib.import_module(pop_mod_file_loc, populator_root)
except STACPopulatorError as e:
warnings.warn(f"Could not load extension {populator_name} because of error {e}")
continue
parser_maker: Callable[[], argparse.ArgumentParser] = getattr(populator_module, "make_parser", None)
populator_runner = getattr(populator_module, "runner", None) # optional, call main directly if not available
populator_caller = getattr(populator_module, "main", None)
Expand Down
5 changes: 5 additions & 0 deletions STACpopulator/exceptions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
class STACPopulatorError(Exception):
pass

class ExtensionLoadError(STACPopulatorError):
pass
7 changes: 6 additions & 1 deletion STACpopulator/extensions/cmip6.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
get_args,
)

import pyessv
import pystac
from pydantic import (
AnyHttpUrl,
Expand All @@ -34,6 +33,7 @@
SummariesExtension,
)

from STACpopulator.exceptions import ExtensionLoadError
from STACpopulator.models import AnyGeometry
from STACpopulator.stac_utils import (
ServiceType,
Expand All @@ -42,6 +42,11 @@
ncattrs_to_geometry,
)

try:
import pyessv
except OSError as e:
raise ExtensionLoadError(str(e)) from e

T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition)

SchemaName = Literal["cmip6"]
Expand Down
1 change: 0 additions & 1 deletion pyessv-archive
Submodule pyessv-archive deleted from 32175d

0 comments on commit c291918

Please sign in to comment.