diff --git a/.readthedocs.yml b/.readthedocs.yml index f19e80b4..fe4c76e3 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,9 +9,13 @@ version: 2 sphinx: configuration: docs/source/conf.py +build: + os: ubuntu-22.04 + tools: + python: "3.11" + # Optionally set the version of Python and requirements required to build your docs python: - version: 3.8 install: - requirements: docs/requirements.txt - method: pip diff --git a/docs/Makefile b/docs/Makefile index 9b5b6042..28667a29 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -175,3 +175,6 @@ pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +live: + sphinx-autobuild -b dirhtml source/ _build/dirhtml/ diff --git a/docs/requirements.txt b/docs/requirements.txt index 5e03d002..83da9d70 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ sphinx>=3.1 sphinx-autosummary-accessors -sphinx_rtd_theme +sphinx_rtd_theme>=1.0 +autodoc_pydantic diff --git a/docs/source/api.rst b/docs/source/api.rst index 7ebf6fa0..d5052970 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -8,7 +8,7 @@ Top-level Rest class ==================== The :class:`~xpublish.Rest` class can be used for publishing a -:class:`xarray.Dataset` object or a collection of Dataset objects. +a collection of :class:`xarray.Dataset` objects. .. autosummary:: :toctree: generated/ @@ -18,12 +18,19 @@ The :class:`~xpublish.Rest` class can be used for publishing a Rest.cache Rest.serve +For serving a single dataset the :class:`~xpublish.SingleDatasetRest` is used instead. + +.. autosummary:: + :toctree: generated/ + + SingleDatasetRest + Dataset.rest (xarray accessor) ============================== This accessor extends :py:class:`xarray.Dataset` with the same interface than -:class:`~xpublish.Rest`. It is a convenient method for publishing one single -dataset. Proper use of this accessor should be like: +:class:`~xpublish.Rest` or :class:`~xpublish.SingleDatasetRest`. It is a convenient +method for publishing one single dataset. Proper use of this accessor should be like: .. code-block:: python @@ -77,3 +84,22 @@ when creating custom API endpoints. get_cache get_zvariables get_zmetadata + +Plugins +======= + +Plugins are inherit from the :class:`~xpublish.Plugin` class, and implement various hooks. + +.. currentmodule:: xpublish + +.. autosummary:: + :toctree: generated/ + + Plugin + hookimpl + hookspec + Dependencies + plugins.hooks.PluginSpec + plugins.manage.find_default_plugins + plugins.manage.load_default_plugins + plugins.manage.configure_plugins diff --git a/docs/source/conf.py b/docs/source/conf.py index a6a7c284..6ef1711e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,6 +43,7 @@ 'sphinx.ext.intersphinx', 'sphinx.ext.extlinks', 'sphinx.ext.napoleon', + 'sphinxcontrib.autodoc_pydantic', 'sphinx_autosummary_accessors', ] diff --git a/docs/source/index.rst b/docs/source/index.rst index d8f86da5..fb6fe480 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -67,6 +67,7 @@ API with the following endpoints: installation tutorial + plugins api contributing diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst new file mode 100644 index 00000000..1293b768 --- /dev/null +++ b/docs/source/plugins.rst @@ -0,0 +1,329 @@ +======= +Plugins +======= + +While :py:class:`fastapi.APIRouter` can get you started building new endpoints +for datasets quickly, the real extendability of Xpublish comes from it's plugin system. + +By using a plugin system, Xpublish becomes incredibly adaptable, and hopefully +easier to develop for also. Individual plugins and their functionality can +evolve independently, there are clear boundaries between types of functionality, +which allows easier reasoning about code. + +There are a few main varieties of plugins that Xpublish supports, but those +provide a lot of flexibility, and can enable whole new categories of plugins +and functionality. + +* `Dataset router `_ +* `App router `_ +* `Dataset provider `_ +* `Hook spec `_ + +Plugins work by implementing specific methods to support a variety of usage, +and marking the implementations with a decorator. A plugin can also implement +methods for multiple varieties, which may be useful for things like dynamic +data providers. + +.. warning:: + + Plugins are new to Xpublish, so we're learning how everything works best together. + + If you have any questions, please ask in `Github Discussions + `_ + (and feel free to tag ``@abkfenris`` for help with the plugin system). + +------------- +Functionality +------------- + +Plugins are built as `Pydantic models `_ +and descend from :py:class:`xpublish.plugins.hooks.Plugin`. +This allows there to be a common way of configuring plugins and their functionality. + +.. code-block:: python + :emphasize-lines: 5 + + from xpublish import Plugin + + + class HelloWorldPlugin(Plugin): + name = "hello_world" + + +At the minimum, a plugin needs to specify a ``name`` attribute. + +Marking implementation methods +------------------------------ + +We'll go deeper into the specific methods below, what they have in common is that any +method that a plugin is hoping to expose to the rest of Xpublish needs to be marked +with a ``@hookimpl`` decorator. + +.. code-block:: python + :emphasize-lines: 7 + + from xpublish import Plugin, hookimpl + from fastapi import APIRouter + + class HelloWorldPlugin(Plugin): + name = "hello_world" + + @hookimpl + def app_router(self): + router = APIRouter() + + @router.get("/hello") + def get_hello(): + return "world" + + return router + +For the plugin system, Xpublish is using `pluggy `_. +Pluggy was developed to support `pytest `_, +but it now is used by several other projects including `Tox `_, +`Datasette `_, +and `Conda `_, among others. + +Pluggy implements plugins as a system of hooks, each one is a distinct way for Xpublish +to communicate with plugins. +Each hook has both reference specifications, and plugin provided implementations. + +Most of the specifications are provided by Xpublish and are methods on +:py:class:`xpublish.plugins.hooks.PluginSpec` that are marked with ``@hookspec``. + +Plugins can then re-implement these methods with all or a subset of the arguments, +which are then marked with ``@hookimpl`` +to tell Pluggy to make them accessible to Xpublish (and other plugins). + +.. note:: + + Over time Xpublish will most likely end up expanding the number of arugments passed + into most hook methods. + + Currently we're starting with a minimum set of arguments as we can always expand, + but currently it is much harder to reduce the number of arguments. + + If there is a new argument that you would like your plugin hooks to have, + please raise an `issue `_ + to discuss including it in a future version. + +In the specification, Xpublish defines if it's supposed to get responses from all +implementations (:py:meth:`xpublish.plugins.hooks.PluginSpec.get_dataset_ids`), +or the first non-``None`` response (:py:meth:`xpublish.plugins.hooks.PluginSpec.get_dataset`). + +Pluggy also provides a lot more advanced functionality that we aren't going to go +into at this point, but could allow for creative things like dataset middleware. + + +Loading Local Plugins +--------------------- + +For plugins that you are not distributing, they can either be loaded directly via the +:py:class:`xpublish.Rest` initializer, or they can use +:py:meth:`xpublish.Rest.register_plugin` to load afterwards. + +.. code-block:: python + + from xpublish import Rest + + rest = Rest(datasets, plugins={"hello-world": HelloWorldPlugin()}) + +.. code-block:: python + + from xpublish import Rest + + rest = Rest(datasets) + rest.register_plugin(HelloWorldPlugin()) + +.. caution:: + + When plugins are provided directly to the :py:class:`xpublish.Rest` initializer + as keyword arguments, it prevents Xpublish from automatically loading other plugins + that are installed. + + For more details of the automatic plugin loading system, + see `entry points `_ below. + +Entry Points +------------ + +When you install a plugin library, the library takes advantage of the +`entry point system `_. + +This allows :py:class:`xpublish.Rest` to automatically find and use plugins. +It only does this if plugins **are not** provided as an keyword argument. + +:py:class:`xpublish.Rest` uses :py:func:`plugins.manage.load_default_plugins` to +load plugins from entry points. +It can be used directly and be set to disable specific plugins from being loaded, +or :py:func:`plugins.manage.find_default_plugins` and :py:func:`plugins.manage.configure_plugins`, +can be used to further tweak loading plugins from entrypoints. + +To completely disable loading of plugins from entry points pass an empty dictionary to +``xpublish.Rest(datasets, plugins={})``. + +Example Entry Point +******************* + +Using `xpublish-edr `_ as an example. + +The plugin is named ``CfEdrPlugin`` and is located in ``xpublish_edr/plugin.py``. + +In ``pyproject.toml`` that then is added to the ``[project.entry-points."xpublish.plugin"]`` table. + +.. code-block:: toml + + [project.entry-points."xpublish.plugin"] + cf_edr = "xpublish_edr.plugin:CfEdrPlugin" + +Dependencies +------------ + +To allow plugins to be more adaptable, they should use +:py:meth:`xpublish.Dependencies.dataset` rather than directly +importing :py:func:`xpublish.dependencies.get_dataset`. + +To facilitate this, :py:class:`xpublish.Dependencies` is passed into +router hook methods. + +.. code-block:: python + + from fastapi import APIRouter, Depends + from xpublish import Plugin, Dependencies, hookimpl + + class DatasetAttrs(Plugin): + name = "dataset-attrs" + + @hookimpl + def dataset_router(self, deps: Dependencies): + router = APIRouter() + + @router.get("/attrs") + def get_attrs(ds = Depends(deps.dataset)): + return ds.attrs + + return router + +:py:class:`xpublish.Dependencies` has several other types of dependency functions that +it includes. + +---------------------- +Dataset Router Plugins +---------------------- + +Dataset router plugins are the next step from passing routers into +:py:class:`xpublish.Rest`. + +By implementing :py:meth:`xpublish.plugins.hooks.PluginSpec.dataset_router` +a developer can add new routes that respond below ``/datasets//``. + +Most dataset routers will have a prefix on their paths, and apply tags. +To make this reasonably standard, those should be specified as ``dataset_router_prefix`` +and ``dataset_router_tags`` on the plugin allowing them to be reasonably overridden. + +Adapted from `xpublish/plugins/included/dataset_info.py `_ + +.. code-block:: python + + from fastapi import APIRouter, Depends + from xpublish import Plugin, Dependencies, hookimpl + + class DatasetInfoPlugin(Plugin): + name = "dataset-info" + + dataset_router_prefix = "/info" + dataset_router_tags = ["info"] + + @hookimpl + def dataset_router(self, deps: Dependencies): + router = APIRouter(prefix=self.dataset_router_prefix, tags=self.dataset_router_tags) + + @router.get("/keys") + def list_keys(dataset=Depends(deps.dataset): + return dataset.variables + + return router + +This plugin will respond to ``/datasets//info/keys`` with a list of the keys in the dataset. + + +------------------ +App Router Plugins +------------------ + +App routers allow new top level routes to be provided by implementing +:py:meth:`xpublish.plugins.hooks.PluginSpec.app_router`. + +Similar to dataset routers, these should have a prefix (``app_router_prefix``) and tags (``app_router_tags``) that can be user overridable. + +.. code-block:: python + + from fastapi import APIRouter, Depends + from xpublish import Plugin, Dependencies, hookimpl + + class PluginInfo(Plugin): + name = "plugin_info" + + app_router_prefix = "/info" + app_router_tags = ["info"] + + @hookimpl + def app_router(self, deps: Dependencies): + router = APIRouter(prefix=self.app_router_prefix, tags=self.app_router_tags) + + @router.get("/plugins") + def plugins(plugins: Dict[str, Plugin] = Depends(deps.plugins)): + return {name: type(plugin) for name, plugin in plugins.items} + + return router + +This will return a dictionary of plugin names, and types at `/info/plugins`. + +------------------------ +Dataset Provider Plugins +------------------------ + +While Xpublish can have datasets passed in to :py:class:`xpublish.Rest` on intialization, +plugins can provide datasets (and they actually have priority over those passed in directly). + +In order for a plugin to provide datasets it needs to implemenent +:py:meth:`xpublish.plugins.hooks.PluginSpec.get_datasets` +and :py:meth:`xpublish.plugins.hooks.PluginSpec.get_dataset` methods. + +The first should return a list of all datasets that a plugin knows about. + +The second is provided a ``dataset_id``. +The plugin should return a dataset if it knows about the dataset corresponding to the id, +otherwise it should return None, so that Xpublish knows to continue looking to the next +plugin or the passed in dictionary of datasets. + +A plugin that provides the Xarray tutorial ``air_temperature`` dataset. + +.. code-block:: python + + from xpublish import Plugin, hookimpl + + + class TutorialDataset(Plugin): + name = "xarray-tutorial-dataset" + + @hookimpl + def get_datasets(self): + return ["air"] + + @hookimpl + def get_dataset(self, dataset_id: str): + if dataset_id == "air": + return xr.tutorial.open_dataset("air_temperature") + + return None + + +----------------- +Hook Spec Plugins +----------------- + +Plugins can also provide new hook specifications that other plugins can then implement. +This allows Xpublish to support things that we haven't even thought of yet. + +These return a class of hookspecs from :py:meth:`xpublish.plugins.hooks.PluginSpec.register_hookspec`. diff --git a/xpublish/plugins/hooks.py b/xpublish/plugins/hooks.py index e35f4ec9..e59d1aa8 100644 --- a/xpublish/plugins/hooks.py +++ b/xpublish/plugins/hooks.py @@ -4,20 +4,42 @@ import pluggy # type: ignore import xarray as xr from fastapi import APIRouter -from pydantic import BaseModel +from pydantic import BaseModel, Field from ..dependencies import get_cache, get_dataset, get_dataset_ids, get_plugin_manager, get_plugins +# Decorator helper to mark functions as Xpublish hook specifications hookspec = pluggy.HookspecMarker('xpublish') + +# Decorator helper to mark functions as Xpublish hook implementations hookimpl = pluggy.HookimplMarker('xpublish') class Dependencies(BaseModel): - dataset_ids: Callable[..., List[str]] = get_dataset_ids - dataset: Callable[..., xr.Dataset] = get_dataset - cache: Callable[..., cachey.Cache] = get_cache - plugins: Callable[..., Dict[str, 'Plugin']] = get_plugins - plugin_manager: Callable[..., pluggy.PluginManager] = get_plugin_manager + """ + A set of dependencies that are passed into plugin routers. + + Some routers may be 'borrowed' by other routers to expose different + geometries of data, thus the default dependencies may need to be overridden. + By depending on the passed in version of this class, the dependencies + can be overridden predictably. + """ + + dataset_ids: Callable[..., List[str]] = Field( + get_dataset_ids, description='Returns a list of all valid dataset ids' + ) + dataset: Callable[[str], xr.Dataset] = Field( + get_dataset, description='Returns a dataset using ``//`` in the path.' + ) + cache: Callable[..., cachey.Cache] = Field( + get_cache, description='Provide access to :py:class:`cachey.Cache`' + ) + plugins: Callable[..., Dict[str, 'Plugin']] = Field( + get_plugins, description='A dictionary of plugins allowing direct access' + ) + plugin_manager: Callable[..., pluggy.PluginManager] = Field( + get_plugin_manager, description='The plugin manager itself, allowing for maximum creativity' + ) def __hash__(self): """Dependency functions aren't easy to hash""" @@ -29,14 +51,14 @@ class Plugin(BaseModel): Xpublish plugins provide ways to extend the core of xpublish with new routers and other functionality. - To create a plugin, subclass ``Plugin` and add attributes that are + To create a plugin, subclass `Plugin` and add attributes that are subclasses of `PluginType` (`Router` for instance). The specific attributes correspond to how Xpublish should use the plugin. """ - name: str + name: str = Field(..., description='Fallback name of plugin') def __hash__(self): """Make sure that the plugin is hashable to load with pluggy""" @@ -64,7 +86,11 @@ def __dir__(self) -> Iterable[str]: class PluginSpec(Plugin): - """Plugin extension points""" + """Plugin extension points + + Plugins do not need to implement all of the methods defined here, + instead they implement + """ @hookspec def app_router(self, deps: Dependencies) -> APIRouter: # type: ignore @@ -97,4 +123,4 @@ def get_dataset(self, dataset_id: str) -> Optional[xr.Dataset]: # type: ignore @hookspec def register_hookspec(self): # type: ignore - """Return additional hookspec classes to register with the plugin manager""" + """Return additional hookspec class to register with the plugin manager"""