Merge pull request #379 from effigies/enh/data-loader

ENH: Add data loader to sdcflows.data, drop pkg_resources

docs/api.rst

@@ -7,6 +7,7 @@ Information on specific functions, classes, and methods.
    :glob:
 
    api/sdcflows.cli
+   api/sdcflows.data
    api/sdcflows.fieldmaps
    api/sdcflows.interfaces
    api/sdcflows.transform

docs/conf.py

@@ -83,7 +83,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -218,7 +218,7 @@
 
 apidoc_module_dir = "../sdcflows"
 apidoc_output_dir = "api"
-apidoc_excluded_paths = ["conftest.py", "*/tests/*", "tests/*", "data/*"]
+apidoc_excluded_paths = ["conftest.py", "*/tests/*", "tests/*"]
 apidoc_separate_modules = True
 apidoc_extra_args = ["--module-first", "-d 1", "-T"]
 

docs/requirements.txt

@@ -1,5 +1,5 @@
 attrs >= 20.1.0
-furo ~= 2021.10.09
+furo
 matplotlib >= 2.2.0
 nibabel
 nipype >= 1.5.1
@@ -9,7 +9,6 @@ numpy
 packaging
 pydot >= 1.2.3
 pydotplus
-sphinx ~= 4.2
+sphinx
 sphinxcontrib-apidoc
-sphinxcontrib-napoleon
 templateflow

sdcflows/__init__.py

@@ -4,11 +4,10 @@
 try:
     from ._version import __version__
 except ModuleNotFoundError:
-    from pkg_resources import get_distribution, DistributionNotFound
-
+    from importlib.metadata import version, PackageNotFoundError
     try:
-        __version__ = get_distribution(__packagename__).version
-    except DistributionNotFound:
-        __version__ = "unknown"
-    del get_distribution
-    del DistributionNotFound
+        __version__ = version(__packagename__)
+    except PackageNotFoundError:
+        __version__ = "0+unknown"
+    del version
+    del PackageNotFoundError

sdcflows/data/__init__.py

@@ -0,0 +1,182 @@
+"""SDCFlows data files
+
+.. autofunction:: load
+
+.. automethod:: load.readable
+
+.. automethod:: load.as_path
+
+.. automethod:: load.cached
+
+.. autoclass:: Loader
+"""
+from __future__ import annotations
+
+import atexit
+import os
+from contextlib import AbstractContextManager, ExitStack
+from functools import cached_property
+from pathlib import Path
+from types import ModuleType
+from typing import Union
+
+try:
+    from functools import cache
+except ImportError:  # PY38
+    from functools import lru_cache as cache
+
+try:  # Prefer backport to leave consistency to dependency spec
+    from importlib_resources import as_file, files
+except ImportError:
+    from importlib.resources import as_file, files  # type: ignore
+
+try:  # Prefer stdlib so Sphinx can link to authoritative documentation
+    from importlib.resources.abc import Traversable
+except ImportError:
+    from importlib_resources.abc import Traversable
+
+__all__ = ["load"]
+
+
+class Loader:
+    """A loader for package files relative to a module
+
+    This class wraps :mod:`importlib.resources` to provide a getter
+    function with an interpreter-lifetime scope. For typical packages
+    it simply passes through filesystem paths as :class:`~pathlib.Path`
+    objects. For zipped distributions, it will unpack the files into
+    a temporary directory that is cleaned up on interpreter exit.
+
+    This loader accepts a fully-qualified module name or a module
+    object.
+
+    Expected usage::
+
+        '''Data package
+
+        .. autofunction:: load_data
+
+        .. automethod:: load_data.readable
+
+        .. automethod:: load_data.as_path
+
+        .. automethod:: load_data.cached
+        '''
+
+        from sdcflows.data import Loader
+
+        load_data = Loader(__package__)
+
+    :class:`~Loader` objects implement the :func:`callable` interface
+    and generate a docstring, and are intended to be treated and documented
+    as functions.
+
+    For greater flexibility and improved readability over the ``importlib.resources``
+    interface, explicit methods are provided to access resources.
+
+    +---------------+----------------+------------------+
+    | On-filesystem | Lifetime       | Method           |
+    +---------------+----------------+------------------+
+    | `True`        | Interpreter    | :meth:`cached`   |
+    +---------------+----------------+------------------+
+    | `True`        | `with` context | :meth:`as_path`  |
+    +---------------+----------------+------------------+
+    | `False`       | n/a            | :meth:`readable` |
+    +---------------+----------------+------------------+
+
+    It is also possible to use ``Loader`` directly::
+
+        from sdcflows.data import Loader
+
+        Loader(other_package).readable('data/resource.ext').read_text()
+
+        with Loader(other_package).as_path('data') as pkgdata:
+            # Call function that requires full Path implementation
+            func(pkgdata)
+
+        # contrast to
+
+        from importlib_resources import files, as_file
+
+        files(other_package).joinpath('data/resource.ext').read_text()
+
+        with as_file(files(other_package) / 'data') as pkgdata:
+            func(pkgdata)
+
+    .. automethod:: readable
+
+    .. automethod:: as_path
+
+    .. automethod:: cached
+    """
+
+    def __init__(self, anchor: Union[str, ModuleType]):
+        self._anchor = anchor
+        self.files = files(anchor)
+        self.exit_stack = ExitStack()
+        atexit.register(self.exit_stack.close)
+        # Allow class to have a different docstring from instances
+        self.__doc__ = self._doc
+
+    @cached_property
+    def _doc(self):
+        """Construct docstring for instances
+
+        Lists the public top-level paths inside the location, where
+        non-public means has a `.` or `_` prefix or is a 'tests'
+        directory.
+        """
+        top_level = sorted(
+            os.path.relpath(p, self.files) + "/"[: p.is_dir()]
+            for p in self.files.iterdir()
+            if p.name[0] not in (".", "_") and p.name != "tests"
+        )
+        doclines = [
+            f"Load package files relative to ``{self._anchor}``.",
+            "",
+            "This package contains the following (top-level) files/directories:",
+            "",
+            *(f"* ``{path}``" for path in top_level),
+        ]
+
+        return "\n".join(doclines)
+
+    def readable(self, *segments) -> Traversable:
+        """Provide read access to a resource through a Path-like interface.
+
+        This file may or may not exist on the filesystem, and may be
+        efficiently used for read operations, including directory traversal.
+
+        This result is not cached or copied to the filesystem in cases where
+        that would be necessary.
+        """
+        return self.files.joinpath(*segments)
+
+    def as_path(self, *segments) -> AbstractContextManager[Path]:
+        """Ensure data is available as a :class:`~pathlib.Path`.
+
+        This method generates a context manager that yields a Path when
+        entered.
+
+        This result is not cached, and any temporary files that are created
+        are deleted when the context is exited.
+        """
+        return as_file(self.files.joinpath(*segments))
+
+    @cache
+    def cached(self, *segments) -> Path:
+        """Ensure data is available as a :class:`~pathlib.Path`.
+
+        Any temporary files that are created remain available throughout
+        the duration of the program, and are deleted when Python exits.
+
+        Results are cached so that multiple calls do not unpack the same
+        data multiple times, but the cache is sensitive to the specific
+        argument(s) passed.
+        """
+        return self.exit_stack.enter_context(as_file(self.files.joinpath(*segments)))
+
+    __call__ = cached
+
+
+load = Loader(__package__)

sdcflows/tests/test_version.py

@@ -22,8 +22,7 @@
 #
 """Test _version.py."""
 import sys
-from collections import namedtuple
-from pkg_resources import DistributionNotFound
+from importlib.metadata import PackageNotFoundError
 from importlib import reload
 import sdcflows
 
@@ -40,25 +39,24 @@ class _version:
 
 
 def test_version_scm1(monkeypatch):
-    """Retrieve the version via pkg_resources."""
+    """Retrieve the version via importlib.metadata."""
     monkeypatch.setitem(sys.modules, "sdcflows._version", None)
 
-    def _dist(name):
-        Distribution = namedtuple("Distribution", ["name", "version"])
-        return Distribution(name, "success")
+    def _version(name):
+        return "9.0.0"
 
-    monkeypatch.setattr("pkg_resources.get_distribution", _dist)
+    monkeypatch.setattr("importlib.metadata.version", _version)
     reload(sdcflows)
-    assert sdcflows.__version__ == "success"
+    assert sdcflows.__version__ == "9.0.0"
 
 
 def test_version_scm2(monkeypatch):
     """Check version could not be interpolated."""
     monkeypatch.setitem(sys.modules, "sdcflows._version", None)
 
     def _raise(name):
-        raise DistributionNotFound("No get_distribution mock")
+        raise PackageNotFoundError("No get_distribution mock")
 
-    monkeypatch.setattr("pkg_resources.get_distribution", _raise)
+    monkeypatch.setattr("importlib.metadata.version", _raise)
     reload(sdcflows)
-    assert sdcflows.__version__ == "unknown"
+    assert sdcflows.__version__ == "0+unknown"

sdcflows/workflows/apply/registration.py

@@ -29,11 +29,12 @@
 The target EPI is the distorted dataset (or a reference thereof).
 
 """
-from pkg_resources import resource_filename as pkgrf
 from nipype.pipeline import engine as pe
 from nipype.interfaces import utility as niu
 from niworkflows.engine.workflows import LiterateWorkflow as Workflow
 
+from ... import data
+
 
 def init_coeff2epi_wf(
     omp_nthreads,
@@ -111,13 +112,9 @@ def init_coeff2epi_wf(
 
     # Register the reference of the fieldmap to the reference
     # of the target image (the one that shall be corrected)
-    ants_settings = pkgrf(
-        "sdcflows", f"data/fmap-any_registration{'_testing' * sloppy}.json"
-    )
-
     coregister = pe.Node(
         Registration(
-            from_file=ants_settings,
+            from_file=data.load(f"fmap-any_registration{'_testing' * sloppy}.json"),
             output_warped_image=debug,
             output_inverse_warped_image=debug,
         ),

sdcflows/workflows/fit/pepolar.py

@@ -21,12 +21,13 @@
 #     https://www.nipreps.org/community/licensing/
 #
 """Datasets with multiple phase encoded directions."""
-from pkg_resources import resource_filename as _pkg_fname
 from nipype.pipeline import engine as pe
 from nipype.interfaces import utility as niu
 
 from niworkflows.engine.workflows import LiterateWorkflow as Workflow
 
+from ... import data
+
 INPUT_FIELDS = ("metadata", "in_data")
 _PEPOLAR_DESC = """\
 A *B<sub>0</sub>*-nonuniformity map (or *fieldmap*) was estimated based on two (or more)
@@ -148,7 +149,7 @@ def init_topup_wf(
     to_las = pe.Node(ReorientImageAndMetadata(target_orientation="LAS"), name="to_las")
     topup = pe.Node(
         TOPUP(
-            config=_pkg_fname("sdcflows", f"data/flirtsch/b02b0{'_quick' * sloppy}.cnf")
+            config=str(data.load(f"flirtsch/b02b0{'_quick' * sloppy}.cnf"))
         ),
         name="topup",
     )
@@ -332,7 +333,7 @@ def init_3dQwarp_wf(omp_nthreads=1, debug=False, name="pepolar_estimate_wf"):
 
     align_pes = pe.Node(
         Registration(
-            from_file=_pkg_fname("sdcflows", "data/translation_rigid.json"),
+            from_file=data.load("translation_rigid.json"),
             output_warped_image=True,
         ),
         name="align_pes",