From 36ea588cc6431c4ad3c5a22b153b43cf6bb5d143 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 11 Oct 2023 21:19:34 -0400 Subject: [PATCH 1/6] MNT: Drop Python 3.7, test on 3.12 --- .github/workflows/pythonpackage.yml | 8 ++++---- setup.cfg | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 097707f4..5f274a07 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -62,12 +62,12 @@ jobs: needs: build strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] mode: ['wheel'] include: - - {python-version: '3.9', mode: 'repo'} - - {python-version: '3.9', mode: 'sdist'} - - {python-version: '3.9', mode: 'editable'} + - {python-version: '3.11', mode: 'repo'} + - {python-version: '3.11', mode: 'sdist'} + - {python-version: '3.11', mode: 'editable'} env: TEMPLATEFLOW_HOME: /tmp/home diff --git a/setup.cfg b/setup.cfg index ba4420eb..e8b7256a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,10 +4,11 @@ classifiers = Intended Audience :: Science/Research Topic :: Scientific/Engineering :: Image Recognition License :: OSI Approved :: Apache Software License - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 description = TemplateFlow Python Client - TemplateFlow is the Zone of neuroimaging templates. license = Apache-2.0 license_file = LICENSE @@ -24,7 +25,7 @@ project_urls = Source Code = https://github.com/templateflow/python-client [options] -python_requires = >= 3.7 +python_requires = >= 3.8 setup_requires = setuptools >= 45 setuptools_scm >= 6.2 From 41388835eb2d0dd6529308ed9c568a1bf4e2148a Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 11 Oct 2023 21:21:10 -0400 Subject: [PATCH 2/6] MNT: Use importlib.metadata instead of pkg_resources Use PEP-440 compatible fallback --- templateflow/__init__.py | 13 ++++++------- templateflow/tests/test_version.py | 17 ++++++++--------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/templateflow/__init__.py b/templateflow/__init__.py index dd99b56d..6cec2933 100644 --- a/templateflow/__init__.py +++ b/templateflow/__init__.py @@ -6,14 +6,13 @@ try: from ._version import __version__ except ModuleNotFoundError: - from pkg_resources import get_distribution, DistributionNotFound - + from importlib.metadata import version, PackageNotFoundError try: - __version__ = get_distribution(__packagename__).version - except DistributionNotFound: - __version__ = "unknown" - del get_distribution - del DistributionNotFound + __version__ = version(__packagename__) + except PackageNotFoundError: + __version__ = "0+unknown" + del version + del PackageNotFoundError import os from . import api diff --git a/templateflow/tests/test_version.py b/templateflow/tests/test_version.py index f3990541..30604d73 100644 --- a/templateflow/tests/test_version.py +++ b/templateflow/tests/test_version.py @@ -1,7 +1,7 @@ """Test _version.py.""" import sys from collections import namedtuple -from pkg_resources import DistributionNotFound +from importlib.metadata import PackageNotFoundError from importlib import reload import templateflow @@ -18,14 +18,13 @@ class _version: def test_version_scm1(monkeypatch): - """Retrieve the version via pkg_resources.""" + """Retrieve the version via importlib.metadata.""" monkeypatch.setitem(sys.modules, "templateflow._version", None) - def _dist(name): - Distribution = namedtuple("Distribution", ["name", "version"]) - return Distribution(name, "success") + def _ver(name): + return "success" - monkeypatch.setattr("pkg_resources.get_distribution", _dist) + monkeypatch.setattr("importlib.metadata.version", _ver) reload(templateflow) assert templateflow.__version__ == "success" @@ -35,8 +34,8 @@ def test_version_scm2(monkeypatch): monkeypatch.setitem(sys.modules, "templateflow._version", None) def _raise(name): - raise DistributionNotFound("No get_distribution mock") + raise PackageNotFoundError("No get_distribution mock") - monkeypatch.setattr("pkg_resources.get_distribution", _raise) + monkeypatch.setattr("importlib.metadata.version", _raise) reload(templateflow) - assert templateflow.__version__ == "unknown" + assert templateflow.__version__ == "0+unknown" From 6c945cf4630a6bfd830a90fe07f4bcf6fd4a446c Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 11 Oct 2023 21:29:56 -0400 Subject: [PATCH 3/6] ENH: Add templateflow._loader to replace pkg_resources --- setup.cfg | 1 + templateflow/_loader.py | 171 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 templateflow/_loader.py diff --git a/setup.cfg b/setup.cfg index e8b7256a..a2964336 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,6 +32,7 @@ setup_requires = wheel install_requires = pybids >= 0.15.2 + importlib_resources >= 5.7; python_version < '3.11' requests tqdm test_requires = diff --git a/templateflow/_loader.py b/templateflow/_loader.py new file mode 100644 index 00000000..bf2534a6 --- /dev/null +++ b/templateflow/_loader.py @@ -0,0 +1,171 @@ +"""Resource loader module + +.. autoclass:: Loader +""" +from __future__ import annotations + +import atexit +import os +from contextlib import AbstractContextManager, ExitStack +from functools import cached_property +from pathlib import Path +from types import ModuleType +from typing import Union + +try: + from functools import cache +except ImportError: # PY38 + from functools import lru_cache as cache + +try: # Prefer backport to leave consistency to dependency spec + from importlib_resources import as_file, files +except ImportError: + from importlib.resources import as_file, files # type: ignore + +try: # Prefer stdlib so Sphinx can link to authoritative documentation + from importlib.resources.abc import Traversable +except ImportError: + from importlib_resources.abc import Traversable + +__all__ = ["Loader"] + + +class Loader: + """A loader for package files relative to a module + + This class wraps :mod:`importlib.resources` to provide a getter + function with an interpreter-lifetime scope. For typical packages + it simply passes through filesystem paths as :class:`~pathlib.Path` + objects. For zipped distributions, it will unpack the files into + a temporary directory that is cleaned up on interpreter exit. + + This loader accepts a fully-qualified module name or a module + object. + + Expected usage:: + + '''Data package + + .. autofunction:: load_data + + .. automethod:: load_data.readable + + .. automethod:: load_data.as_path + + .. automethod:: load_data.cached + ''' + + from templateflow._loader import Loader + + load_data = Loader(__package__) + + :class:`~Loader` objects implement the :func:`callable` interface + and generate a docstring, and are intended to be treated and documented + as functions. + + For greater flexibility and improved readability over the ``importlib.resources`` + interface, explicit methods are provided to access resources. + + +---------------+----------------+------------------+ + | On-filesystem | Lifetime | Method | + +---------------+----------------+------------------+ + | `True` | Interpreter | :meth:`cached` | + +---------------+----------------+------------------+ + | `True` | `with` context | :meth:`as_path` | + +---------------+----------------+------------------+ + | `False` | n/a | :meth:`readable` | + +---------------+----------------+------------------+ + + It is also possible to use ``Loader`` directly:: + + from templateflow._loader import Loader + + Loader(other_package).readable('data/resource.ext').read_text() + + with Loader(other_package).as_path('data') as pkgdata: + # Call function that requires full Path implementation + func(pkgdata) + + # contrast to + + from importlib_resources import files, as_file + + files(other_package).joinpath('data/resource.ext').read_text() + + with as_file(files(other_package) / 'data') as pkgdata: + func(pkgdata) + + .. automethod:: readable + + .. automethod:: as_path + + .. automethod:: cached + """ + + def __init__(self, anchor: Union[str, ModuleType]): + self._anchor = anchor + self.files = files(anchor) + self.exit_stack = ExitStack() + atexit.register(self.exit_stack.close) + # Allow class to have a different docstring from instances + self.__doc__ = self._doc + + @cached_property + def _doc(self): + """Construct docstring for instances + + Lists the public top-level paths inside the location, where + non-public means has a `.` or `_` prefix or is a 'tests' + directory. + """ + top_level = sorted( + os.path.relpath(p, self.files) + "/"[: p.is_dir()] + for p in self.files.iterdir() + if p.name[0] not in (".", "_") and p.name != "tests" + ) + doclines = [ + f"Load package files relative to ``{self._anchor}``.", + "", + "This package contains the following (top-level) files/directories:", + "", + *(f"* ``{path}``" for path in top_level), + ] + + return "\n".join(doclines) + + def readable(self, *segments) -> Traversable: + """Provide read access to a resource through a Path-like interface. + + This file may or may not exist on the filesystem, and may be + efficiently used for read operations, including directory traversal. + + This result is not cached or copied to the filesystem in cases where + that would be necessary. + """ + return self.files.joinpath(*segments) + + def as_path(self, *segments) -> AbstractContextManager[Path]: + """Ensure data is available as a :class:`~pathlib.Path`. + + This method generates a context manager that yields a Path when + entered. + + This result is not cached, and any temporary files that are created + are deleted when the context is exited. + """ + return as_file(self.files.joinpath(*segments)) + + @cache + def cached(self, *segments) -> Path: + """Ensure data is available as a :class:`~pathlib.Path`. + + Any temporary files that are created remain available throughout + the duration of the program, and are deleted when Python exits. + + Results are cached so that multiple calls do not unpack the same + data multiple times, but the cache is sensitive to the specific + argument(s) passed. + """ + return self.exit_stack.enter_context(as_file(self.files.joinpath(*segments))) + + __call__ = cached From e8a5c108e919246ce574cb3be5531c9e7c8dca7d Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 11 Oct 2023 21:35:51 -0400 Subject: [PATCH 4/6] RF: Drop pkg_resources for Loader --- templateflow/conf/__init__.py | 3 +++ templateflow/conf/_s3.py | 9 ++++----- templateflow/conf/bids.py | 5 +++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/templateflow/conf/__init__.py b/templateflow/conf/__init__.py index 2d782cb8..cf239c59 100644 --- a/templateflow/conf/__init__.py +++ b/templateflow/conf/__init__.py @@ -5,6 +5,9 @@ from pathlib import Path from contextlib import suppress from functools import wraps +from .._loader import Loader + +load_data = Loader(__package__) TF_DEFAULT_HOME = Path.home() / ".cache" / "templateflow" TF_HOME = Path(getenv("TEMPLATEFLOW_HOME", str(TF_DEFAULT_HOME))) diff --git a/templateflow/conf/_s3.py b/templateflow/conf/_s3.py index 4051ce86..078cd9e7 100644 --- a/templateflow/conf/_s3.py +++ b/templateflow/conf/_s3.py @@ -1,16 +1,15 @@ """Tooling to handle S3 downloads.""" from pathlib import Path from tempfile import mkstemp -from pkg_resources import resource_filename + +from . import load_data TF_SKEL_URL = ( "https://raw.githubusercontent.com/templateflow/python-client/" "{release}/templateflow/conf/templateflow-skel.{ext}" ).format -TF_SKEL_PATH = Path(resource_filename("templateflow", "conf/templateflow-skel.zip")) -TF_SKEL_MD5 = Path( - resource_filename("templateflow", "conf/templateflow-skel.md5") -).read_text() +TF_SKEL_PATH = load_data("templateflow-skel.zip") +TF_SKEL_MD5 = load_data.readable("templateflow-skel.md5").read_text() def update(dest, local=True, overwrite=True, silent=False): diff --git a/templateflow/conf/bids.py b/templateflow/conf/bids.py index a400b588..8326d348 100644 --- a/templateflow/conf/bids.py +++ b/templateflow/conf/bids.py @@ -1,8 +1,9 @@ """Extending pyBIDS for querying TemplateFlow.""" -from pkg_resources import resource_filename from bids.layout import BIDSLayout, add_config_paths -add_config_paths(templateflow=resource_filename("templateflow", "conf/config.json")) +from . import load_data + +add_config_paths(templateflow=load_data("config.json")) class Layout(BIDSLayout): From 4df632765bfb8942334571542282c71ba08726ac Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 11 Oct 2023 21:52:38 -0400 Subject: [PATCH 5/6] CI: Update installation requirements for tests --- .circleci/config.yml | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a5354d8..d57ea35c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: source /tmp/venv/bin/activate pip install -U pip pip install -r /tmp/src/templateflow/requirements.txt - pip install "datalad ~= 0.11.8" + pip install datalad pip install "setuptools>=45" "setuptools_scm >= 6.2" nipreps-versions build twine codecov - run: diff --git a/requirements.txt b/requirements.txt index 71abf0f0..fd8235af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ pybids>=0.15.2 +importlib_resources >= 5.7; python_version < '3.11' requests tqdm pytest From 9bd9d27139ecd4d5b4c9803890c943a88de98bab Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 11 Oct 2023 22:11:10 -0400 Subject: [PATCH 6/6] MNT: Ignore coverage of fallback imports --- templateflow/_loader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templateflow/_loader.py b/templateflow/_loader.py index bf2534a6..a9f2d521 100644 --- a/templateflow/_loader.py +++ b/templateflow/_loader.py @@ -14,17 +14,17 @@ try: from functools import cache -except ImportError: # PY38 +except ImportError: # PY38 # pragma: no cover from functools import lru_cache as cache try: # Prefer backport to leave consistency to dependency spec from importlib_resources import as_file, files -except ImportError: +except ImportError: # pragma: no cover from importlib.resources import as_file, files # type: ignore try: # Prefer stdlib so Sphinx can link to authoritative documentation from importlib.resources.abc import Traversable -except ImportError: +except ImportError: # pragma: no cover from importlib_resources.abc import Traversable __all__ = ["Loader"]