SCHEMACODE: Add data loader class (#1506)

* ENH: Add data loader to bst.data and bst.tests.data

* RF: Use data loaders in place of __file__

* DOC: Install pyparsing, mock pytest

tools/schemacode/bidsschematools/data/__init__.py

@@ -0,0 +1,99 @@
+"""Module containing data files, including the schema source files
+
+.. autofunction:: load_resource
+"""
+import atexit
+import os
+from contextlib import ExitStack
+from functools import cached_property
+from pathlib import Path
+from types import ModuleType
+from typing import Union
+
+try:
+    from functools import cache
+except ImportError:  # PY38
+    from functools import lru_cache as cache
+
+try:  # Prefer backport to leave consistency to dependency spec
+    from importlib_resources import as_file, files
+except ImportError:
+    from importlib.resources import as_file, files  # type: ignore
+
+__all__ = ["load_resource"]
+
+
+class Loader:
+    """A loader for package files relative to a module
+
+    This class wraps :mod:`importlib.resources` to provide a getter
+    function with an interpreter-lifetime scope. For typical packages
+    it simply passes through filesystem paths as :class:`~pathlib.Path`
+    objects. For zipped distributions, it will unpack the files into
+    a temporary directory that is cleaned up on interpreter exit.
+
+    This loader accepts a fully-qualified module name or a module
+    object.
+
+    Expected usage::
+
+        '''Data package
+
+        .. autofunction:: load_data
+        '''
+
+        from bidsschematools.data import Loader
+
+        load_data = Loader(__package__)
+
+    :class:`~Loader` objects implement the :func:`callable` interface
+    and generate a docstring, and are intended to be treated and documented
+    as functions.
+    """
+
+    def __init__(self, anchor: Union[str, ModuleType]):
+        self._anchor = anchor
+        self.files = files(anchor)
+        self.exit_stack = ExitStack()
+        atexit.register(self.exit_stack.close)
+        # Allow class to have a different docstring from instances
+        self.__doc__ = self._doc
+
+    @cached_property
+    def _doc(self):
+        """Construct docstring for instances
+
+        Lists the public top-level paths inside the location, where
+        non-public means has a `.` or `_` prefix or is a 'tests'
+        directory.
+        """
+        top_level = sorted(
+            os.path.relpath(p, self.files) + "/"[: p.is_dir()]
+            for p in self.files.iterdir()
+            if p.name[0] not in (".", "_") and p.name != "tests"
+        )
+        doclines = [
+            f"Load package files relative to ``{self._anchor}``.",
+            "",
+            "This package contains the following (top-level) files/directories:",
+            "",
+            *(f"* ``{path}``" for path in top_level),
+        ]
+
+        return "\n".join(doclines)
+
+    @cache
+    def __call__(self, *segments) -> Path:
+        """Ensure data is available as a :class:`~pathlib.Path`.
+
+        Any temporary files that are created remain available throughout
+        the duration of the program, and are deleted when Python exits.
+
+        Results are cached so that multiple calls do not unpack the same
+        data multiple times, but the cache is sensitive to the specific
+        argument(s) passed.
+        """
+        return self.exit_stack.enter_context(as_file(self.files.joinpath(*segments)))
+
+
+load_resource = Loader(__package__)

tools/schemacode/bidsschematools/tests/data/__init__.py

@@ -0,0 +1,9 @@
+"""Test data module
+
+.. autofunction:: load_test_data
+"""
+from ...data import Loader
+
+__all__ = ("load_test_data",)
+
+load_test_data = Loader(__package__)

tools/schemacode/bidsschematools/tests/test_schema.py

@@ -6,15 +6,12 @@
 
 from bidsschematools import __bids_version__, schema, types
 
+from ..data import load_resource
+
 
 def test__get_bids_version(tmp_path):
     # Is the version being read in correctly?
-    schema_path = os.path.join(
-        os.path.abspath(os.path.dirname(__file__)),
-        os.pardir,
-        "data",
-        "schema",
-    )
+    schema_path = str(load_resource("schema"))
     bids_version = schema._get_bids_version(schema_path)
     assert bids_version == __bids_version__
 

tools/schemacode/bidsschematools/tests/test_validator.py

@@ -6,6 +6,9 @@
 from bidsschematools.conftest import BIDS_ERROR_SELECTION, BIDS_SELECTION
 from bidsschematools.validator import select_schema_path, validate_bids
 
+from ..data import load_resource
+from .data import load_test_data
+
 
 def test_inheritance_examples():
     correct_inheritance = [
@@ -68,20 +71,11 @@ def test_write_report(tmp_path):
         "rawdata/sub-EXC022/anat/sub-EXC022_ses-MRI_flip-1_VFA.nii.gz"
     ]
 
-    report_path = os.path.join(
-        tmp_path,
-        "output_bids_validator_xs_write.log",
-    )
-    expected_report_path = os.path.join(
-        os.path.abspath(os.path.dirname(__file__)),
-        "data/expected_bids_validator_xs_write.log",
-    )
-    write_report(validation_result, report_path=report_path)
-    with open(report_path, "r") as f:
-        report_text = f.read()
-    with open(expected_report_path, "r") as f:
-        expected_report_text = f.read()
-    assert report_text == expected_report_text
+    report_path = tmp_path / "output_bids_validator_xs_write.log"
+    write_report(validation_result, report_path=str(report_path))
+
+    expected_report_path = load_test_data("expected_bids_validator_xs_write.log")
+    assert report_path.read_text() == expected_report_path.read_text()
 
 
 @pytest.mark.skipif(
@@ -125,12 +119,7 @@ def test_validate_bids(bids_examples, tmp_path):
     assert len(result["path_tracking"]) == 0
 
     # Is the schema version recorded correctly?
-    schema_path = os.path.join(
-        os.path.abspath(os.path.dirname(__file__)),
-        os.pardir,
-        "data",
-        "schema",
-    )
+    schema_path = load_resource("schema")
     with open(os.path.join(schema_path, "BIDS_VERSION")) as f:
         expected_version = f.readline().rstrip()
     assert result["bids_version"] == expected_version
@@ -148,10 +137,7 @@ def test_broken_json_dataset(bids_examples, tmp_path):
     dataset_path = os.path.join(bids_examples, dataset)
     dataset_json = os.path.join(dataset_path, "dataset_description.json")
 
-    broken_json = os.path.join(
-        os.path.abspath(os.path.dirname(__file__)),
-        "data/broken_dataset_description.json",
-    )
+    broken_json = load_test_data("broken_dataset_description.json")
     shutil.copyfile(broken_json, dataset_json)
 
     # No assert, will simply raise JSON reader error if not catching it properly.

tools/schemacode/bidsschematools/utils.py

@@ -1,6 +1,7 @@
 """Utility functions for the bids-specification schema."""
 import logging
-import os.path as op
+
+from . import data
 
 
 def get_bundled_schema_path():
@@ -11,7 +12,7 @@ def get_bundled_schema_path():
     str
         Absolute path to the directory containing schema-related files.
     """
-    return op.abspath(op.join(op.dirname(__file__), "data", "schema"))
+    return str(data.load_resource("schema"))
 
 
 def get_logger(name=None):

tools/schemacode/docs/conf.py

@@ -51,8 +51,9 @@
 exclude_patterns = ["_build", "tests"]
 
 
-# Mock internal modules to avoid building docs
 autosummary_mock_imports = [
+    "pytest",
+    # Mock internal modules to avoid building docs
     "bidsschematools.conftest",
 ]
 

tools/schemacode/docs/requirements.txt

@@ -1,3 +1,4 @@
 sphinx
 furo
 myst_parser
+pyparsing