diff --git a/.circleci/config.yml b/.circleci/config.yml index 135a2d05..0dfa7a6d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,44 +1,85 @@ -version: 2 +version: 2.1 +orbs: + docker: circleci/docker@2.7.0 + codecov: codecov/codecov@4.1.0 + python: circleci/python@2.1.1 + jobs: - tests: - docker: - - image: cimg/python:3.11 - auth: - username: $DOCKER_USER - password: $DOCKER_PAT - working_directory: /tmp/tests - environment: - - OSF_MIRROR_PATH: /tmp/data/templateflow - steps: - - checkout: - path: /tmp/src/templateflow + build: + executor: + name: python/default + tag: '3.12' + + # docker: + # - auth: + # username: $DOCKER_USERNAME + # password: $DOCKER_PASSWORD + steps: + - checkout - run: - name: Generate requirements.txt command: | - python /tmp/src/templateflow/.maint/update_requirements.py - - - restore_cache: - keys: - - deps-v11-{{ checksum "/tmp/src/templateflow/dev-requirements.txt"}}-{{ epoch }} - - deps-v11-{{ checksum "/tmp/src/templateflow/dev-requirements.txt"}}- - - deps-v11- + echo 'build' > requirements.txt + echo 'wheel' >> requirements.txt + echo 'twine' >> requirements.txt + - python/install-packages: + pkg-manager: pip + cache-version: build-v1 - run: - name: Prepare environment command: | - python -m venv /tmp/venv - source /tmp/venv/bin/activate - python -m pip install -U pip - python -m pip install -r /tmp/src/templateflow/dev-requirements.txt - python -m pip install -U "datalad ~= 0.19.0" - python -m pip install -U build hatch twine pkginfo codecov + python -m build + python -m twine check dist/* + name: Test packaging + - persist_to_workspace: + root: ~/project + paths: + - . + + deploy_pypi: + executor: + name: python/default + tag: '3.12' + + # docker: + # - auth: + # username: $DOCKER_USERNAME + # password: $DOCKER_PASSWORD + + steps: + - attach_workspace: + at: ~/project + + - python/install-packages: + args: twine + pkg-manager: pip + - run: + command: python -m twine upload dist/* + name: Upload to Pypi + + tests: + executor: + name: python/default + tag: '3.12' + + # docker: + # - auth: + # username: $DOCKER_USERNAME + # password: $DOCKER_PASSWORD + + environment: + - OSF_MIRROR_PATH: /tmp/data/templateflow + steps: + - restore_cache: + keys: + - annex-v1-{{ epoch }} + - annex-v1- - run: name: Install git and git-annex command: | if [[ ! -e "/tmp/cache/git-annex-standalone.tar.gz" ]]; then wget -O- http://neuro.debian.net/lists/focal.us-ca.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list - sudo apt-key add /tmp/src/templateflow/.neurodebian/neurodebian.gpg + sudo apt-key add .neurodebian/neurodebian.gpg sudo apt-key adv --recv-keys --keyserver hkps://keys.openpgp.org 0xA5D32F012649A5A9 || true sudo apt update && sudo apt-get install -y --no-install-recommends git-annex-standalone mkdir -p /tmp/cache @@ -50,83 +91,80 @@ jobs: git config --global user.email "email@domain.com" - save_cache: - key: deps-v11-{{ checksum "/tmp/src/templateflow/dev-requirements.txt"}}-{{ epoch }} + key: annex-v1-{{ epoch }} paths: - "/tmp/cache" - - "/tmp/venv" + + - attach_workspace: + at: ~/project - run: - name: Test packaging command: | - source /tmp/venv/bin/activate - cd /tmp/src/templateflow - python -m build - python -m twine check dist/* + python .maint/update_requirements.py + name: Generate requirements.txt + + - python/install-packages: + pip-dependency-file: dev-requirements.txt + pkg-manager: pip + cache-version: v1 - run: - name: Run tests (w/ DataLad) + name: Run tests (config and others) command: | - source /tmp/venv/bin/activate - export TEMPLATEFLOW_USE_DATALAD=on - pytest --junit-xml=/tmp/tests/datalad.xml --cov templateflow --cov-report xml:/tmp/cov_api_dl.xml \ - --doctest-modules /tmp/src/templateflow/templateflow + python -m pytest \ + --junit-xml=~/tests/conftests.xml --cov templateflow --cov-report xml:~/coverage/cov_config.xml \ + templateflow/tests/ + + - codecov/upload: + file: ~/coverage/cov_config.xml + flags: config + upload_name: General - run: - name: Submit api test coverage + name: Run tests (w/ DataLad) command: | - source /tmp/venv/bin/activate - codecov --file /tmp/cov_api_dl.xml --root /tmp/src/templateflow \ - --flags api,datalad -e CIRCLE_JOB + mkdir -p ~/tests/ ~/coverage/ + export TEMPLATEFLOW_USE_DATALAD=on + python -m pytest \ + --junit-xml=~/tests/datalad.xml --cov templateflow --cov-report xml:~/coverage/cov_api_dl.xml \ + --doctest-modules templateflow/api.py + + - codecov/upload: + file: ~/coverage/cov_api_dl.xml + flags: api,datalad + upload_name: Datalad tests - run: name: Run tests (pulling from S3) command: | - source /tmp/venv/bin/activate export TEMPLATEFLOW_USE_DATALAD=off export TEMPLATEFLOW_HOME=$HOME/templateflow-s3 - pytest --junit-xml=/tmp/tests/s3.xml --cov templateflow --cov-report xml:/tmp/cov_api_s3.xml \ - --doctest-modules /tmp/src/templateflow/templateflow/api.py + python -m pytest \ + --junit-xml=~/tests/s3.xml --cov templateflow --cov-report xml:~/coverage/cov_api_s3.xml \ + --doctest-modules templateflow/api.py - - run: - name: Submit api test coverage - command: | - source /tmp/venv/bin/activate - codecov --file /tmp/cov_api_s3.xml --root /tmp/src/templateflow \ - --flags api,s3 -e CIRCLE_JOB + - codecov/upload: + file: ~/coverage/cov_api_s3.xml + flags: api,s3 + upload_name: S3 tests - run: name: Run tests (w/ DataLad, bypassed via S3) command: | - source /tmp/venv/bin/activate export TEMPLATEFLOW_USE_DATALAD=off export TEMPLATEFLOW_HOME=$HOME/templateflow-clean datalad install -r -s https://github.com/templateflow/templateflow $TEMPLATEFLOW_HOME - pytest --junit-xml=/tmp/tests/dl+s3.xml --cov templateflow --cov-report xml:/tmp/cov_api_dl_s3.xml \ - --doctest-modules /tmp/src/templateflow/templateflow/api.py + python -m pytest \ + --junit-xml=~/tests/dl+s3.xml --cov templateflow --cov-report xml:~/coverage/cov_api_dl_s3.xml \ + --doctest-modules templateflow/api.py - - run: - name: Submit api test coverage - command: | - source /tmp/venv/bin/activate - codecov --file /tmp/cov_api_dl_s3.xml --root /tmp/src/templateflow \ - --flags api,dls3 -e CIRCLE_JOB - - - run: - name: Run tests (config, parameterized TEMPLATEFLOW_USE_DATALAD) - command: | - source /tmp/venv/bin/activate - pytest --junit-xml=/tmp/tests/conftests.xml --cov templateflow --cov-report xml:/tmp/cov_config.xml \ - /tmp/src/templateflow/templateflow/conf/tests/test_conf.py - - - run: - name: Submit config test coverage - command: | - source /tmp/venv/bin/activate - codecov --file /tmp/cov_config.xml --root /tmp/src/templateflow \ - --flags config -e CIRCLE_JOB + - codecov/upload: + file: ~/coverage/cov_api_dl_s3.xml + flags: api,dls3 + upload_name: Datalad-S3-bypass - store_test_results: - path: /tmp/tests + path: ~/tests build_docs: machine: @@ -179,54 +217,38 @@ jobs: - store_artifacts: path: ~/html - deploy_pypi: - docker: - - image: cimg/python:3.9 - working_directory: /tmp/src/templateflow - steps: - - attach_workspace: - at: /tmp - - - checkout: - path: /tmp/src/templateflow - - - run: - name: Generate requirements.txt - command: | - python /tmp/src/templateflow/.maint/update_requirements.py - - - restore_cache: - keys: - - deps-v11-{{ checksum "/tmp/src/templateflow/dev-requirements.txt"}}-{{ epoch }} - - deps-v11-{{ checksum "/tmp/src/templateflow/dev-requirements.txt"}}- - - deps-v11- - - - run: - name: Deploy to PyPi - command: | - source /tmp/venv/bin/activate - python -m pip install build -U twine pkginfo - python -m build - python -m twine check dist/* - python -m twine upload dist/* - workflows: version: 2 build_test_deploy: jobs: + - build: + context: + - nipreps-common + filters: + branches: + ignore: + - /docs?\/.*/ + tags: + only: /.*/ + - tests: context: - nipreps-common + requires: + - build filters: branches: ignore: - /docs?\/.*/ tags: only: /.*/ + - deploy_pypi: context: - nipreps-common requires: + - build + - tests - build_docs filters: branches: diff --git a/.maint/update_requirements.py b/.maint/update_requirements.py index bbce29a9..f3d2e110 100755 --- a/.maint/update_requirements.py +++ b/.maint/update_requirements.py @@ -16,13 +16,13 @@ reqs_dev = repo_root / 'dev-requirements.txt' requirements = [ - Requirement(req) - for req in loads(pyproject.read_text())['project']['dependencies'] + Requirement(req) for req in loads(pyproject.read_text())['project']['dependencies'] ] requirements_dev = [ Requirement(req) - for req in loads(pyproject.read_text())['project']['optional-dependencies']['test'] + for group in ('test', 'datalad') + for req in loads(pyproject.read_text())['project']['optional-dependencies'][group] ] script_name = Path(__file__).relative_to(repo_root) diff --git a/pyproject.toml b/pyproject.toml index 507d5146..51e999f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,13 +37,15 @@ Documentation = "https://www.templateflow.org/python-client/" [project.optional-dependencies] test = [ + "coverage ~= 5.0.0", "pytest", + "pytest-cov", + "pytest-env", "pytest-xdist", - "pytest-cov == 2.5.1", - "coverage", + "toml", ] datalad = [ - "datalad ~= 0.19.0" + "datalad ~= 1.0.0" ] doc = [ "nbsphinx", @@ -113,7 +115,7 @@ per-file-ignores = [ [tool.pytest.ini_options] norecursedirs = [".git"] -addopts = "-svx --doctest-modules" +addopts = "-svx" doctest_optionflags = "ALLOW_UNICODE NORMALIZE_WHITESPACE ELLIPSIS" env = "PYTHONHASHSEED=0" filterwarnings = ["ignore::DeprecationWarning"] @@ -121,7 +123,7 @@ junit_family = "xunit2" [tool.coverage.run] branch = true -concurrency = 'multiprocessing' +concurrency = ['multiprocessing'] omit = [ '*/tests/*', '*/conftest.py', diff --git a/templateflow/__init__.py b/templateflow/__init__.py index 2d2a5ce5..ee150506 100644 --- a/templateflow/__init__.py +++ b/templateflow/__init__.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """TemplateFlow is the Zone of Templates.""" + from datetime import datetime as _dt from datetime import timezone as _tz @@ -30,6 +31,7 @@ from ._version import __version__ except ModuleNotFoundError: from importlib.metadata import PackageNotFoundError, version + try: __version__ = version(__packagename__) except PackageNotFoundError: diff --git a/templateflow/_loader.py b/templateflow/_loader.py index 123a17cd..37d33f9e 100644 --- a/templateflow/_loader.py +++ b/templateflow/_loader.py @@ -24,6 +24,7 @@ .. autoclass:: Loader """ + from __future__ import annotations import atexit diff --git a/templateflow/api.py b/templateflow/api.py index 19f42441..6d1d51f0 100644 --- a/templateflow/api.py +++ b/templateflow/api.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """TemplateFlow's Python Client.""" + import sys from json import loads from pathlib import Path @@ -35,9 +36,7 @@ requires_layout, ) -_layout_dir = tuple( - item for item in dir(TF_LAYOUT) if item.startswith('get_') -) +_layout_dir = tuple(item for item in dir(TF_LAYOUT) if item.startswith('get_')) @requires_layout @@ -92,10 +91,9 @@ def ls(template, **kwargs): kwargs['extension'] = _normalize_ext(kwargs['extension']) return [ - Path(p) for p in TF_LAYOUT.get( - template=Query.ANY if template is None else template, - return_type='file', - **kwargs + Path(p) + for p in TF_LAYOUT.get( + template=Query.ANY if template is None else template, return_type='file', **kwargs ) ] @@ -176,25 +174,23 @@ def get(template, raise_empty=False, **kwargs): not_fetched = [str(p) for p in out_file if not p.is_file() or p.stat().st_size == 0] if not_fetched: - msg = 'Could not fetch template files: %s.' % ', '.join(not_fetched) + msg = 'Could not fetch template files: {}.'.format(', '.join(not_fetched)) if dl_missing and not TF_USE_DATALAD: msg += ( - """\ -The $TEMPLATEFLOW_HOME folder %s seems to contain an initiated DataLad \ + f"""\ +The $TEMPLATEFLOW_HOME folder {TF_LAYOUT.root} seems to contain an initiated DataLad \ dataset, but the environment variable $TEMPLATEFLOW_USE_DATALAD is not \ set or set to one of (false, off, 0). Please set $TEMPLATEFLOW_USE_DATALAD \ on (possible values: true, on, 1).""" - % TF_LAYOUT.root ) if s3_missing and TF_USE_DATALAD: msg += ( - """\ -The $TEMPLATEFLOW_HOME folder %s seems to contain an plain \ + f"""\ +The $TEMPLATEFLOW_HOME folder {TF_LAYOUT.root} seems to contain an plain \ dataset, but the environment variable $TEMPLATEFLOW_USE_DATALAD is \ set to one of (true, on, 1). Please set $TEMPLATEFLOW_USE_DATALAD \ off (possible values: false, off, 0).""" - % TF_LAYOUT.root ) raise RuntimeError(msg) @@ -251,7 +247,7 @@ def get_metadata(template): """ tf_home = Path(TF_LAYOUT.root) - filepath = tf_home / ('tpl-%s' % template) / 'template_description.json' + filepath = tf_home / (f'tpl-{template}') / 'template_description.json' # Ensure that template is installed and file is available if not filepath.is_file(): @@ -324,7 +320,7 @@ def _s3_get(filepath): path = filepath.relative_to(TF_LAYOUT.root).as_posix() url = f'{TF_S3_ROOT}/{path}' - print('Downloading %s' % url, file=stderr) + print(f'Downloading {url}', file=stderr) # Streaming, so we can iterate over the response. r = requests.get(url, stream=True, timeout=TF_GET_TIMEOUT) diff --git a/templateflow/cli.py b/templateflow/cli.py index 22b06f14..f536eed1 100644 --- a/templateflow/cli.py +++ b/templateflow/cli.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """The TemplateFlow Python Client command-line interface (CLI).""" + from __future__ import annotations import json @@ -31,20 +32,20 @@ from templateflow import __package__, api from templateflow._loader import Loader as _Loader -from templateflow.conf import TF_HOME, TF_USE_DATALAD, TF_AUTOUPDATE +from templateflow.conf import TF_AUTOUPDATE, TF_HOME, TF_USE_DATALAD load_data = _Loader(__package__) ENTITY_SHORTHANDS = { # 'template': ('--tpl', '-t'), - 'resolution': ('--res', ), - 'density': ('--den', ), - 'atlas': ('-a', ), - 'suffix': ('-s', ), + 'resolution': ('--res',), + 'density': ('--den',), + 'atlas': ('-a',), + 'suffix': ('-s',), 'desc': ('-d', '--description'), 'extension': ('--ext', '-x'), - 'label': ('-l', ), - 'segmentation': ('--seg', ), + 'label': ('-l',), + 'segmentation': ('--seg',), } ENTITY_EXCLUDE = {'template', 'description'} TEMPLATE_LIST = api.get_templates() @@ -57,16 +58,12 @@ def _nulls(s): def entity_opts(): """Attaches all entities as options to the command.""" - entities = json.loads( - Path(load_data('conf/config.json')).read_text() - )['entities'] + entities = json.loads(Path(load_data('conf/config.json')).read_text())['entities'] args = [ - ( - f"--{e['name']}", - *ENTITY_SHORTHANDS.get(e['name'], ()) - ) - for e in entities if e['name'] not in ENTITY_EXCLUDE + (f"--{e['name']}", *ENTITY_SHORTHANDS.get(e['name'], ())) + for e in entities + if e['name'] not in ENTITY_EXCLUDE ] def decorator(f: FC) -> FC: @@ -135,9 +132,7 @@ def update(local, overwrite): def ls(template, **kwargs): """List the assets corresponding to template and optional filters.""" entities = {k: _nulls(v) for k, v in kwargs.items() if v != ''} - click.echo( - '\n'.join(f'{match}' for match in api.ls(template, **entities)) - ) + click.echo('\n'.join(f'{match}' for match in api.ls(template, **entities))) @main.command() @@ -146,9 +141,7 @@ def ls(template, **kwargs): def get(template, **kwargs): """Fetch the assets corresponding to template and optional filters.""" entities = {k: _nulls(v) for k, v in kwargs.items() if v != ''} - click.echo( - '\n'.join(f'{match}' for match in api.get(template, **entities)) - ) + click.echo('\n'.join(f'{match}' for match in api.get(template, **entities))) if __name__ == '__main__': diff --git a/templateflow/conf/__init__.py b/templateflow/conf/__init__.py index a75bfd85..85940474 100644 --- a/templateflow/conf/__init__.py +++ b/templateflow/conf/__init__.py @@ -1,4 +1,5 @@ """Configuration and settings.""" + import re from contextlib import suppress from functools import wraps @@ -35,7 +36,7 @@ def _env_to_bool(envvar: str, default: bool) -> bool: TF_DEFAULT_HOME = Path.home() / '.cache' / 'templateflow' -TF_HOME = Path(getenv('TEMPLATEFLOW_HOME', str(TF_DEFAULT_HOME))) +TF_HOME = Path(getenv('TEMPLATEFLOW_HOME', str(TF_DEFAULT_HOME))).absolute() TF_GITHUB_SOURCE = 'https://github.com/templateflow/templateflow.git' TF_S3_ROOT = 'https://templateflow.s3.amazonaws.com' TF_USE_DATALAD = _env_to_bool('TEMPLATEFLOW_USE_DATALAD', False) @@ -43,9 +44,19 @@ def _env_to_bool(envvar: str, default: bool) -> bool: TF_CACHED = True TF_GET_TIMEOUT = 10 +if TF_USE_DATALAD: + try: + from datalad.api import install + except ImportError: + warn('DataLad is not installed ➔ disabled.', stacklevel=2) + TF_USE_DATALAD = False + +if not TF_USE_DATALAD: + from templateflow.conf._s3 import update as _update_s3 + def _init_cache(): - global TF_HOME, TF_CACHED, TF_USE_DATALAD + global TF_CACHED if not TF_HOME.exists() or not list(TF_HOME.iterdir()): TF_CACHED = False @@ -58,17 +69,9 @@ def _init_cache(): stacklevel=2, ) if TF_USE_DATALAD: - try: - from datalad.api import install - except ImportError: - TF_USE_DATALAD = False - else: - TF_HOME.parent.mkdir(exist_ok=True, parents=True) - install(path=str(TF_HOME), source=TF_GITHUB_SOURCE, recursive=True) - - if not TF_USE_DATALAD: - from ._s3 import update as _update_s3 - + TF_HOME.parent.mkdir(exist_ok=True, parents=True) + install(path=str(TF_HOME), source=TF_GITHUB_SOURCE, recursive=True) + else: _update_s3(TF_HOME, local=True, overwrite=TF_AUTOUPDATE, silent=True) @@ -85,9 +88,7 @@ def wrapper(*args, **kwargs): if TF_LAYOUT is None: from bids import __version__ - raise RuntimeError( - f'A layout with PyBIDS <{__version__}> could not be initiated' - ) + raise RuntimeError(f'A layout with PyBIDS <{__version__}> could not be initiated') return func(*args, **kwargs) return wrapper @@ -95,8 +96,8 @@ def wrapper(*args, **kwargs): def update(local=False, overwrite=True, silent=False): """Update an existing DataLad or S3 home.""" - if TF_USE_DATALAD and _update_datalad(): - success = True + if TF_USE_DATALAD: + success = _update_datalad() else: from ._s3 import update as _update_s3 @@ -116,7 +117,6 @@ def update(local=False, overwrite=True, silent=False): def wipe(): """Clear the cache if functioning in S3 mode.""" - global TF_USE_DATALAD, TF_HOME if TF_USE_DATALAD: print('TemplateFlow is configured in DataLad mode, wipe() has no effect') @@ -131,9 +131,7 @@ def _onerror(func, path, excinfo): from pathlib import Path if Path(path).exists(): - print( - f'Warning: could not delete <{path}>, please clear the cache manually.' - ) + print(f'Warning: could not delete <{path}>, please clear the cache manually.') rmtree(TF_HOME, onerror=_onerror) _init_cache() diff --git a/templateflow/conf/_s3.py b/templateflow/conf/_s3.py index d70c3cb3..eda04909 100644 --- a/templateflow/conf/_s3.py +++ b/templateflow/conf/_s3.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """Tooling to handle S3 downloads.""" + from pathlib import Path from tempfile import mkstemp @@ -86,18 +87,18 @@ def _update_skeleton(skel_file, dest, overwrite=True, silent=False): newfiles = allfiles else: current_files = [s.relative_to(dest) for s in dest.glob('**/*')] - existing = sorted({'%s/' % s.parent for s in current_files}) + [ + existing = sorted({f'{s.parent}/' for s in current_files}) + [ str(s) for s in current_files ] newfiles = sorted(set(allfiles) - set(existing)) if newfiles: if not silent: - print( - 'Updating TEMPLATEFLOW_HOME using S3. Adding:\n%s' - % '\n'.join(newfiles) - ) + print('Updating TEMPLATEFLOW_HOME using S3. Adding:') + for fl in newfiles: + if not silent: + print(fl) localpath = dest / fl if localpath.exists(): continue diff --git a/templateflow/conf/bids.py b/templateflow/conf/bids.py index 3430e711..a143e4de 100644 --- a/templateflow/conf/bids.py +++ b/templateflow/conf/bids.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """Extending pyBIDS for querying TemplateFlow.""" + from bids.layout import BIDSLayout, add_config_paths from templateflow.conf import load_data @@ -34,7 +35,5 @@ def __repr__(self): s = """\ TemplateFlow Layout - Home: {} - - Templates: {}.""".format( - self.root, ', '.join(sorted(self.get_templates())) - ) + - Templates: {}.""".format(self.root, ', '.join(sorted(self.get_templates()))) return s diff --git a/templateflow/conf/tests/__init__.py b/templateflow/conf/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/templateflow/conf/tests/test_conf.py b/templateflow/conf/tests/test_conf.py deleted file mode 100644 index 355bbd12..00000000 --- a/templateflow/conf/tests/test_conf.py +++ /dev/null @@ -1,151 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -# -# Copyright 2024 The NiPreps Developers -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# We support and encourage derived works from this project, please read -# about our expectations at -# -# https://www.nipreps.org/community/licensing/ -# -from importlib import reload -from shutil import rmtree - -import pytest - -from templateflow import conf as tfc - - -def _find_message(lines, msg, reverse=True): - if isinstance(lines, str): - lines = lines.splitlines() - - for line in reversed(lines): - if line.strip().startswith(msg): - return True - return False - - -@pytest.mark.parametrize('use_datalad', ['off', 'on']) -def test_conf_init(monkeypatch, tmp_path, capsys, use_datalad): - """Check the correct functioning of config set-up.""" - home = (tmp_path / '-'.join(('tf', 'dl', use_datalad))).resolve() - monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', use_datalad) - monkeypatch.setenv('TEMPLATEFLOW_HOME', str(home)) - - # First execution, the S3 stub is created (or datalad install) - reload(tfc) - assert tfc.TF_CACHED is False - assert str(tfc.TF_HOME) == str(home) - - reload(tfc) - assert tfc.TF_CACHED is True - assert str(tfc.TF_HOME) == str(home) - - -@pytest.mark.parametrize('use_datalad', ['off', 'on']) -def test_setup_home(monkeypatch, tmp_path, capsys, use_datalad): - """Check the correct functioning of the installation hook.""" - home = (tmp_path / '-'.join(('tf', 'dl', use_datalad))).resolve() - monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', use_datalad) - monkeypatch.setenv('TEMPLATEFLOW_HOME', str(home)) - - reload(tfc) - # First execution, the S3 stub is created (or datalad install) - assert tfc.TF_CACHED is False - assert tfc.setup_home() is False - - out = capsys.readouterr().out - assert _find_message(out, 'TemplateFlow was not cached') - assert ('TEMPLATEFLOW_HOME=%s' % home) in out - assert home.exists() - assert len(list(home.iterdir())) > 0 - - updated = tfc.setup_home(force=True) # Templateflow is now cached - out = capsys.readouterr()[0] - assert _find_message(out, 'TemplateFlow was not cached') is False - - if use_datalad == 'on': - assert _find_message(out, 'Updating TEMPLATEFLOW_HOME using DataLad') - assert updated is True - - elif use_datalad == 'off': - # At this point, S3 should be up-to-date - assert updated is False - assert _find_message(out, 'TEMPLATEFLOW_HOME directory (S3 type) was up-to-date.') - - # Let's force an update - rmtree(str(home / 'tpl-MNI152NLin2009cAsym')) - updated = tfc.setup_home(force=True) - out = capsys.readouterr()[0] - assert updated is True - assert _find_message(out, 'Updating TEMPLATEFLOW_HOME using S3.') - - reload(tfc) - assert tfc.TF_CACHED is True - updated = tfc.setup_home() # Templateflow is now cached - out = capsys.readouterr()[0] - assert not _find_message(out, 'TemplateFlow was not cached') - - if use_datalad == 'on': - assert _find_message(out, 'Updating TEMPLATEFLOW_HOME using DataLad') - assert updated is True - - elif use_datalad == 'off': - # At this point, S3 should be up-to-date - assert updated is False - assert _find_message(out, 'TEMPLATEFLOW_HOME directory (S3 type) was up-to-date.') - - # Let's force an update - rmtree(str(home / 'tpl-MNI152NLin2009cAsym')) - updated = tfc.setup_home() - out = capsys.readouterr()[0] - assert updated is True - assert _find_message(out, 'Updating TEMPLATEFLOW_HOME using S3.') - - -def test_layout(monkeypatch, tmp_path): - monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', 'off') - - lines = ('%s' % tfc.TF_LAYOUT).splitlines() - assert lines[0] == 'TemplateFlow Layout' - assert lines[1] == ' - Home: %s' % tfc.TF_HOME - assert lines[2].startswith(' - Templates:') - - -def test_layout_errors(monkeypatch): - """Check regression of #71.""" - import builtins - import sys - from importlib import __import__ as oldimport - - @tfc.requires_layout - def myfunc(): - return 'okay' - - def mock_import(name, globals=None, locals=None, fromlist=(), level=0): # noqa: A002 - if name == 'bids': - raise ModuleNotFoundError - return oldimport(name, globals=globals, locals=locals, fromlist=fromlist, level=level) - - with monkeypatch.context() as m: - m.setattr(tfc, 'TF_LAYOUT', None) - with pytest.raises(RuntimeError): - myfunc() - - m.delitem(sys.modules, 'bids') - m.setattr(builtins, '__import__', mock_import) - with pytest.raises(ImportError): - myfunc() diff --git a/templateflow/conf/tests/test_s3.py b/templateflow/conf/tests/test_s3.py deleted file mode 100644 index 0971f662..00000000 --- a/templateflow/conf/tests/test_s3.py +++ /dev/null @@ -1,74 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -# -# Copyright 2024 The NiPreps Developers -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# We support and encourage derived works from this project, please read -# about our expectations at -# -# https://www.nipreps.org/community/licensing/ -# -"""Check S3-type repo tooling.""" -# import pytest -from pathlib import Path - -import requests - -from .. import _s3 as s3 - - -def test_get_skel_file(monkeypatch): - """Exercise the skeleton file generation.""" - local_md5 = s3.TF_SKEL_MD5 - monkeypatch.setattr(s3, 'TF_SKEL_MD5', 'invent') - new_skel = s3._get_skeleton_file() - assert new_skel is not None - assert Path(new_skel).exists() - assert Path(new_skel).stat().st_size > 0 - - latest_md5 = ( - requests.get(s3.TF_SKEL_URL(release='master', ext='md5', allow_redirects=True), timeout=10) - .content.decode() - .split()[0] - ) - monkeypatch.setattr(s3, 'TF_SKEL_MD5', latest_md5) - assert s3._get_skeleton_file() is None - - monkeypatch.setattr(s3, 'TF_SKEL_MD5', local_md5) - monkeypatch.setattr(s3, 'TF_SKEL_URL', 'http://weird/{release}/{ext}'.format) - assert s3._get_skeleton_file() is None - - monkeypatch.setattr( - s3, 'TF_SKEL_URL', s3.TF_SKEL_URL(release='{release}', ext='{ext}z').format - ) - assert s3._get_skeleton_file() is None - - -def test_update_s3(tmp_path, monkeypatch): - """Exercise updating the S3 skeleton.""" - newhome = tmp_path / 'templateflow' - assert s3.update(newhome) - assert not s3.update(newhome, overwrite=False) - for p in (newhome / 'tpl-MNI152NLin6Sym').glob('*.nii.gz'): - p.unlink() - assert s3.update(newhome, overwrite=False) - - # This should cover the remote zip file fetching - monkeypatch.setattr(s3, 'TF_SKEL_MD5', 'invent') - assert s3.update(newhome, local=False) - assert not s3.update(newhome, local=False, overwrite=False) - for p in (newhome / 'tpl-MNI152NLin6Sym').glob('*.nii.gz'): - p.unlink() - assert s3.update(newhome, local=False, overwrite=False) diff --git a/templateflow/tests/test_api.py b/templateflow/tests/test_api.py index fc773319..fe2aea3b 100644 --- a/templateflow/tests/test_api.py +++ b/templateflow/tests/test_api.py @@ -24,7 +24,7 @@ import pytest -from .. import api +from templateflow import api class Bibtex: @@ -154,9 +154,7 @@ def assert_same(self, other): journal={Cerebral Cortex} }""" -fslr_lbib = ( - 'https://github.com/Washington-University/HCPpipelines/tree/master/global/templates' -) +fslr_lbib = 'https://github.com/Washington-University/HCPpipelines/tree/master/global/templates' fsaverage_fbib = """\ @article{Fischl_1999, @@ -181,9 +179,7 @@ def assert_same(self, other): ('fsLR', fslr_urls, fslr_fbib, fslr_lbib), ( 'fsaverage', - [ - 'https://doi.org/10.1002/(sici)1097-0193(1999)8:4%3C272::aid-hbm10%3E3.0.co;2-4' - ], + ['https://doi.org/10.1002/(sici)1097-0193(1999)8:4%3C272::aid-hbm10%3E3.0.co;2-4'], fsaverage_fbib, None, ), diff --git a/templateflow/tests/test_conf.py b/templateflow/tests/test_conf.py index 461d5bd2..c30ea877 100644 --- a/templateflow/tests/test_conf.py +++ b/templateflow/tests/test_conf.py @@ -21,32 +21,146 @@ # https://www.nipreps.org/community/licensing/ # """Tests the config module.""" -from pathlib import Path + +from importlib import reload +from shutil import rmtree import pytest -from .. import api, conf - - -@pytest.mark.skipif(conf.TF_USE_DATALAD, reason='S3 only') -def test_update_s3(tmp_path): - conf.TF_HOME = tmp_path / 'templateflow' - conf.TF_HOME.mkdir(exist_ok=True) - - # replace TF_SKEL_URL with the path of a legacy skeleton - _skel_url = conf._s3.TF_SKEL_URL - conf._s3.TF_SKEL_URL = ( - 'https://github.com/templateflow/python-client/raw/0.5.0/' - 'templateflow/conf/templateflow-skel.{ext}'.format - ) - # initialize templateflow home, making sure to pull the legacy skeleton - conf.update(local=False) - # ensure we can grab a file - assert Path(api.get('MNI152NLin2009cAsym', resolution=2, desc='brain', suffix='mask')).exists() - # and ensure we can't fetch one that doesn't yet exist - assert not api.get('Fischer344', hemi='L', desc='brain', suffix='mask') - - # refresh the skeleton using the most recent skeleton - conf._s3.TF_SKEL_URL = _skel_url - conf.update(local=True, overwrite=True) - assert Path(api.get('Fischer344', hemi='L', desc='brain', suffix='mask')).exists() +from templateflow import conf as tfc + + +def _find_message(lines, msg, reverse=True): + if isinstance(lines, str): + lines = lines.splitlines() + + for line in reversed(lines): + if line.strip().startswith(msg): + return True + return False + + +@pytest.mark.parametrize('use_datalad', ['off', 'on']) +def test_conf_init(monkeypatch, tmp_path, use_datalad): + """Check the correct functioning of config set-up.""" + home = (tmp_path / f'conf-init-{use_datalad}').resolve() + monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', use_datalad) + monkeypatch.setenv('TEMPLATEFLOW_HOME', str(home)) + + # First execution, the S3 stub is created (or datalad install) + reload(tfc) + assert tfc.TF_CACHED is False + assert str(tfc.TF_HOME) == str(home) + + reload(tfc) + assert tfc.TF_CACHED is True + assert str(tfc.TF_HOME) == str(home) + + +@pytest.mark.parametrize('use_datalad', ['on', 'off']) +def test_setup_home(monkeypatch, tmp_path, capsys, use_datalad): + """Check the correct functioning of the installation hook.""" + + if use_datalad == 'on': + # ImportError if not installed + pass + + home = (tmp_path / f'setup-home-{use_datalad}').absolute() + monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', use_datalad) + monkeypatch.setenv('TEMPLATEFLOW_HOME', str(home)) + + use_post = tfc._env_to_bool('TEMPLATEFLOW_USE_DATALAD', False) + assert use_post is (use_datalad == 'on') + + with capsys.disabled(): + reload(tfc) + + # Ensure mocks are up-to-date + assert tfc.TF_USE_DATALAD is (use_datalad == 'on') + assert str(tfc.TF_HOME) == str(home) + # First execution, the S3 stub is created (or datalad install) + assert tfc.TF_CACHED is False + assert tfc.setup_home() is False + + out = capsys.readouterr().out + assert _find_message(out, 'TemplateFlow was not cached') + assert (f'TEMPLATEFLOW_HOME={home}') in out + assert home.exists() + assert len(list(home.iterdir())) > 0 + + updated = tfc.setup_home(force=True) # Templateflow is now cached + out = capsys.readouterr()[0] + assert _find_message(out, 'TemplateFlow was not cached') is False + + if use_datalad == 'on': + assert _find_message(out, 'Updating TEMPLATEFLOW_HOME using DataLad') + assert updated is True + + elif use_datalad == 'off': + # At this point, S3 should be up-to-date + assert updated is False + assert _find_message(out, 'TEMPLATEFLOW_HOME directory (S3 type) was up-to-date.') + + # Let's force an update + rmtree(str(home / 'tpl-MNI152NLin2009cAsym')) + updated = tfc.setup_home(force=True) + out = capsys.readouterr()[0] + assert updated is True + assert _find_message(out, 'Updating TEMPLATEFLOW_HOME using S3.') + + reload(tfc) + assert tfc.TF_CACHED is True + updated = tfc.setup_home() # Templateflow is now cached + out = capsys.readouterr()[0] + assert not _find_message(out, 'TemplateFlow was not cached') + + if use_datalad == 'on': + assert _find_message(out, 'Updating TEMPLATEFLOW_HOME using DataLad') + assert updated is True + + elif use_datalad == 'off': + # At this point, S3 should be up-to-date + assert updated is False + assert _find_message(out, 'TEMPLATEFLOW_HOME directory (S3 type) was up-to-date.') + + # Let's force an update + rmtree(str(home / 'tpl-MNI152NLin2009cAsym')) + updated = tfc.setup_home() + out = capsys.readouterr()[0] + assert updated is True + assert _find_message(out, 'Updating TEMPLATEFLOW_HOME using S3.') + + +def test_layout(monkeypatch, tmp_path): + monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', 'off') + + lines = (f'{tfc.TF_LAYOUT}').splitlines() + assert lines[0] == 'TemplateFlow Layout' + assert lines[1] == f' - Home: {tfc.TF_HOME}' + assert lines[2].startswith(' - Templates:') + + +def test_layout_errors(monkeypatch): + """Check regression of #71.""" + import builtins + import sys + from importlib import __import__ as oldimport + + @tfc.requires_layout + def myfunc(): + return 'okay' + + def mock_import(name, globals=None, locals=None, fromlist=(), level=0): # noqa: A002 + if name == 'bids': + raise ModuleNotFoundError + return oldimport(name, globals=globals, locals=locals, fromlist=fromlist, level=level) + + with monkeypatch.context() as m: + m.setattr(tfc, 'TF_LAYOUT', None) + with pytest.raises(RuntimeError): + myfunc() + + m.delitem(sys.modules, 'bids') + m.setattr(builtins, '__import__', mock_import) + with pytest.raises(ImportError): + myfunc() diff --git a/templateflow/tests/test_s3.py b/templateflow/tests/test_s3.py new file mode 100644 index 00000000..e8984438 --- /dev/null +++ b/templateflow/tests/test_s3.py @@ -0,0 +1,89 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +# +# Copyright 2024 The NiPreps Developers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# We support and encourage derived works from this project, please read +# about our expectations at +# +# https://www.nipreps.org/community/licensing/ +# +"""Check S3-type repo tooling.""" + +from importlib import reload +from pathlib import Path + +import requests + +from templateflow import conf as tfc + + +def test_get_skel_file(tmp_path, monkeypatch): + """Exercise the skeleton file generation.""" + + home = (tmp_path / 's3-skel-file').resolve() + monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', 'off') + monkeypatch.setenv('TEMPLATEFLOW_HOME', str(home)) + + # First execution, the S3 stub is created (or datalad install) + reload(tfc) + + local_md5 = tfc._s3.TF_SKEL_MD5 + monkeypatch.setattr(tfc._s3, 'TF_SKEL_MD5', 'invent') + new_skel = tfc._s3._get_skeleton_file() + assert new_skel is not None + assert Path(new_skel).exists() + assert Path(new_skel).stat().st_size > 0 + + latest_md5 = ( + requests.get( + tfc._s3.TF_SKEL_URL(release='master', ext='md5', allow_redirects=True), timeout=10 + ) + .content.decode() + .split()[0] + ) + monkeypatch.setattr(tfc._s3, 'TF_SKEL_MD5', latest_md5) + assert tfc._s3._get_skeleton_file() is None + + monkeypatch.setattr(tfc._s3, 'TF_SKEL_MD5', local_md5) + monkeypatch.setattr(tfc._s3, 'TF_SKEL_URL', 'http://weird/{release}/{ext}'.format) + assert tfc._s3._get_skeleton_file() is None + + monkeypatch.setattr( + tfc._s3, 'TF_SKEL_URL', tfc._s3.TF_SKEL_URL(release='{release}', ext='{ext}z').format + ) + assert tfc._s3._get_skeleton_file() is None + + +def test_update_s3(tmp_path, monkeypatch): + """Exercise updating the S3 skeleton.""" + + newhome = (tmp_path / 's3-update').resolve() + monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', 'off') + monkeypatch.setenv('TEMPLATEFLOW_HOME', str(newhome)) + + assert tfc._s3.update(newhome) + assert not tfc._s3.update(newhome, overwrite=False) + for p in (newhome / 'tpl-MNI152NLin6Sym').glob('*.nii.gz'): + p.unlink() + assert tfc._s3.update(newhome, overwrite=False) + + # This should cover the remote zip file fetching + monkeypatch.setattr(tfc._s3, 'TF_SKEL_MD5', 'invent') + assert tfc._s3.update(newhome, local=False) + assert not tfc._s3.update(newhome, local=False, overwrite=False) + for p in (newhome / 'tpl-MNI152NLin6Sym').glob('*.nii.gz'): + p.unlink() + assert tfc._s3.update(newhome, local=False, overwrite=False) diff --git a/templateflow/tests/test_version.py b/templateflow/tests/test_version.py index e7178f40..d0a8df6c 100644 --- a/templateflow/tests/test_version.py +++ b/templateflow/tests/test_version.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """Test version retrieval.""" + import sys from importlib import reload from importlib.metadata import PackageNotFoundError