From 2695eef270ce6e14e99e3e368aabbebeb7ed5592 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Fri, 23 Jun 2023 14:12:32 +0100 Subject: [PATCH 01/20] Use extreqs for optional extras --- pyproject.toml | 3 +++ requirements.txt | 5 +++++ setup.py | 12 ++++++------ 3 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a6433d5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "extreqs"] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 7e1232d..ae86d4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,11 @@ scipy>=1.3.0 six>=1.11.0 tqdm>=4.50.0 psutil>=5.4.3 + +#extra: extras +fuzzywuzzy[speedup]~=0.17.0 +ujson~=1.35 + # diskcache>=4.0.0 # Below are optional dependencies diff --git a/setup.py b/setup.py index dabc62a..e1f4ef8 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,8 @@ from setuptools import setup, find_packages import re +from pathlib import Path + +from extreqs import parse_requirement_files VERSIONFILE = "pymaid/__init__.py" @@ -11,9 +14,7 @@ else: raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,)) -with open('requirements.txt') as f: - requirements = f.read().splitlines() - requirements = [l for l in requirements if not l.startswith('#')] +install_requires, extras_require = parse_requirement_files(Path("requirements.txt")) setup( name='python-catmaid', @@ -45,9 +46,8 @@ 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', ], - install_requires=requirements, - extras_require={'extras': ['fuzzywuzzy[speedup]~=0.17.0', - 'ujson~=1.35']}, + install_requires=install_requires, + extras_require=extras_require, python_requires='>=3.9', zip_safe=False ) From b50bf6fba04f10f528941d8d90a4caabbb4b26ba Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Fri, 23 Jun 2023 14:13:14 +0100 Subject: [PATCH 02/20] Add zarr extra requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index ae86d4c..43a81de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ psutil>=5.4.3 #extra: extras fuzzywuzzy[speedup]~=0.17.0 ujson~=1.35 +zarr # diskcache>=4.0.0 From 0850d89507b1bb3598dea63e8f0557442e63a42e Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Fri, 23 Jun 2023 17:16:34 +0100 Subject: [PATCH 03/20] WIP zarr stores from CATMAID stacks --- pymaid/stack.py | 312 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 + 2 files changed, 314 insertions(+) create mode 100644 pymaid/stack.py diff --git a/pymaid/stack.py b/pymaid/stack.py new file mode 100644 index 0000000..9af9c1d --- /dev/null +++ b/pymaid/stack.py @@ -0,0 +1,312 @@ +from __future__ import annotations +from io import BytesIO +from typing import Literal, Optional, Sequence, Type, TypeVar, Generic, TypedDict, Union +import numpy as np +from abc import ABC +from numpy.typing import DTypeLike, ArrayLike +import zarr +from pydantic import BaseModel +from pydantic.tools import parse_obj_as +from dask import array as da +from . import utils +from zarr.storage import BaseStore +import json +import sys +import requests +import imageio.v3 as iio + +Dimension = Literal["x", "y", "z"] +Orientation = Literal["xy", "xz", "zy"] +HALF_PX = 0.5 +ENDIAN = "<" if sys.byteorder == "little" else ">" + + +class MirrorInfo(BaseModel): + id: int + title: str + image_base: str + tile_width: int + tile_height: int + tile_source_type: int + file_extension: str + position: int + + +N = TypeVar("N", int, float) + + +class Coord(TypedDict, Generic[N]): + x: N + y: N + z: N + + +class StackInfo(BaseModel): + sid: int + pid: int + ptitle: str + stitle: str + downsample_factors: list[Coord[float]] + num_zoom_levels: int + translation: Coord[float] + resolution: Coord[float] + dimension: Coord[int] + comment: str + description: str + metadata: str + broken_slices: dict[int, int] + mirrors: list[MirrorInfo] + orientation: Orientation + attribution: str + canary_location: Coord[int] + placeholder_colour: dict[str, float] # actually {r g b a} + + +def to_array( + coord: Union[Coord[N], ArrayLike], + dtype: DTypeLike = np.float64, + order: Sequence[Dimension] = ("z", "y", "x"), +) -> np.ndarray: + if isinstance(coord, dict): + coord = [coord[d] for d in order] + return np.asarray(coord, dtype=dtype) + + +class TileStore(BaseStore, ABC): + """ + Must include instance variable 'fmt', + which is a format string with variables: + image_base, zoom_level, file_extension, row, col, slice_idx + """ + tile_source_type: int + fmt: str + _writeable = False + _erasable = False + _listable = False + + def __init__( + self, + stack_info: StackInfo, + mirror_info: MirrorInfo, + zoom_level: int, + session: Optional[requests.Session] = None, + ) -> None: + if mirror_info.tile_source_type != self.tile_source_type: + raise ValueError("Mismatched tile source type") + self.stack_info = stack_info + self.mirror_info = mirror_info + self.zoom_level = zoom_level + + if session is None: + cm = utils._eval_remote_instance(None) + self.session = cm._session + else: + self.session = session + + order = full_orientation[self.stack_info.orientation] + self.metadata_payload = json.dumps( + { + "zarr_format": 2, + "shape": to_array(stack_info.dimension, order, int).tolist(), + "chunks": [mirror_info.tile_width, mirror_info.tile_height, 1], + "dtype": ENDIAN + "u1", + "compressor": None, + "fill_value": 0, + "order": "C", + "filters": None, + "dimension_separator": ".", + } + ).encode() + + self.empty = np.zeros( + ( + self.mirror_info.tile_width, + self.mirror_info.tile_height, + 1, + ), + "uint8", + ).tobytes() + + def _format_url(self, row: int, col: int, slice_idx: int) -> str: + return self.fmt.format( + zoom_level=self.zoom_level, + slice_idx=slice_idx, + row=row, + col=col, + file_extension=self.mirror_info.file_extension, + ) + + def __getitem__(self, key): + last = key.split("/")[-1] + if last == ".zarray": + return self.metadata_payload + # todo: check order + slice_idx, col, row = (int(i) for i in last.split(".")) + url = self._format_url(row, col, slice_idx) + response = self.session.get(url) + if response.status_code == 404: + return self.empty + response.raise_for_status() + arr = iio.imread( + BytesIO(response.content), + extension=self.mirror_info.file_extension, + mode="L", + ) + return arr.tobytes() + + def to_array(self) -> zarr.Array: + return zarr.open_array(self, "r") + + def to_dask(self) -> da.Array: + return da.from_zarr(self.to_array()) + + +class TileStore1(TileStore): + tile_source_type = 1 + fmt = "{image_base}{slice_idx}/{row}_{col}_{zoom_level}.{file_extension}" + + +class TileStore4(TileStore): + tile_source_type = 4 + fmt = "{image_base}{slice_idx}/{zoom_level}/{row}_{col}.{file_extension}" + + +class TileStore5(TileStore): + tile_source_type = 5 + fmt = "{image_base}{zoom_level}/{slice_idx}/{row}/{col}.{file_extension}" + + +tile_stores: dict[int, Type[TileStore]] = { + t.tile_source_type: t for t in [TileStore1, TileStore4, TileStore5] +} + + +class Stack: + def __init__(self, stack_info: StackInfo, mirror_id: Optional[int] = None): + self.stack_info = stack_info + self.mirror_info: Optional[MirrorInfo] = None + + if mirror_id is not None: + self.set_mirror(mirror_id) + + @classmethod + def from_catmaid( + cls, stack_id: int, mirror_id: Optional[int] = None, remote_instance=None + ): + cm = utils._eval_remote_instance(remote_instance) + info = cm.make_url("stack", stack_id, "info") + sinfo = parse_obj_as(StackInfo, info) + return cls(sinfo, mirror_id) + + def _get_mirror_info(self, mirror_id: Optional[int] = None) -> MirrorInfo: + if mirror_id is None: + if self.mirror_info is None: + raise ValueError("No default mirror ID set") + return self.mirror_info + for mirror in self.stack_info.mirrors: + if mirror.id == mirror_id: + return mirror + raise ValueError( + f"Mirror ID {mirror_id} not found for stack {self.stack_info.sid}" + ) + + def set_mirror(self, mirror_id: int): + self.mirror_id = self._get_mirror_info(mirror_id) + + def _res_for_scale(self, scale_level: int) -> np.ndarray: + return to_array(self.stack_info.resolution) * to_array( + self.stack_info.downsample_factors[scale_level] + ) + + def _from_array(self, arr, scale_level: int) -> ImageVolume: + return ImageVolume( + arr, + self.stack_info.translation, + self._res_for_scale(scale_level), + self.stack_info.orientation, + ) + + def get_scale( + self, scale_level: int, mirror_id: Optional[int] = None + ) -> ImageVolume: + mirror_info = self._get_mirror_info(mirror_id) + if scale_level > self.stack_info.num_zoom_levels: + raise ValueError( + f"Scale level {scale_level} does not exist " + f"for stack {self.stack_info.sid} " + f"with {self.stack_info.num_zoom_levels} stack levels" + ) + + if mirror_info.tile_source_type in tile_stores: + store_class = tile_stores[mirror_info.tile_source_type] + store = store_class(self.stack_info, mirror_info, scale_level, None) + return self._from_array(store.to_dask(), scale_level) + elif mirror_info.tile_source_type == 11: + formatted = mirror_info.image_base.replace( + "%SCALE_DATASET%", f"s{scale_level}" + ) + *components, transpose_str = formatted.split("/") + transpose = [int(t) for t in transpose_str.split("_")] + + store = zarr.N5FSStore("/".join(components)) + arr = zarr.open_array(store, "r") + darr = da.from_zarr(arr).transpose(transpose) + return self._from_array(darr, scale_level) + + raise NotImplementedError( + f"Tile source type {mirror_info.tile_source_type} not implemented" + ) + + +full_orientation: dict[Orientation, Sequence[Dimension]] = { + "xy": "xyz", + "xz": "xzy", + "zy": "zyx", +} + + +class ImageVolume: + def __init__(self, array, offset, resolution, orientation: Orientation): + self.array = array + self.offset = offset + self.resolution = resolution + self.offset = to_array(offset, dtype="float64") + self.resolution = to_array(resolution, dtype="float64") + self.orientation = orientation + + @property + def full_orientation(self): + return full_orientation[self.orientation] + + @property + def offset_oriented(self): + return to_array(self.offset, "float64", self.full_orientation) + + @property + def resolution_oriented(self): + return to_array(self.resolution, "float64", self.full_orientation) + + def __getitem__(self, selection): + return self.array.__getitem__(selection) + + def get_roi( + self, offset: Coord[float], shape: Coord[float] + ) -> tuple[Coord[float], np.ndarray]: + order = self.full_orientation + offset_o = to_array(offset, order=order) + shape_o = to_array(shape, order=order) + mins = (offset_o / self.resolution - self.offset - HALF_PX).astype("uint64") + maxes = np.ceil( + (offset_o + shape_o) / self.resolution - self.offset - HALF_PX + ).astype("uint64") + slicing = tuple(slice(mi, ma) for mi, ma in zip(mins, maxes)) + # todo: finalise orientation + actual_offset = Coord( + **{ + d: m + for d, m in zip( + order, mins * self.resolution_oriented + self.offset_oriented + ) + } + ) + return actual_offset, self[slicing] diff --git a/requirements.txt b/requirements.txt index 43a81de..b02e109 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,8 @@ psutil>=5.4.3 fuzzywuzzy[speedup]~=0.17.0 ujson~=1.35 zarr +pydantic +imageio # diskcache>=4.0.0 From 762fbcf8f954da31e9256bd58a24b6434f9f3772 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Fri, 14 Jul 2023 12:36:53 +0100 Subject: [PATCH 04/20] N5 and JPEG stacks --- pymaid/stack.py | 375 +++++++++++++++++++++++++++++++++-------------- requirements.txt | 6 +- 2 files changed, 271 insertions(+), 110 deletions(-) diff --git a/pymaid/stack.py b/pymaid/stack.py index 9af9c1d..cb4a63e 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -1,13 +1,14 @@ from __future__ import annotations from io import BytesIO -from typing import Literal, Optional, Sequence, Type, TypeVar, Generic, TypedDict, Union +from typing import Literal, Optional, Sequence, Type, TypeVar, Union import numpy as np from abc import ABC +from enum import IntEnum from numpy.typing import DTypeLike, ArrayLike import zarr from pydantic import BaseModel -from pydantic.tools import parse_obj_as from dask import array as da +import xarray as xr from . import utils from zarr.storage import BaseStore import json @@ -16,11 +17,43 @@ import imageio.v3 as iio Dimension = Literal["x", "y", "z"] -Orientation = Literal["xy", "xz", "zy"] +# Orientation = Literal["xy", "xz", "zy"] HALF_PX = 0.5 ENDIAN = "<" if sys.byteorder == "little" else ">" +class Orientation(IntEnum): + XY = 0 + # todo: check these + XZ = 1 + ZY = 2 + + def __bool__(self) -> bool: + return True + + def full_orientation(self, reverse=False) -> tuple[Dimension, Dimension, Dimension]: + out = [ + ("x", "y", "z"), + ("x", "z", "y"), + ("z", "y", "x"), + ][self.value] + if reverse: + out = out[::-1] + return out + + @classmethod + def from_dims(cls, dims: Sequence[Dimension]): + pair = (dims[0].lower(), dims[1].lower()) + out = { + ("x", "y"): cls.XY, + ("x", "z"): cls.XZ, + ("z", "y"): cls.ZY, + }.get(pair) + if out is None: + raise ValueError(f"Unknown dimensions: {dims}") + return out + + class MirrorInfo(BaseModel): id: int title: str @@ -35,35 +68,98 @@ class MirrorInfo(BaseModel): N = TypeVar("N", int, float) -class Coord(TypedDict, Generic[N]): - x: N - y: N - z: N - - class StackInfo(BaseModel): sid: int pid: int ptitle: str stitle: str - downsample_factors: list[Coord[float]] + downsample_factors: Optional[list[dict[Dimension, float]]] num_zoom_levels: int - translation: Coord[float] - resolution: Coord[float] - dimension: Coord[int] + translation: dict[Dimension, float] + resolution: dict[Dimension, float] + dimension: dict[Dimension, int] comment: str description: str - metadata: str + metadata: Optional[str] broken_slices: dict[int, int] mirrors: list[MirrorInfo] orientation: Orientation attribution: str - canary_location: Coord[int] - placeholder_colour: dict[str, float] # actually {r g b a} + canary_location: dict[Dimension, int] + placeholder_color: dict[str, float] # actually {r g b a} + + def get_downsample(self, scale_level=0) -> dict[Dimension, float]: + """Get the downsample factors for a given scale level. + + If the downsample factors are explicit in the stack info, + use that value. + Otherwise, use the CATMAID default: + scale by a factor of 2 per scale level. + If number of scale levels is known, + ensure the scale level exists. + + Parameters + ---------- + scale_level : int, optional + + Returns + ------- + dict[Dimension, float] + + Raises + ------ + IndexError + If the scale level is known not to exist + """ + if self.downsample_factors is not None: + return self.downsample_factors[scale_level] + if self.num_zoom_levels > 0 and scale_level >= self.num_zoom_levels: + raise IndexError("list index out of range") + + first, second, slicing = self.orientation.full_orientation() + return {first: 2**scale_level, second: 2**scale_level, slicing: 1} + + def to_coords(self, scale_level: int = 0) -> dict[Dimension, np.ndarray]: + dims = self.orientation.full_orientation() + # todo: not sure if this is desired? + dims = dims[::-1] + + downsamples = self.get_downsample(scale_level) + + out: dict[Dimension, np.ndarray] = dict() + for d in dims: + c = np.arange(self.dimension[d], dtype=float) + c *= self.resolution[d] + c *= downsamples[d] + c += self.translation[d] + out[d] = c + return out + + +def select_cli(prompt: str, options: dict[int, str]) -> Optional[int]: + out = None + print(prompt) + for k, v in sorted(options.items()): + print(f"\t{k}.\t{v}") + p = "Type number and press enter (empty to cancel): " + while out is None: + result_str = input(p).strip() + if not result_str: + break + try: + result = int(result_str) + except ValueError: + print("Not an integer, try again") + continue + if result not in options: + print("Not a valid option, try again") + continue + out = result + return out def to_array( - coord: Union[Coord[N], ArrayLike], + coord: Union[dict[Dimension, N], ArrayLike], dtype: DTypeLike = np.float64, order: Sequence[Dimension] = ("z", "y", "x"), ) -> np.ndarray: @@ -72,12 +168,13 @@ def to_array( return np.asarray(coord, dtype=dtype) -class TileStore(BaseStore, ABC): +class JpegStore(BaseStore, ABC): """ Must include instance variable 'fmt', which is a format string with variables: image_base, zoom_level, file_extension, row, col, slice_idx """ + tile_source_type: int fmt: str _writeable = False @@ -103,12 +200,12 @@ def __init__( else: self.session = session - order = full_orientation[self.stack_info.orientation] - self.metadata_payload = json.dumps( + order = self.stack_info.orientation.full_orientation(reverse=True) + self.metadata_bytes = json.dumps( { "zarr_format": 2, - "shape": to_array(stack_info.dimension, order, int).tolist(), - "chunks": [mirror_info.tile_width, mirror_info.tile_height, 1], + "shape": to_array(stack_info.dimension, int, order).tolist(), + "chunks": [1, mirror_info.tile_height, mirror_info.tile_width], "dtype": ENDIAN + "u1", "compressor": None, "fill_value": 0, @@ -117,6 +214,13 @@ def __init__( "dimension_separator": ".", } ).encode() + self.attrs_bytes = json.dumps( + { + "stack_info": self.stack_info.model_dump(), + "mirror_info": self.mirror_info.model_dump(), + "scale_level": self.zoom_level, + } + ).encode() self.empty = np.zeros( ( @@ -129,6 +233,7 @@ def __init__( def _format_url(self, row: int, col: int, slice_idx: int) -> str: return self.fmt.format( + image_base=self.mirror_info.image_base, zoom_level=self.zoom_level, slice_idx=slice_idx, row=row, @@ -136,49 +241,107 @@ def _format_url(self, row: int, col: int, slice_idx: int) -> str: file_extension=self.mirror_info.file_extension, ) + def __delitem__(self, __key) -> None: + raise NotImplementedError() + + def __iter__(self): + raise NotImplementedError() + + def __len__(self) -> int: + raise NotImplementedError() + + def __setitem__(self, __key, __value) -> None: + raise NotImplementedError() + def __getitem__(self, key): last = key.split("/")[-1] if last == ".zarray": - return self.metadata_payload + return self.metadata_bytes + elif last == ".zattrs": + return self.attrs_bytes + # todo: check order - slice_idx, col, row = (int(i) for i in last.split(".")) + slice_idx, row, col = (int(i) for i in last.split(".")) url = self._format_url(row, col, slice_idx) response = self.session.get(url) if response.status_code == 404: return self.empty response.raise_for_status() + ext = self.mirror_info.file_extension.split("?")[0] + if not ext.startswith("."): + ext = "." + ext arr = iio.imread( BytesIO(response.content), - extension=self.mirror_info.file_extension, + extension=ext, mode="L", ) return arr.tobytes() - def to_array(self) -> zarr.Array: + def to_zarr_array(self) -> zarr.Array: return zarr.open_array(self, "r") - def to_dask(self) -> da.Array: - return da.from_zarr(self.to_array()) + def to_dask_array(self) -> xr.DataArray: + # todo: transpose? + as_zarr = self.to_zarr_array() + return da.from_zarr(as_zarr) + + def to_xarray(self) -> xr.DataArray: + as_dask = self.to_dask_array() + return xr.DataArray( + as_dask, + coords=self.stack_info.to_coords(self.zoom_level), + dims=self.stack_info.orientation.full_orientation(True), + ) -class TileStore1(TileStore): +class TileStore1(JpegStore): tile_source_type = 1 fmt = "{image_base}{slice_idx}/{row}_{col}_{zoom_level}.{file_extension}" -class TileStore4(TileStore): +class TileStore4(JpegStore): tile_source_type = 4 fmt = "{image_base}{slice_idx}/{zoom_level}/{row}_{col}.{file_extension}" -class TileStore5(TileStore): +class TileStore5(JpegStore): tile_source_type = 5 fmt = "{image_base}{zoom_level}/{slice_idx}/{row}/{col}.{file_extension}" -tile_stores: dict[int, Type[TileStore]] = { - t.tile_source_type: t for t in [TileStore1, TileStore4, TileStore5] +# class TileStore10(JpegStore): +# tile_source_type = 10 +# fmt = "{image_base}.{file_extension}" + +# # todo: manually change quality? + +# def _format_url(self, row: int, col: int, slice_idx: int) -> str: +# s = self.fmt.format( +# image_base=self.mirror_info.image_base, +# file_extension=self.mirror_info.file_extension, +# ) +# s = s.replace("%SCALE_DATASET%", f"s{self.zoom_level}") +# s = s.replace("%AXIS_0%", str(col * self.mirror_info.tile_width)) +# s = s.replace("%AXIS_1%", str(row * self.mirror_info.tile_height)) +# s = s.replace("%AXIS_2%", str(slice_idx)) +# return s + + +tile_stores: dict[int, Type[JpegStore]] = { + t.tile_source_type: t for t in [ + TileStore1, TileStore4, TileStore5, + # TileStore10 + ] } +supported_sources = {11}.union(tile_stores) + + +def select_stack(remote_instance=None) -> Optional[int]: + cm = utils._eval_remote_instance(remote_instance) + url = cm.make_url(cm.project_id, "stacks") + stacks = cm.fetch(url) + options = {s["id"]: s["title"] for s in stacks} + return select_cli("Select stack:", options) class Stack: @@ -194,8 +357,9 @@ def from_catmaid( cls, stack_id: int, mirror_id: Optional[int] = None, remote_instance=None ): cm = utils._eval_remote_instance(remote_instance) - info = cm.make_url("stack", stack_id, "info") - sinfo = parse_obj_as(StackInfo, info) + url = cm.make_url(cm.project_id, "stack", stack_id, "info") + info = cm.fetch(url) + sinfo = StackInfo.model_validate(info) return cls(sinfo, mirror_id) def _get_mirror_info(self, mirror_id: Optional[int] = None) -> MirrorInfo: @@ -211,26 +375,56 @@ def _get_mirror_info(self, mirror_id: Optional[int] = None) -> MirrorInfo: ) def set_mirror(self, mirror_id: int): - self.mirror_id = self._get_mirror_info(mirror_id) - - def _res_for_scale(self, scale_level: int) -> np.ndarray: - return to_array(self.stack_info.resolution) * to_array( - self.stack_info.downsample_factors[scale_level] - ) - - def _from_array(self, arr, scale_level: int) -> ImageVolume: - return ImageVolume( - arr, - self.stack_info.translation, - self._res_for_scale(scale_level), - self.stack_info.orientation, + self.mirror_info = self._get_mirror_info(mirror_id) + + def select_mirror(self): + options = { + m.id: m.title + for m in self.stack_info.mirrors + if m.tile_source_type in supported_sources + } + if not options: + print("No mirrors with supported tile source type") + return + + result = select_cli( + f"Select mirror for stack '{self.stack_info.stitle}':", + options, ) + if result is not None: + self.set_mirror(result) def get_scale( self, scale_level: int, mirror_id: Optional[int] = None - ) -> ImageVolume: + ) -> xr.DataArray: + """Get an xarray.DataArray representing th given scale level. + + Note that depending on the metadata available, + missing scale levels may throw different errors. + + Parameters + ---------- + scale_level : int + 0 for full resolution + mirror_id : Optional[int], optional + By default the one set on the class. + + Returns + ------- + xr.DataArray + + Raises + ------ + ValueError + Scale level does not exist, according to metadata + NotImplementedError + Unknown tile source type for this mirror + """ mirror_info = self._get_mirror_info(mirror_id) - if scale_level > self.stack_info.num_zoom_levels: + if ( + self.stack_info.num_zoom_levels > 0 + and scale_level > self.stack_info.num_zoom_levels + ): raise ValueError( f"Scale level {scale_level} does not exist " f"for stack {self.stack_info.sid} " @@ -240,7 +434,7 @@ def get_scale( if mirror_info.tile_source_type in tile_stores: store_class = tile_stores[mirror_info.tile_source_type] store = store_class(self.stack_info, mirror_info, scale_level, None) - return self._from_array(store.to_dask(), scale_level) + return store.to_xarray() elif mirror_info.tile_source_type == 11: formatted = mirror_info.image_base.replace( "%SCALE_DATASET%", f"s{scale_level}" @@ -248,65 +442,28 @@ def get_scale( *components, transpose_str = formatted.split("/") transpose = [int(t) for t in transpose_str.split("_")] - store = zarr.N5FSStore("/".join(components)) - arr = zarr.open_array(store, "r") - darr = da.from_zarr(arr).transpose(transpose) - return self._from_array(darr, scale_level) + container_comp = [] + arr_comp = [] + this = container_comp + for comp in components: + this.append(comp) + if comp.lower().endswith(".n5"): + this = arr_comp + + if not arr_comp: + raise ValueError("N5 container must have '.n5' suffix") + + store = zarr.N5FSStore("/".join(container_comp)) + container = zarr.open(store, "r") + as_zarr = container["/".join(arr_comp)] + # todo: check this transpose + as_dask = da.from_zarr(as_zarr).transpose(transpose) + return xr.DataArray( + as_dask, + coords=self.stack_info.to_coords(scale_level), + dims=self.stack_info.orientation.full_orientation(True), + ) raise NotImplementedError( f"Tile source type {mirror_info.tile_source_type} not implemented" ) - - -full_orientation: dict[Orientation, Sequence[Dimension]] = { - "xy": "xyz", - "xz": "xzy", - "zy": "zyx", -} - - -class ImageVolume: - def __init__(self, array, offset, resolution, orientation: Orientation): - self.array = array - self.offset = offset - self.resolution = resolution - self.offset = to_array(offset, dtype="float64") - self.resolution = to_array(resolution, dtype="float64") - self.orientation = orientation - - @property - def full_orientation(self): - return full_orientation[self.orientation] - - @property - def offset_oriented(self): - return to_array(self.offset, "float64", self.full_orientation) - - @property - def resolution_oriented(self): - return to_array(self.resolution, "float64", self.full_orientation) - - def __getitem__(self, selection): - return self.array.__getitem__(selection) - - def get_roi( - self, offset: Coord[float], shape: Coord[float] - ) -> tuple[Coord[float], np.ndarray]: - order = self.full_orientation - offset_o = to_array(offset, order=order) - shape_o = to_array(shape, order=order) - mins = (offset_o / self.resolution - self.offset - HALF_PX).astype("uint64") - maxes = np.ceil( - (offset_o + shape_o) / self.resolution - self.offset - HALF_PX - ).astype("uint64") - slicing = tuple(slice(mi, ma) for mi, ma in zip(mins, maxes)) - # todo: finalise orientation - actual_offset = Coord( - **{ - d: m - for d, m in zip( - order, mins * self.resolution_oriented + self.offset_oriented - ) - } - ) - return actual_offset, self[slicing] diff --git a/requirements.txt b/requirements.txt index b02e109..2f42d16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,8 +13,12 @@ psutil>=5.4.3 #extra: extras fuzzywuzzy[speedup]~=0.17.0 ujson~=1.35 + +#extra: stacks zarr -pydantic +fsspec[http] +pydantic>=2 +xarray[parallel] imageio # diskcache>=4.0.0 From baa0fb5c4c168e8c5a1ffcb46e7d1349a6fbc869 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Fri, 14 Jul 2023 12:43:56 +0100 Subject: [PATCH 05/20] handle broken slices --- pymaid/stack.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pymaid/stack.py b/pymaid/stack.py index cb4a63e..cda3989 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -253,6 +253,13 @@ def __len__(self) -> int: def __setitem__(self, __key, __value) -> None: raise NotImplementedError() + def _resolve_broken_slices(self, slice_idx: int) -> int: + while True: + incr = self.stack_info.broken_slices.get(slice_idx) + if incr is None: + return slice_idx + slice_idx += incr + def __getitem__(self, key): last = key.split("/")[-1] if last == ".zarray": @@ -262,6 +269,8 @@ def __getitem__(self, key): # todo: check order slice_idx, row, col = (int(i) for i in last.split(".")) + slice_idx = self._resolve_broken_slices(slice_idx) + url = self._format_url(row, col, slice_idx) response = self.session.get(url) if response.status_code == 404: @@ -436,6 +445,9 @@ def get_scale( store = store_class(self.stack_info, mirror_info, scale_level, None) return store.to_xarray() elif mirror_info.tile_source_type == 11: + # do we need to handle broken slices here? + # or is that metadata just telling the frontend to skip regions which exist + # (as fill values) in the N5? formatted = mirror_info.image_base.replace( "%SCALE_DATASET%", f"s{scale_level}" ) From 32d6a1b5fc4f57f71a27a7114f0ad0b4d9e9d88b Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Fri, 14 Jul 2023 12:47:14 +0100 Subject: [PATCH 06/20] better broken slice handling --- pymaid/stack.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pymaid/stack.py b/pymaid/stack.py index cda3989..b2694c8 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -200,6 +200,13 @@ def __init__( else: self.session = session + brok_sl = {int(k): int(k) + v for k, v in self.stack_info.broken_slices.items()} + self.broken_slices = dict() + for k, v in brok_sl.items(): + while v in brok_sl: + v = brok_sl[v] + self.broken_slices[k] = v + order = self.stack_info.orientation.full_orientation(reverse=True) self.metadata_bytes = json.dumps( { @@ -254,11 +261,7 @@ def __setitem__(self, __key, __value) -> None: raise NotImplementedError() def _resolve_broken_slices(self, slice_idx: int) -> int: - while True: - incr = self.stack_info.broken_slices.get(slice_idx) - if incr is None: - return slice_idx - slice_idx += incr + return self.broken_slices.get(slice_idx, slice_idx) def __getitem__(self, key): last = key.split("/")[-1] @@ -337,8 +340,11 @@ class TileStore5(JpegStore): tile_stores: dict[int, Type[JpegStore]] = { - t.tile_source_type: t for t in [ - TileStore1, TileStore4, TileStore5, + t.tile_source_type: t + for t in [ + TileStore1, + TileStore4, + TileStore5, # TileStore10 ] } From 6d884e61b58eb99d8a86c600f064006f11bd80a0 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Fri, 14 Jul 2023 14:10:11 +0100 Subject: [PATCH 07/20] functions for getting stack info --- docs/source/api.rst | 11 ++ docs/source/whats_new.rst | 1 + pymaid/fetch/__init__.py | 2 + pymaid/fetch/stack.py | 251 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 265 insertions(+) create mode 100644 pymaid/fetch/stack.py diff --git a/docs/source/api.rst b/docs/source/api.rst index 6b2274d..02efe22 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -172,6 +172,17 @@ Functions for reconstruction samplers: pymaid.get_sampler_domains pymaid.get_sampler_counts +Image metadata +-------------- +Functions to fetch information about the image stacks CATMAID knows about. + +.. autosummary:: + :toctree: generated/ + + pymaid.fetch.stacks.get_stacks + pymaid.fetch.stacks.get_stack_info + pymaid.fetch.stacks.get_mirror_info + Image data (tiles) ------------------ Functions to fetch and process image data. Note that this is not imported at diff --git a/docs/source/whats_new.rst b/docs/source/whats_new.rst index cb6fc6d..b273ba2 100644 --- a/docs/source/whats_new.rst +++ b/docs/source/whats_new.rst @@ -19,6 +19,7 @@ What's new? - 27/05/23 - - :func:`pymaid.get_annotation_graph` deprecated in favour of the new :func:`pymaid.get_entity_graph`. + - :func:`pymaid.get_stacks`, :func:`pymaid.get_stack_info`, :func:`pymaid.get_mirror_info` functions for getting information about image data * - 2.1.0 - 04/04/22 - With this release we mainly follow some renamed functions in ``navis`` but diff --git a/pymaid/fetch/__init__.py b/pymaid/fetch/__init__.py index fc150b7..f01ae11 100644 --- a/pymaid/fetch/__init__.py +++ b/pymaid/fetch/__init__.py @@ -57,6 +57,7 @@ from .landmarks import get_landmarks, get_landmark_groups from .skeletons import get_skeleton_ids from .annotations import get_annotation_graph, get_entity_graph, get_annotation_id +from .stack import get_stacks, get_stack_info, get_mirror_info __all__ = ['get_annotation_details', 'get_annotation_id', @@ -93,6 +94,7 @@ 'get_landmarks', 'get_landmark_groups', 'get_skeleton_ids', + 'get_stacks', 'get_stack_info', 'get_mirror_info', ] # Set up logging diff --git a/pymaid/fetch/stack.py b/pymaid/fetch/stack.py new file mode 100644 index 0000000..04b9cc2 --- /dev/null +++ b/pymaid/fetch/stack.py @@ -0,0 +1,251 @@ +from typing import Any, Optional, Union, Literal, Sequence +import numpy as np +from ..utils import _eval_remote_instance +from ..client import CatmaidInstance +from enum import IntEnum +from dataclasses import dataclass, asdict + +Dimension = Literal["x", "y", "z"] + + +class Orientation(IntEnum): + XY = 0 + # todo: check these + XZ = 1 + ZY = 2 + + def __bool__(self) -> bool: + return True + + def full_orientation(self, reverse=False) -> tuple[Dimension, Dimension, Dimension]: + out = [ + ("x", "y", "z"), + ("x", "z", "y"), + ("z", "y", "x"), + ][self.value] + if reverse: + out = out[::-1] + return out + + @classmethod + def from_dims(cls, dims: Sequence[Dimension]): + pair = (dims[0].lower(), dims[1].lower()) + out = { + ("x", "y"): cls.XY, + ("x", "z"): cls.XZ, + ("z", "y"): cls.ZY, + }.get(pair) + if out is None: + raise ValueError(f"Unknown dimensions: {dims}") + return out + + +@dataclass +class StackSummary: + id: int + pid: int + title: str + comment: str + + +def get_stacks(remote_instance: Optional[CatmaidInstance] = None) -> list[StackSummary]: + """Get summary of all stacks in the project. + + Parameters + ---------- + remote_instance : Optional[CatmaidInstance], optional + By default global instance. + + Returns + ------- + stacks + List of StackSummary objects. + """ + cm = _eval_remote_instance(remote_instance) + url = cm.make_url(cm.project_id, "stacks") + return [StackSummary(**r) for r in cm.fetch(url)] + + +@dataclass +class MirrorInfo: + id: int + title: str + image_base: str + tile_width: int + tile_height: int + tile_source_type: int + file_extension: str + position: int + + def to_jso(self): + return asdict(self) + + +@dataclass +class Color: + r: float + g: float + b: float + a: float + + +@dataclass +class StackInfo: + sid: int + pid: int + ptitle: str + stitle: str + downsample_factors: Optional[list[dict[Dimension, float]]] + num_zoom_levels: int + translation: dict[Dimension, float] + resolution: dict[Dimension, float] + dimension: dict[Dimension, int] + comment: str + description: str + metadata: Optional[str] + broken_slices: dict[int, int] + mirrors: list[MirrorInfo] + orientation: Orientation + attribution: str + canary_location: dict[Dimension, int] + placeholder_color: Color + + @classmethod + def from_jso(cls, sinfo: dict[str, Any]): + sinfo["orientation"] = Orientation(sinfo["orientation"]) + sinfo["placeholder_color"] = Color(**sinfo["placeholder_color"]) + sinfo["mirrors"] = [MirrorInfo(**m) for m in sinfo["mirrors"]] + return StackInfo(**sinfo) + + def to_jso(self): + return asdict(self) + + def get_downsample(self, scale_level=0) -> dict[Dimension, float]: + """Get the downsample factors for a given scale level. + + If the downsample factors are explicit in the stack info, + use that value. + Otherwise, use the CATMAID default: + scale by a factor of 2 per scale level in everything except the slicing dimension. + If number of scale levels is known, + ensure the scale level exists. + + Parameters + ---------- + scale_level : int, optional + + Returns + ------- + dict[Dimension, float] + + Raises + ------ + IndexError + If the scale level is known not to exist + """ + if self.downsample_factors is not None: + return self.downsample_factors[scale_level] + if self.num_zoom_levels > 0 and scale_level >= self.num_zoom_levels: + raise IndexError("list index out of range") + + first, second, slicing = self.orientation.full_orientation() + return {first: 2**scale_level, second: 2**scale_level, slicing: 1} + + def get_coords(self, scale_level: int = 0) -> dict[Dimension, np.ndarray]: + dims = self.orientation.full_orientation() + dims = dims[::-1] + + downsamples = self.get_downsample(scale_level) + + out: dict[Dimension, np.ndarray] = dict() + for d in dims: + c = np.arange(self.dimension[d], dtype=float) + c *= self.resolution[d] + c *= downsamples[d] + c += self.translation[d] + out[d] = c + return out + + +def get_stack_info( + stack: Union[int, str], remote_instance: Optional[CatmaidInstance] = None +) -> StackInfo: + """Get information about an image stack. + + Parameters + ---------- + stack : Union[int, str] + Integer ID or string title of the stack. + remote_instance : Optional[CatmaidInstance], optional + By default global. + + Returns + ------- + StackInfo + + Raises + ------ + ValueError + If an unknown stack title is given. + """ + cm = _eval_remote_instance(remote_instance) + if isinstance(stack, str): + stacks = get_stacks(cm) + for s in stacks: + if s.title == stack: + stack_id = s.id + break + else: + raise ValueError(f"No stack with title '{stack}'") + else: + stack_id = int(stack) + + url = cm.make_url(cm.project_id, "stack", stack_id, "info") + sinfo = cm.fetch(url) + return StackInfo.from_jso(sinfo) + + +def get_mirror_info( + stack: Union[int, str, StackInfo], + mirror: Union[int, str], + remote_instance: Optional[CatmaidInstance] = None, +) -> MirrorInfo: + """Get information about a stack mirror. + + Parameters + ---------- + stack : Union[int, str, StackInfo] + Integer stack ID, string stack title, + or an existing StackInfo object (avoids server request). + mirror : Union[int, str] + Integer mirror ID, or string mirror title. + remote_instance : Optional[CatmaidInstance] + By default, global. + + Returns + ------- + MirrorInfo + + Raises + ------ + ValueError + No mirror matching given ID/ title. + """ + if isinstance(stack, StackInfo): + stack_info = stack + else: + stack_info = get_stack_info(stack, remote_instance) + + if isinstance(mirror, str): + key = "title" + else: + key = "id" + mirror = int(mirror) + + for m in stack_info.mirrors: + if getattr(m, key) == mirror: + return m + + raise ValueError( + f"No mirror for stack '{stack_info.stitle}' with {key} {repr(mirror)}" + ) From 1025730402e1e6d1c3f72a98b38d679963191144 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Fri, 14 Jul 2023 14:10:30 +0100 Subject: [PATCH 08/20] refactor to make use of pymaid data fetching --- pymaid/stack.py | 161 +++++++---------------------------------------- requirements.txt | 1 - 2 files changed, 23 insertions(+), 139 deletions(-) diff --git a/pymaid/stack.py b/pymaid/stack.py index b2694c8..5535c98 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -1,141 +1,34 @@ from __future__ import annotations from io import BytesIO -from typing import Literal, Optional, Sequence, Type, TypeVar, Union -import numpy as np +from typing import Any, Literal, Optional, Sequence, Type, Union from abc import ABC -from enum import IntEnum +import sys + +import numpy as np from numpy.typing import DTypeLike, ArrayLike import zarr -from pydantic import BaseModel from dask import array as da import xarray as xr -from . import utils from zarr.storage import BaseStore import json -import sys import requests import imageio.v3 as iio +from . import utils +from .fetch.stack import ( + StackInfo, + MirrorInfo, + get_stacks, + get_stack_info, + get_mirror_info, +) + Dimension = Literal["x", "y", "z"] # Orientation = Literal["xy", "xz", "zy"] HALF_PX = 0.5 ENDIAN = "<" if sys.byteorder == "little" else ">" -class Orientation(IntEnum): - XY = 0 - # todo: check these - XZ = 1 - ZY = 2 - - def __bool__(self) -> bool: - return True - - def full_orientation(self, reverse=False) -> tuple[Dimension, Dimension, Dimension]: - out = [ - ("x", "y", "z"), - ("x", "z", "y"), - ("z", "y", "x"), - ][self.value] - if reverse: - out = out[::-1] - return out - - @classmethod - def from_dims(cls, dims: Sequence[Dimension]): - pair = (dims[0].lower(), dims[1].lower()) - out = { - ("x", "y"): cls.XY, - ("x", "z"): cls.XZ, - ("z", "y"): cls.ZY, - }.get(pair) - if out is None: - raise ValueError(f"Unknown dimensions: {dims}") - return out - - -class MirrorInfo(BaseModel): - id: int - title: str - image_base: str - tile_width: int - tile_height: int - tile_source_type: int - file_extension: str - position: int - - -N = TypeVar("N", int, float) - - -class StackInfo(BaseModel): - sid: int - pid: int - ptitle: str - stitle: str - downsample_factors: Optional[list[dict[Dimension, float]]] - num_zoom_levels: int - translation: dict[Dimension, float] - resolution: dict[Dimension, float] - dimension: dict[Dimension, int] - comment: str - description: str - metadata: Optional[str] - broken_slices: dict[int, int] - mirrors: list[MirrorInfo] - orientation: Orientation - attribution: str - canary_location: dict[Dimension, int] - placeholder_color: dict[str, float] # actually {r g b a} - - def get_downsample(self, scale_level=0) -> dict[Dimension, float]: - """Get the downsample factors for a given scale level. - - If the downsample factors are explicit in the stack info, - use that value. - Otherwise, use the CATMAID default: - scale by a factor of 2 per scale level. - If number of scale levels is known, - ensure the scale level exists. - - Parameters - ---------- - scale_level : int, optional - - Returns - ------- - dict[Dimension, float] - - Raises - ------ - IndexError - If the scale level is known not to exist - """ - if self.downsample_factors is not None: - return self.downsample_factors[scale_level] - if self.num_zoom_levels > 0 and scale_level >= self.num_zoom_levels: - raise IndexError("list index out of range") - - first, second, slicing = self.orientation.full_orientation() - return {first: 2**scale_level, second: 2**scale_level, slicing: 1} - - def to_coords(self, scale_level: int = 0) -> dict[Dimension, np.ndarray]: - dims = self.orientation.full_orientation() - # todo: not sure if this is desired? - dims = dims[::-1] - - downsamples = self.get_downsample(scale_level) - - out: dict[Dimension, np.ndarray] = dict() - for d in dims: - c = np.arange(self.dimension[d], dtype=float) - c *= self.resolution[d] - c *= downsamples[d] - c += self.translation[d] - out[d] = c - return out - - def select_cli(prompt: str, options: dict[int, str]) -> Optional[int]: out = None print(prompt) @@ -159,7 +52,7 @@ def select_cli(prompt: str, options: dict[int, str]) -> Optional[int]: def to_array( - coord: Union[dict[Dimension, N], ArrayLike], + coord: Union[dict[Dimension, Any], ArrayLike], dtype: DTypeLike = np.float64, order: Sequence[Dimension] = ("z", "y", "x"), ) -> np.ndarray: @@ -223,8 +116,8 @@ def __init__( ).encode() self.attrs_bytes = json.dumps( { - "stack_info": self.stack_info.model_dump(), - "mirror_info": self.mirror_info.model_dump(), + "stack_info": self.stack_info.to_jso(), + "mirror_info": self.mirror_info.to_jso(), "scale_level": self.zoom_level, } ).encode() @@ -301,7 +194,7 @@ def to_xarray(self) -> xr.DataArray: as_dask = self.to_dask_array() return xr.DataArray( as_dask, - coords=self.stack_info.to_coords(self.zoom_level), + coords=self.stack_info.get_coords(self.zoom_level), dims=self.stack_info.orientation.full_orientation(True), ) @@ -352,10 +245,8 @@ class TileStore5(JpegStore): def select_stack(remote_instance=None) -> Optional[int]: - cm = utils._eval_remote_instance(remote_instance) - url = cm.make_url(cm.project_id, "stacks") - stacks = cm.fetch(url) - options = {s["id"]: s["title"] for s in stacks} + stacks = get_stacks(remote_instance) + options = {s.id: s.title for s in stacks} return select_cli("Select stack:", options) @@ -372,9 +263,7 @@ def from_catmaid( cls, stack_id: int, mirror_id: Optional[int] = None, remote_instance=None ): cm = utils._eval_remote_instance(remote_instance) - url = cm.make_url(cm.project_id, "stack", stack_id, "info") - info = cm.fetch(url) - sinfo = StackInfo.model_validate(info) + sinfo = get_stack_info(stack_id, cm) return cls(sinfo, mirror_id) def _get_mirror_info(self, mirror_id: Optional[int] = None) -> MirrorInfo: @@ -382,12 +271,8 @@ def _get_mirror_info(self, mirror_id: Optional[int] = None) -> MirrorInfo: if self.mirror_info is None: raise ValueError("No default mirror ID set") return self.mirror_info - for mirror in self.stack_info.mirrors: - if mirror.id == mirror_id: - return mirror - raise ValueError( - f"Mirror ID {mirror_id} not found for stack {self.stack_info.sid}" - ) + + return get_mirror_info(self.stack_info, mirror_id) def set_mirror(self, mirror_id: int): self.mirror_info = self._get_mirror_info(mirror_id) @@ -478,7 +363,7 @@ def get_scale( as_dask = da.from_zarr(as_zarr).transpose(transpose) return xr.DataArray( as_dask, - coords=self.stack_info.to_coords(scale_level), + coords=self.stack_info.get_coords(scale_level), dims=self.stack_info.orientation.full_orientation(True), ) diff --git a/requirements.txt b/requirements.txt index 2f42d16..21021c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,6 @@ ujson~=1.35 #extra: stacks zarr fsspec[http] -pydantic>=2 xarray[parallel] imageio From 0baf2326ae272da3e4fb78cb9fabba07fafc4e07 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Fri, 14 Jul 2023 14:16:29 +0100 Subject: [PATCH 09/20] use shorter imports in api docs --- docs/source/api.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index 02efe22..1a44f40 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -179,9 +179,9 @@ Functions to fetch information about the image stacks CATMAID knows about. .. autosummary:: :toctree: generated/ - pymaid.fetch.stacks.get_stacks - pymaid.fetch.stacks.get_stack_info - pymaid.fetch.stacks.get_mirror_info + pymaid.get_stacks + pymaid.get_stack_info + pymaid.get_mirror_info Image data (tiles) ------------------ From f17ec7bae0312b7e09810b1f5f5d1f078b039a89 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Fri, 14 Jul 2023 14:31:40 +0100 Subject: [PATCH 10/20] docs for Stack class --- docs/source/api.rst | 5 ++- docs/source/whats_new.rst | 3 +- pymaid/stack.py | 71 ++++++++++++++++++++++++++++++++------- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index 1a44f40..58f5f88 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -183,19 +183,22 @@ Functions to fetch information about the image stacks CATMAID knows about. pymaid.get_stack_info pymaid.get_mirror_info -Image data (tiles) +Image data (tiles and N5 volumes) ------------------ Functions to fetch and process image data. Note that this is not imported at top level but has to be imported explicitly:: >>> from pymaid import tiles >>> help(tiles.crop_neuron) + >>> from pymaid.stack import Stack + >>> help(Stack) .. autosummary:: :toctree: generated/ pymaid.tiles.TileLoader pymaid.tiles.crop_neuron + pymaid.stack.Stack .. _api_misc: diff --git a/docs/source/whats_new.rst b/docs/source/whats_new.rst index b273ba2..8975c0d 100644 --- a/docs/source/whats_new.rst +++ b/docs/source/whats_new.rst @@ -15,11 +15,12 @@ What's new? - BREAKING: Drop python 3.7 support. - - :class:`pymaid.neuron_label.NeuronLabeller` added for labelling neurons like in the CATMAID frontend. + - :func:`pymaid.get_stacks`, :func:`pymaid.get_stack_info`, :func:`pymaid.get_mirror_info` functions for getting information about image data + - :class:`pymaid.stack.Stack` class for accessing N5 and JPEG tile image data as a :class:`xarray.DataArray` * - 2.4.0 - 27/05/23 - - :func:`pymaid.get_annotation_graph` deprecated in favour of the new :func:`pymaid.get_entity_graph`. - - :func:`pymaid.get_stacks`, :func:`pymaid.get_stack_info`, :func:`pymaid.get_mirror_info` functions for getting information about image data * - 2.1.0 - 04/04/22 - With this release we mainly follow some renamed functions in ``navis`` but diff --git a/pymaid/stack.py b/pymaid/stack.py index 5535c98..d6ac3d7 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -245,39 +245,85 @@ class TileStore5(JpegStore): def select_stack(remote_instance=None) -> Optional[int]: + """""" stacks = get_stacks(remote_instance) options = {s.id: s.title for s in stacks} return select_cli("Select stack:", options) class Stack: - def __init__(self, stack_info: StackInfo, mirror_id: Optional[int] = None): + """Class representing a CATMAID stack of images. + + Stacks are usually a scale pyramid. + This class can, for certain stack mirror types, + allow access to individual scale levels as arrays + which can be queried in voxel or world coordinates. + """ + def __init__(self, stack_info: StackInfo, mirror: Optional[Union[int, str]] = None): + """The :func:`Stack.from_catmaid` constructor may be more convenient. + + Parameters + ---------- + stack_info : StackInfo + mirror_id : Optional[int], optional + """ self.stack_info = stack_info self.mirror_info: Optional[MirrorInfo] = None - if mirror_id is not None: - self.set_mirror(mirror_id) + if mirror is not None: + self.set_mirror(mirror) + + @classmethod + def select_from_catmaid(cls, remote_instance=None): + """Interactively select a stack and mirror from those available. + + Parameters + ---------- + remote_instance : CatmaidInstance, optional + By default global. + """ + stacks = get_stacks(remote_instance) + options = {s.id: s.title for s in stacks} + sid = select_cli("Select stack:", options) + if not sid: + return None + out = cls.from_catmaid(sid, remote_instance=remote_instance) + out.select_mirror() + return out @classmethod def from_catmaid( - cls, stack_id: int, mirror_id: Optional[int] = None, remote_instance=None + cls, stack: Union[str, int], mirror: Optional[Union[int, str]] = None, remote_instance=None ): - cm = utils._eval_remote_instance(remote_instance) - sinfo = get_stack_info(stack_id, cm) - return cls(sinfo, mirror_id) + """Fetch relevant data from CATMAID and build a Stack. - def _get_mirror_info(self, mirror_id: Optional[int] = None) -> MirrorInfo: - if mirror_id is None: + Parameters + ---------- + stack : Union[str, int] + Integer stack ID or string stack title. + mirror : Optional[int, str], optional + Integer mirror ID or string mirror title, by default None + remote_instance : CatmaidInstance, optional + By default global. + """ + sinfo = get_stack_info(stack, remote_instance) + return cls(sinfo, mirror) + + def _get_mirror_info(self, mirror: Optional[Union[int, str]] = None) -> MirrorInfo: + if mirror is None: if self.mirror_info is None: raise ValueError("No default mirror ID set") return self.mirror_info - return get_mirror_info(self.stack_info, mirror_id) + return get_mirror_info(self.stack_info, mirror) - def set_mirror(self, mirror_id: int): - self.mirror_info = self._get_mirror_info(mirror_id) + def set_mirror(self, mirror: Union[int, str]): + """Set the mirror using its int ID or str title.""" + self.mirror_info = self._get_mirror_info(mirror) def select_mirror(self): + """Interactively select a mirror from those available. + """ options = { m.id: m.title for m in self.stack_info.mirrors @@ -312,6 +358,7 @@ def get_scale( Returns ------- xr.DataArray + Can be queried in voxel or world space. Raises ------ From f16364bdb272f7ec3a4c9f48674c676a2e539772 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Fri, 28 Jul 2023 16:40:11 +0100 Subject: [PATCH 11/20] Use old typing annotations --- pymaid/fetch/annotations.py | 2 +- pymaid/fetch/stack.py | 28 ++++++++++++++-------------- pymaid/neuron_label.py | 22 +++++++++++----------- pymaid/stack.py | 6 +++--- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/pymaid/fetch/annotations.py b/pymaid/fetch/annotations.py index 1db874e..cba400a 100644 --- a/pymaid/fetch/annotations.py +++ b/pymaid/fetch/annotations.py @@ -391,7 +391,7 @@ def get_entity_graph( Neurons additionally have - - skeleton_ids: list[int] + - skeleton_ids: List[int] Skeletons additionally have diff --git a/pymaid/fetch/stack.py b/pymaid/fetch/stack.py index 04b9cc2..0ac8997 100644 --- a/pymaid/fetch/stack.py +++ b/pymaid/fetch/stack.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Union, Literal, Sequence +from typing import Any, Optional, Union, Literal, Sequence, Tuple, Dict, List import numpy as np from ..utils import _eval_remote_instance from ..client import CatmaidInstance @@ -17,7 +17,7 @@ class Orientation(IntEnum): def __bool__(self) -> bool: return True - def full_orientation(self, reverse=False) -> tuple[Dimension, Dimension, Dimension]: + def full_orientation(self, reverse=False) -> Tuple[Dimension, Dimension, Dimension]: out = [ ("x", "y", "z"), ("x", "z", "y"), @@ -48,7 +48,7 @@ class StackSummary: comment: str -def get_stacks(remote_instance: Optional[CatmaidInstance] = None) -> list[StackSummary]: +def get_stacks(remote_instance: Optional[CatmaidInstance] = None) -> List[StackSummary]: """Get summary of all stacks in the project. Parameters @@ -95,23 +95,23 @@ class StackInfo: pid: int ptitle: str stitle: str - downsample_factors: Optional[list[dict[Dimension, float]]] + downsample_factors: Optional[List[Dict[Dimension, float]]] num_zoom_levels: int - translation: dict[Dimension, float] - resolution: dict[Dimension, float] - dimension: dict[Dimension, int] + translation: Dict[Dimension, float] + resolution: Dict[Dimension, float] + dimension: Dict[Dimension, int] comment: str description: str metadata: Optional[str] - broken_slices: dict[int, int] - mirrors: list[MirrorInfo] + broken_slices: Dict[int, int] + mirrors: List[MirrorInfo] orientation: Orientation attribution: str - canary_location: dict[Dimension, int] + canary_location: Dict[Dimension, int] placeholder_color: Color @classmethod - def from_jso(cls, sinfo: dict[str, Any]): + def from_jso(cls, sinfo: Dict[str, Any]): sinfo["orientation"] = Orientation(sinfo["orientation"]) sinfo["placeholder_color"] = Color(**sinfo["placeholder_color"]) sinfo["mirrors"] = [MirrorInfo(**m) for m in sinfo["mirrors"]] @@ -120,7 +120,7 @@ def from_jso(cls, sinfo: dict[str, Any]): def to_jso(self): return asdict(self) - def get_downsample(self, scale_level=0) -> dict[Dimension, float]: + def get_downsample(self, scale_level=0) -> Dict[Dimension, float]: """Get the downsample factors for a given scale level. If the downsample factors are explicit in the stack info, @@ -151,13 +151,13 @@ def get_downsample(self, scale_level=0) -> dict[Dimension, float]: first, second, slicing = self.orientation.full_orientation() return {first: 2**scale_level, second: 2**scale_level, slicing: 1} - def get_coords(self, scale_level: int = 0) -> dict[Dimension, np.ndarray]: + def get_coords(self, scale_level: int = 0) -> Dict[Dimension, np.ndarray]: dims = self.orientation.full_orientation() dims = dims[::-1] downsamples = self.get_downsample(scale_level) - out: dict[Dimension, np.ndarray] = dict() + out: Dict[Dimension, np.ndarray] = dict() for d in dims: c = np.arange(self.dimension[d], dtype=float) c *= self.resolution[d] diff --git a/pymaid/neuron_label.py b/pymaid/neuron_label.py index 4f90844..ae46cbb 100644 --- a/pymaid/neuron_label.py +++ b/pymaid/neuron_label.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from functools import cache -from typing import Optional, Union +from typing import Optional, Union, List, Tuple import re import networkx as nx @@ -38,7 +38,7 @@ def __init__( self, skeleton_id: Optional[int] = None, name: Optional[str] = None, - annotations: Optional[list[str]] = None, + annotations: Optional[List[str]] = None, remote_instance: Optional[CatmaidInstance] = None, ) -> None: """ @@ -51,7 +51,7 @@ def __init__( If None, determined from name. name : Optional[str], optional If None, determined from skeleton ID. - annotations : Optional[list[str]], optional + annotations : Optional[List[str]], optional If None, determined from skeleton ID or name. remote_instance : Optional[CatmaidInstance], optional If None, uses global instance. @@ -90,7 +90,7 @@ def name(self) -> str: return self._name @property - def annotations(self) -> list[str]: + def annotations(self) -> List[str]: if self._annotations is None: skid = self.skeleton_id skid_to_anns = pymaid.get_annotations(skid) @@ -155,8 +155,8 @@ def __init__( super().__init__() def _filter_by_author( - self, annotations: list[str], remote_instance: CatmaidInstance - ) -> list[str]: + self, annotations: List[str], remote_instance: CatmaidInstance + ) -> List[str]: if self.annotator_name is None or not annotations: return annotations @@ -166,8 +166,8 @@ def _filter_by_author( return [a for a in annotations if a in allowed] def _filter_by_annotation( - self, annotations: list[str], remote_instance: CatmaidInstance - ) -> list[str]: + self, annotations: List[str], remote_instance: CatmaidInstance + ) -> List[str]: if self.annotated_with is None or not annotations: return annotations @@ -193,7 +193,7 @@ def dedup_whitespace(s: str): @cache def parse_components( fmt: str, -) -> tuple[list[str], list[tuple[str, int, Optional[str]]]]: +) -> Tuple[List[str], List[Tuple[str, int, Optional[str]]]]: joiners = [] components = [] last_end = 0 @@ -264,7 +264,7 @@ class NeuronLabeller: """Class for calculating neurons' labels, as used in the CATMAID frontend.""" def __init__( self, - components: Optional[list[LabelComponent]] = None, + components: Optional[List[LabelComponent]] = None, fmt="%0", trim_empty=True, remove_neighboring_duplicates=True, @@ -273,7 +273,7 @@ def __init__( Parameters ---------- - components : list[LabelComponent], optional + components : List[LabelComponent], optional The label components as used in CATMAID's user settings. See `SkeletonId`, `NeuronName`, and `Annotations`. First component should be ``SkeletonId()`` for compatibility with CATMAID. diff --git a/pymaid/stack.py b/pymaid/stack.py index d6ac3d7..0144170 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -29,7 +29,7 @@ ENDIAN = "<" if sys.byteorder == "little" else ">" -def select_cli(prompt: str, options: dict[int, str]) -> Optional[int]: +def select_cli(prompt: str, options: Dict[int, str]) -> Optional[int]: out = None print(prompt) for k, v in sorted(options.items()): @@ -52,7 +52,7 @@ def select_cli(prompt: str, options: dict[int, str]) -> Optional[int]: def to_array( - coord: Union[dict[Dimension, Any], ArrayLike], + coord: Union[Dict[Dimension, Any], ArrayLike], dtype: DTypeLike = np.float64, order: Sequence[Dimension] = ("z", "y", "x"), ) -> np.ndarray: @@ -232,7 +232,7 @@ class TileStore5(JpegStore): # return s -tile_stores: dict[int, Type[JpegStore]] = { +tile_stores: Dict[int, Type[JpegStore]] = { t.tile_source_type: t for t in [ TileStore1, From 5de01b01f762c2f6b392366c62a91605c6f15a7c Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Thu, 9 Nov 2023 12:13:40 +0000 Subject: [PATCH 12/20] Refactor for clarity --- pymaid/stack.py | 169 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 125 insertions(+), 44 deletions(-) diff --git a/pymaid/stack.py b/pymaid/stack.py index 0144170..ea2f620 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -1,6 +1,11 @@ +"""Access to image data as xarray.DataArrays. + +CATMAID's image source conventions are documented here +https://catmaid.readthedocs.io/en/stable/tile_sources.html +""" from __future__ import annotations from io import BytesIO -from typing import Any, Literal, Optional, Sequence, Type, Union +from typing import Any, Callable, Literal, Optional, Sequence, Type, Union, Dict from abc import ABC import sys @@ -14,6 +19,8 @@ import requests import imageio.v3 as iio +from pymaid.client import CatmaidInstance + from . import utils from .fetch.stack import ( StackInfo, @@ -61,7 +68,7 @@ def to_array( return np.asarray(coord, dtype=dtype) -class JpegStore(BaseStore, ABC): +class ImageIoStore(BaseStore, ABC): """ Must include instance variable 'fmt', which is a format string with variables: @@ -79,7 +86,7 @@ def __init__( stack_info: StackInfo, mirror_info: MirrorInfo, zoom_level: int, - session: Optional[requests.Session] = None, + session: Union[requests.Session, CatmaidInstance, None] = None, ) -> None: if mirror_info.tile_source_type != self.tile_source_type: raise ValueError("Mismatched tile source type") @@ -87,11 +94,12 @@ def __init__( self.mirror_info = mirror_info self.zoom_level = zoom_level - if session is None: - cm = utils._eval_remote_instance(None) - self.session = cm._session - else: + if isinstance(session, CatmaidInstance): + self.session = session._session + elif isinstance(session, requests.Session): self.session = session + elif session is None: + session = requests.Session() brok_sl = {int(k): int(k) + v for k, v in self.stack_info.broken_slices.items()} self.broken_slices = dict() @@ -199,22 +207,26 @@ def to_xarray(self) -> xr.DataArray: ) -class TileStore1(JpegStore): +class TileStore1(ImageIoStore): + """File-based image stack.""" tile_source_type = 1 fmt = "{image_base}{slice_idx}/{row}_{col}_{zoom_level}.{file_extension}" -class TileStore4(JpegStore): +class TileStore4(ImageIoStore): + """File-based image stack with zoom level directories.""" tile_source_type = 4 fmt = "{image_base}{slice_idx}/{zoom_level}/{row}_{col}.{file_extension}" -class TileStore5(JpegStore): +class TileStore5(ImageIoStore): + """Directory-based image stack with zoom, z, and row directories.""" tile_source_type = 5 fmt = "{image_base}{zoom_level}/{slice_idx}/{row}/{col}.{file_extension}" -# class TileStore10(JpegStore): +# class TileStore10(ImageIoStore): +# """H2N5 tile stack.""" # tile_source_type = 10 # fmt = "{image_base}.{file_extension}" @@ -232,7 +244,7 @@ class TileStore5(JpegStore): # return s -tile_stores: Dict[int, Type[JpegStore]] = { +tile_stores: Dict[int, Type[ImageIoStore]] = { t.tile_source_type: t for t in [ TileStore1, @@ -259,7 +271,11 @@ class Stack: allow access to individual scale levels as arrays which can be queried in voxel or world coordinates. """ - def __init__(self, stack_info: StackInfo, mirror: Optional[Union[int, str]] = None): + def __init__( + self, + stack_info: StackInfo, + mirror: Optional[Union[int, str]] = None, + ): """The :func:`Stack.from_catmaid` constructor may be more convenient. Parameters @@ -269,10 +285,39 @@ def __init__(self, stack_info: StackInfo, mirror: Optional[Union[int, str]] = No """ self.stack_info = stack_info self.mirror_info: Optional[MirrorInfo] = None + self.mirror_session_factory: Dict[int, Callable[[], Any]] = dict() if mirror is not None: self.set_mirror(mirror) + def set_mirror_session_factory( + self, mirror: Union[int, str, None], factory: Callable[[], Any] + ): + """Set functions which construct the session for fetching image data, per mirror. + + For most tile stacks, this is a + `requests.Session `_. + See ``get_remote_instance_session`` to use the session from a given + ``CatmaidInstance`` (including the global). + + For N5 (tile source 11), this is a + `aiohttp.ClientSession `_. + + Parameters + ---------- + mirror : Union[int, str, None] + Mirror, as integer ID, string name, or None to use the one defined on the class. + factory : Callable[[], Any] + Function which creates a session of the appropriate type. + For example, to re-use the ``requests.Session`` from the + global ``CatmaidInstance`` for mirror with ID 1, use + ``my_stack.set_mirror_instance_factory(1, get_remote_instance_session)``. + To use HTTP basic auth for an N5 stack mirror (tile source 11) with ID 2, use + ``my_stack.set_mirror_instance_factor(2, lambda: aiohttp.ClientSession(auth=aiohttp.BasicAuth("myusername", "mypassword")))``. + """ + minfo = self._get_mirror_info(mirror) + self.mirror_session_factory[minfo.id] = factory + @classmethod def select_from_catmaid(cls, remote_instance=None): """Interactively select a stack and mirror from those available. @@ -340,6 +385,15 @@ def select_mirror(self): if result is not None: self.set_mirror(result) + def _get_session_factory(self, mirror_id: int, default: Optional[Callable[[], Any]]=None): + try: + return self.mirror_session_factory[mirror_id] + except KeyError: + if default is None: + raise + else: + return default + def get_scale( self, scale_level: int, mirror_id: Optional[int] = None ) -> xr.DataArray: @@ -380,40 +434,67 @@ def get_scale( if mirror_info.tile_source_type in tile_stores: store_class = tile_stores[mirror_info.tile_source_type] - store = store_class(self.stack_info, mirror_info, scale_level, None) - return store.to_xarray() - elif mirror_info.tile_source_type == 11: - # do we need to handle broken slices here? - # or is that metadata just telling the frontend to skip regions which exist - # (as fill values) in the N5? - formatted = mirror_info.image_base.replace( - "%SCALE_DATASET%", f"s{scale_level}" + fac = self._get_session_factory( + mirror_info.id, + requests.Session, ) - *components, transpose_str = formatted.split("/") - transpose = [int(t) for t in transpose_str.split("_")] - - container_comp = [] - arr_comp = [] - this = container_comp - for comp in components: - this.append(comp) - if comp.lower().endswith(".n5"): - this = arr_comp - - if not arr_comp: - raise ValueError("N5 container must have '.n5' suffix") - - store = zarr.N5FSStore("/".join(container_comp)) - container = zarr.open(store, "r") - as_zarr = container["/".join(arr_comp)] - # todo: check this transpose - as_dask = da.from_zarr(as_zarr).transpose(transpose) - return xr.DataArray( - as_dask, - coords=self.stack_info.get_coords(scale_level), - dims=self.stack_info.orientation.full_orientation(True), + store = store_class( + self.stack_info, mirror_info, scale_level, fac() ) + return store.to_xarray() + elif mirror_info.tile_source_type == 11: + return self._get_n5(mirror_info, scale_level) raise NotImplementedError( f"Tile source type {mirror_info.tile_source_type} not implemented" ) + + def _get_n5( + self, + mirror_info: MirrorInfo, + scale_level: int, + ) -> xr.DataArray: + if mirror_info.tile_source_type != 11: + raise ValueError("Mirror info not from an N5 tile source") + formatted = mirror_info.image_base.replace( + "%SCALE_DATASET%", f"s{scale_level}" + ) + *components, transpose_str = formatted.split("/") + transpose = [int(t) for t in transpose_str.split("_")] + + container_comp = [] + arr_comp = [] + this = container_comp + for comp in components: + this.append(comp) + if comp.lower().endswith(".n5"): + this = arr_comp + + if not arr_comp: + raise ValueError("N5 container must have '.n5' suffix") + + kwargs = dict() + fac = self._get_session_factory(mirror_info.id, None) + if fac is not None: + kwargs["get_client"] = fac + + store = zarr.N5FSStore("/".join(container_comp), **kwargs) + + container = zarr.open(store, "r") + as_zarr = container["/".join(arr_comp)] + # todo: check this transpose + as_dask = da.from_zarr(as_zarr).transpose(transpose) + return xr.DataArray( + as_dask, + coords=self.stack_info.get_coords(scale_level), + dims=self.stack_info.orientation.full_orientation(True), + ) + + +def get_remote_instance_session(remote_instance: Optional[CatmaidInstance] = None): + """Get the ``requests.Session`` from the given ``CatmaidInstance``. + + If ``None`` is given, use the global ``CatmaidInstance``. + """ + cm = utils._eval_remote_instance(remote_instance) + return cm._session From 67bd05d4a3e3f29d2b77b474ca42e9b3e14c05be Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Thu, 9 Nov 2023 12:15:05 +0000 Subject: [PATCH 13/20] stack: enable users to create their own session This is unwieldy but necessary when a stack is not hosted in the same place as a CATMAID instance (not uncommon), or when an N5 stack is used. --- pymaid/stack.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pymaid/stack.py b/pymaid/stack.py index ea2f620..d78d387 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -270,6 +270,12 @@ class Stack: This class can, for certain stack mirror types, allow access to individual scale levels as arrays which can be queried in voxel or world coordinates. + + HTTP requests to fetch stack data are often configured + differently for different stack mirrors and tile source types. + For most non-public mirrors, you will need to define a function + which creats an object to make these requests: + see the ``my_stack.set_mirror_session_factory()`` method. """ def __init__( self, @@ -434,12 +440,12 @@ def get_scale( if mirror_info.tile_source_type in tile_stores: store_class = tile_stores[mirror_info.tile_source_type] - fac = self._get_session_factory( + session = self._get_session_factory( mirror_info.id, requests.Session, - ) + )() store = store_class( - self.stack_info, mirror_info, scale_level, fac() + self.stack_info, mirror_info, scale_level, session ) return store.to_xarray() elif mirror_info.tile_source_type == 11: From 19b0af989967d3c6949625edce08c7a6e6cd2c25 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Tue, 14 Nov 2023 09:28:53 +0000 Subject: [PATCH 14/20] stack: update source wrangling --- pymaid/stack.py | 72 +++++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/pymaid/stack.py b/pymaid/stack.py index d78d387..d1384ad 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -11,13 +11,8 @@ import numpy as np from numpy.typing import DTypeLike, ArrayLike -import zarr -from dask import array as da -import xarray as xr -from zarr.storage import BaseStore import json import requests -import imageio.v3 as iio from pymaid.client import CatmaidInstance @@ -30,6 +25,20 @@ get_mirror_info, ) +try: + import aiohttp + from dask import array as da + import imageio.v3 as iio + import xarray as xr + import zarr + from zarr.storage import BaseStore +except ImportError as e: + raise ImportError( + 'Optional dependencies for stack viewing are not available. ' + 'Make sure the appropriate extra is installed: `pip install navis[stacks]`. ' + f'Original error: "{str(e)}"' + ) + Dimension = Literal["x", "y", "z"] # Orientation = Literal["xy", "xz", "zy"] HALF_PX = 0.5 @@ -230,11 +239,10 @@ class TileStore5(ImageIoStore): # tile_source_type = 10 # fmt = "{image_base}.{file_extension}" -# # todo: manually change quality? - # def _format_url(self, row: int, col: int, slice_idx: int) -> str: # s = self.fmt.format( # image_base=self.mirror_info.image_base, +# # todo: manually change quality? # file_extension=self.mirror_info.file_extension, # ) # s = s.replace("%SCALE_DATASET%", f"s{self.zoom_level}") @@ -253,7 +261,8 @@ class TileStore5(ImageIoStore): # TileStore10 ] } -supported_sources = {11}.union(tile_stores) +source_client_types = {k: (requests.Session,) for k in tile_stores} +source_client_types[11] = (aiohttp.ClientSession,) def select_stack(remote_instance=None) -> Optional[int]: @@ -275,7 +284,7 @@ class Stack: differently for different stack mirrors and tile source types. For most non-public mirrors, you will need to define a function which creats an object to make these requests: - see the ``my_stack.set_mirror_session_factory()`` method. + see the ``my_stack.set_mirror_session()`` method. """ def __init__( self, @@ -291,13 +300,13 @@ def __init__( """ self.stack_info = stack_info self.mirror_info: Optional[MirrorInfo] = None - self.mirror_session_factory: Dict[int, Callable[[], Any]] = dict() + self.mirror_session: Dict[int, Any] = dict() if mirror is not None: self.set_mirror(mirror) - def set_mirror_session_factory( - self, mirror: Union[int, str, None], factory: Callable[[], Any] + def set_mirror_session( + self, mirror: Union[int, str, None], session, ): """Set functions which construct the session for fetching image data, per mirror. @@ -313,16 +322,16 @@ def set_mirror_session_factory( ---------- mirror : Union[int, str, None] Mirror, as integer ID, string name, or None to use the one defined on the class. - factory : Callable[[], Any] - Function which creates a session of the appropriate type. + session : Callable[[], Any] + HTTP session of the appropriate type. For example, to re-use the ``requests.Session`` from the global ``CatmaidInstance`` for mirror with ID 1, use - ``my_stack.set_mirror_instance_factory(1, get_remote_instance_session)``. + ``my_stack.set_mirror_instance(1, get_remote_instance_session())``. To use HTTP basic auth for an N5 stack mirror (tile source 11) with ID 2, use - ``my_stack.set_mirror_instance_factor(2, lambda: aiohttp.ClientSession(auth=aiohttp.BasicAuth("myusername", "mypassword")))``. + ``my_stack.set_mirror_instance_factor(2, aiohttp.ClientSession(auth=aiohttp.BasicAuth("myusername", "mypassword")))``. """ minfo = self._get_mirror_info(mirror) - self.mirror_session_factory[minfo.id] = factory + self.mirror_session[minfo.id] = session @classmethod def select_from_catmaid(cls, remote_instance=None): @@ -378,7 +387,7 @@ def select_mirror(self): options = { m.id: m.title for m in self.stack_info.mirrors - if m.tile_source_type in supported_sources + if m.tile_source_type in source_client_types } if not options: print("No mirrors with supported tile source type") @@ -391,9 +400,9 @@ def select_mirror(self): if result is not None: self.set_mirror(result) - def _get_session_factory(self, mirror_id: int, default: Optional[Callable[[], Any]]=None): + def _get_session(self, mirror_id: int, default: Optional[Any]=None): try: - return self.mirror_session_factory[mirror_id] + return self.mirror_session[mirror_id] except KeyError: if default is None: raise @@ -440,10 +449,11 @@ def get_scale( if mirror_info.tile_source_type in tile_stores: store_class = tile_stores[mirror_info.tile_source_type] - session = self._get_session_factory( + session = self._get_session( mirror_info.id, - requests.Session, - )() + requests.Session(), + ) + check_session_type(session, mirror_info.tile_source_type) store = store_class( self.stack_info, mirror_info, scale_level, session ) @@ -480,9 +490,10 @@ def _get_n5( raise ValueError("N5 container must have '.n5' suffix") kwargs = dict() - fac = self._get_session_factory(mirror_info.id, None) - if fac is not None: - kwargs["get_client"] = fac + session = self._get_session(mirror_info.id, None) + if session is not None: + check_session_type(session, 11) + kwargs["get_client"] = lambda: session store = zarr.N5FSStore("/".join(container_comp), **kwargs) @@ -497,6 +508,15 @@ def _get_n5( ) +def check_session_type(session, tile_source_type: int): + expected = source_client_types[tile_source_type] + if not isinstance(session, expected): + raise ValueError( + f"Incorrect HTTP client type for tile source {tile_source_type}. " + f"Got {type(session)} but expected one of {expected}." + ) + + def get_remote_instance_session(remote_instance: Optional[CatmaidInstance] = None): """Get the ``requests.Session`` from the given ``CatmaidInstance``. From 8853f5b1c40ef905f90e181c4c92fc4227a100ad Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Tue, 14 Nov 2023 09:44:36 +0000 Subject: [PATCH 15/20] stack: improve docs --- pymaid/stack.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pymaid/stack.py b/pymaid/stack.py index d1384ad..9101170 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -1,11 +1,11 @@ -"""Access to image data as xarray.DataArrays. +"""Access to image data as ``xarray.DataArray``s. CATMAID's image source conventions are documented here https://catmaid.readthedocs.io/en/stable/tile_sources.html """ from __future__ import annotations from io import BytesIO -from typing import Any, Callable, Literal, Optional, Sequence, Type, Union, Dict +from typing import Any, Literal, Optional, Sequence, Type, Union, Dict from abc import ABC import sys @@ -282,9 +282,14 @@ class Stack: HTTP requests to fetch stack data are often configured differently for different stack mirrors and tile source types. - For most non-public mirrors, you will need to define a function - which creats an object to make these requests: + For most non-public mirrors, you will need to set the object to make these requests: see the ``my_stack.set_mirror_session()`` method. + + See the ``my_stack.get_scale()`` method for getting an + `xarray.DataArray `_ + representing that scale level. + This can be queried in stack/ voxel or project/ world coordinates, + efficiently sliced and transposed etc.. """ def __init__( self, @@ -412,7 +417,7 @@ def _get_session(self, mirror_id: int, default: Optional[Any]=None): def get_scale( self, scale_level: int, mirror_id: Optional[int] = None ) -> xr.DataArray: - """Get an xarray.DataArray representing th given scale level. + """Get an xarray.DataArray representing the given scale level. Note that depending on the metadata available, missing scale levels may throw different errors. From 103e01a966866fd7c79508028e4e170ba988b735 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Wed, 3 Jan 2024 16:57:12 +0000 Subject: [PATCH 16/20] minor: stacks->stack extra, fix docstring --- pymaid/stack.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymaid/stack.py b/pymaid/stack.py index 9101170..e86af84 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -35,7 +35,7 @@ except ImportError as e: raise ImportError( 'Optional dependencies for stack viewing are not available. ' - 'Make sure the appropriate extra is installed: `pip install navis[stacks]`. ' + 'Make sure the appropriate extra is installed: `pip install pymaid[stack]`. ' f'Original error: "{str(e)}"' ) diff --git a/requirements.txt b/requirements.txt index 21021c9..c60ceb6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ psutil>=5.4.3 fuzzywuzzy[speedup]~=0.17.0 ujson~=1.35 -#extra: stacks +#extra: stack zarr fsspec[http] xarray[parallel] From 3a927848539a9a05168585584af1d3cdc3dfea74 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Wed, 3 Jan 2024 17:03:59 +0000 Subject: [PATCH 17/20] Improve docs for mirror session --- pymaid/stack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pymaid/stack.py b/pymaid/stack.py index e86af84..3b6e3d5 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -264,6 +264,8 @@ class TileStore5(ImageIoStore): source_client_types = {k: (requests.Session,) for k in tile_stores} source_client_types[11] = (aiohttp.ClientSession,) +Client = Union[requests.Session, aiohttp.ClientSession] + def select_stack(remote_instance=None) -> Optional[int]: """""" @@ -311,7 +313,7 @@ def __init__( self.set_mirror(mirror) def set_mirror_session( - self, mirror: Union[int, str, None], session, + self, mirror: Union[int, str, None], session: Client, ): """Set functions which construct the session for fetching image data, per mirror. @@ -327,7 +329,7 @@ def set_mirror_session( ---------- mirror : Union[int, str, None] Mirror, as integer ID, string name, or None to use the one defined on the class. - session : Callable[[], Any] + session : Union[requests.Session, aiohttp.ClientSession] HTTP session of the appropriate type. For example, to re-use the ``requests.Session`` from the global ``CatmaidInstance`` for mirror with ID 1, use From 3cb3238d25f1c2958a35e93afda5ef4f4a1a0f2a Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Wed, 3 Jan 2024 17:08:44 +0000 Subject: [PATCH 18/20] Print to stderr --- pymaid/stack.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pymaid/stack.py b/pymaid/stack.py index 3b6e3d5..1425b29 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -4,6 +4,7 @@ https://catmaid.readthedocs.io/en/stable/tile_sources.html """ from __future__ import annotations +from functools import wraps from io import BytesIO from typing import Any, Literal, Optional, Sequence, Type, Union, Dict from abc import ABC @@ -45,11 +46,18 @@ ENDIAN = "<" if sys.byteorder == "little" else ">" +@wraps(print) +def eprint(*args, **kwargs): + """Thin wrapper around ``print`` which defaults to stderr""" + kwargs.setdefault("file", sys.stderr) + return print(*args, **kwargs) + + def select_cli(prompt: str, options: Dict[int, str]) -> Optional[int]: out = None - print(prompt) + eprint(prompt) for k, v in sorted(options.items()): - print(f"\t{k}.\t{v}") + eprint(f"\t{k}.\t{v}") p = "Type number and press enter (empty to cancel): " while out is None: result_str = input(p).strip() @@ -58,10 +66,10 @@ def select_cli(prompt: str, options: Dict[int, str]) -> Optional[int]: try: result = int(result_str) except ValueError: - print("Not an integer, try again") + eprint("Not an integer, try again") continue if result not in options: - print("Not a valid option, try again") + eprint("Not a valid option, try again") continue out = result return out @@ -397,7 +405,7 @@ def select_mirror(self): if m.tile_source_type in source_client_types } if not options: - print("No mirrors with supported tile source type") + eprint("No mirrors with supported tile source type") return result = select_cli( From ff0a5cf3f40e8696a33ed2e9756ec147d96f3f2e Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Wed, 3 Jan 2024 17:14:43 +0000 Subject: [PATCH 19/20] Remove unused dependency, add [all] extra --- requirements.txt | 1 - setup.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c60ceb6..fa52bee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,6 @@ tqdm>=4.50.0 psutil>=5.4.3 #extra: extras -fuzzywuzzy[speedup]~=0.17.0 ujson~=1.35 #extra: stack diff --git a/setup.py b/setup.py index e1f4ef8..083fc11 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +import itertools from setuptools import setup, find_packages import re from pathlib import Path @@ -15,6 +16,7 @@ raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,)) install_requires, extras_require = parse_requirement_files(Path("requirements.txt")) +extras_require["all"] = list(set(itertools.chain.from_iterable(extras_require.values()))) setup( name='python-catmaid', From 7b402bc632c4f14047a7416fd5c387b7732f52f2 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Mon, 15 Jan 2024 12:30:04 +0000 Subject: [PATCH 20/20] stack: add auth convenience method --- pymaid/stack.py | 46 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/pymaid/stack.py b/pymaid/stack.py index 1425b29..0c2b267 100644 --- a/pymaid/stack.py +++ b/pymaid/stack.py @@ -293,7 +293,9 @@ class Stack: HTTP requests to fetch stack data are often configured differently for different stack mirrors and tile source types. For most non-public mirrors, you will need to set the object to make these requests: - see the ``my_stack.set_mirror_session()`` method. + see the ``my_stack.set_mirror_session()`` method; + if you just need to set HTTP Basic authentication headers, + see the ``my_stack.set_mirror_auth()`` convenience method. See the ``my_stack.get_scale()`` method for getting an `xarray.DataArray `_ @@ -320,8 +322,42 @@ def __init__( if mirror is not None: self.set_mirror(mirror) + def set_mirror_auth(self, mirror: Union[int, str, None, MirrorInfo], http_user: str, http_password: str): + """Set the HTTP Basic credentials for a particular stack mirror. + + This will replace any other session configured for that mirror. + + For more fine-grained control (e.g. setting other headers), + or to re-use the session object from a ``CatmaidInstance``, + see ``my_stack.set_mirror_session()``. + + Parameters + ---------- + mirror : Union[int, str, None, MirrorInfo] + Mirror, as MirrorInfo, intger ID, string title, or None (use default) + http_user : str + HTTP Basic username + http_password : str + HTTP Basic password + + Raises + ------ + ValueError + If the given mirror is not supported. + """ + minfo = self._get_mirror_info(mirror) + if minfo.tile_source_type == 11: + s = aiohttp.ClientSession(auth=aiohttp.BasicAuth(http_user, http_password)) + return self.set_mirror_session(mirror, s) + elif minfo.tile_source_type in source_client_types: + s = requests.Session() + s.auth = (http_user, http_password) + return self.set_mirror_session(mirror, s) + else: + raise ValueError("Mirror's tile source type is unsupported: %s", minfo.tile_source_type) + def set_mirror_session( - self, mirror: Union[int, str, None], session: Client, + self, mirror: Union[int, str, None, MirrorInfo], session: Client, ): """Set functions which construct the session for fetching image data, per mirror. @@ -345,6 +381,7 @@ def set_mirror_session( To use HTTP basic auth for an N5 stack mirror (tile source 11) with ID 2, use ``my_stack.set_mirror_instance_factor(2, aiohttp.ClientSession(auth=aiohttp.BasicAuth("myusername", "mypassword")))``. """ + minfo = self._get_mirror_info(mirror) self.mirror_session[minfo.id] = session @@ -384,7 +421,10 @@ def from_catmaid( sinfo = get_stack_info(stack, remote_instance) return cls(sinfo, mirror) - def _get_mirror_info(self, mirror: Optional[Union[int, str]] = None) -> MirrorInfo: + def _get_mirror_info(self, mirror: Union[int, str, None, MirrorInfo] = None) -> MirrorInfo: + if isinstance(mirror, MirrorInfo): + return mirror + if mirror is None: if self.mirror_info is None: raise ValueError("No default mirror ID set")