From c844ae9ab16ec3dc4d3396b727e4bc7db3edcd36 Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Sat, 26 Aug 2023 16:08:31 -0400 Subject: [PATCH 1/9] feat: multiscale ome-zarr reading compatability --- funlib/persistence/arrays/datasets.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/funlib/persistence/arrays/datasets.py b/funlib/persistence/arrays/datasets.py index 7d4f4db..88c0e6c 100644 --- a/funlib/persistence/arrays/datasets.py +++ b/funlib/persistence/arrays/datasets.py @@ -14,10 +14,19 @@ logger = logging.getLogger(__name__) +def access_parent(node): + """ + Get the parent (zarr.Group) of a zarr array or group. + """ + parent_path = "/".join(node.path.split("/")[:-1]) + return zarr.open(node.store.path, mode="r")[parent_path] + + def _read_voxel_size_offset(ds, order="C"): voxel_size = None offset = None dims = None + group = access_parent(ds) if "resolution" in ds.attrs: voxel_size = Coordinate(ds.attrs["resolution"]) @@ -28,7 +37,6 @@ def _read_voxel_size_offset(ds, order="C"): elif "pixelResolution" in ds.attrs: voxel_size = Coordinate(ds.attrs["pixelResolution"]["dimensions"]) dims = len(voxel_size) - elif "transform" in ds.attrs: # Davis saves transforms in C order regardless of underlying # memory format (i.e. n5 or zarr). May be explicitly provided @@ -38,6 +46,23 @@ def _read_voxel_size_offset(ds, order="C"): if transform_order != order: voxel_size = Coordinate(voxel_size[::-1]) dims = len(voxel_size) + elif (multiscales := group.attrs.get("multiscales", None)) is not None: + logger.info("Found multiscales attributes") + if len(multiscales) == 0: + raise ValueError("Multiscales attribute was empty.") + scale = ds.path.split("/")[-1] + for level in multiscales[0][ + "datasets" + ]: # TODO: flimsy parsing may break with other formats + if level["path"] == scale: + logger.info("Found scale attribute") + for attr in level["coordinateTransformations"]: + if attr["type"] == "scale": + voxel_size = Coordinate(attr["scale"]) + elif attr["type"] == "translation": + offset = Coordinate(attr["translation"]) + break + dims = len(voxel_size) if "offset" in ds.attrs: offset = Coordinate(ds.attrs["offset"]) From 34b982837100b447ce8d14350725c030c8419ad0 Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Wed, 30 Aug 2023 11:09:55 -0400 Subject: [PATCH 2/9] feat: add windows os support --- funlib/persistence/arrays/datasets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/funlib/persistence/arrays/datasets.py b/funlib/persistence/arrays/datasets.py index 88c0e6c..fc7dd7a 100644 --- a/funlib/persistence/arrays/datasets.py +++ b/funlib/persistence/arrays/datasets.py @@ -10,6 +10,7 @@ import os import shutil from typing import Optional +from pathlib import Path logger = logging.getLogger(__name__) @@ -18,7 +19,7 @@ def access_parent(node): """ Get the parent (zarr.Group) of a zarr array or group. """ - parent_path = "/".join(node.path.split("/")[:-1]) + parent_path = Path(node.path).parent.absolute() return zarr.open(node.store.path, mode="r")[parent_path] From 982bc1a8036ed131f4ef030e1ad291b56a1c6b43 Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Fri, 1 Sep 2023 10:30:38 -0400 Subject: [PATCH 3/9] bugfix: access_parent for ngff multiscale datasets --- funlib/persistence/arrays/datasets.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/funlib/persistence/arrays/datasets.py b/funlib/persistence/arrays/datasets.py index fc7dd7a..f5fb54f 100644 --- a/funlib/persistence/arrays/datasets.py +++ b/funlib/persistence/arrays/datasets.py @@ -19,8 +19,7 @@ def access_parent(node): """ Get the parent (zarr.Group) of a zarr array or group. """ - parent_path = Path(node.path).parent.absolute() - return zarr.open(node.store.path, mode="r")[parent_path] + return zarr.open(node.store.path, mode="r")[Path(node.path).parent] def _read_voxel_size_offset(ds, order="C"): From ea97663c0bcb26a6e6bceb6646f3ee1e22451f92 Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Thu, 14 Sep 2023 16:03:43 -0400 Subject: [PATCH 4/9] bugfix: handle non-ngff multiscale --- .gitignore | 3 +++ funlib/persistence/arrays/datasets.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index b6e4761..fa1ab66 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ dmypy.json # Pyre type checker .pyre/ + +# VSCode project settings +.vscode/ \ No newline at end of file diff --git a/funlib/persistence/arrays/datasets.py b/funlib/persistence/arrays/datasets.py index f5fb54f..486da26 100644 --- a/funlib/persistence/arrays/datasets.py +++ b/funlib/persistence/arrays/datasets.py @@ -26,7 +26,8 @@ def _read_voxel_size_offset(ds, order="C"): voxel_size = None offset = None dims = None - group = access_parent(ds) + group = None + multiscales = None if "resolution" in ds.attrs: voxel_size = Coordinate(ds.attrs["resolution"]) @@ -46,7 +47,11 @@ def _read_voxel_size_offset(ds, order="C"): if transform_order != order: voxel_size = Coordinate(voxel_size[::-1]) dims = len(voxel_size) - elif (multiscales := group.attrs.get("multiscales", None)) is not None: + else: + group = access_parent(ds) + multiscales = group.attrs.get("multiscales", None) + + if multiscales is not None: logger.info("Found multiscales attributes") if len(multiscales) == 0: raise ValueError("Multiscales attribute was empty.") @@ -334,7 +339,7 @@ def prepare_ds( root = zarr.open(filename, mode="a") ds = root.create_dataset( - ds_name, shape=shape, chunks=chunk_shape, dtype=dtype, compressor=compressor + ds_name, shape=shape, chunks=chunk_shape, dtype=dtype, compressor=compressor, overwrite=delete ) if file_format == "zarr": From 973a8dea566029764346f0bb38df2d906c33c208 Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Thu, 4 Jan 2024 15:57:21 -0500 Subject: [PATCH 5/9] fixed multi-scale zarr without parent group --- funlib/persistence/arrays/datasets.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/funlib/persistence/arrays/datasets.py b/funlib/persistence/arrays/datasets.py index 486da26..cc05517 100644 --- a/funlib/persistence/arrays/datasets.py +++ b/funlib/persistence/arrays/datasets.py @@ -19,7 +19,7 @@ def access_parent(node): """ Get the parent (zarr.Group) of a zarr array or group. """ - return zarr.open(node.store.path, mode="r")[Path(node.path).parent] + return zarr.open(node.store.path, mode="r")[Path(node.path).parent.name] def _read_voxel_size_offset(ds, order="C"): @@ -147,7 +147,11 @@ def open_ds(filename: str, ds_name: str, mode: str = "r") -> Array: logger.error("failed to open %s/%s" % (filename, ds_name)) raise e - voxel_size, offset = _read_voxel_size_offset(ds, ds.order) + try: + order = ds.attrs["order"] + except KeyError: + order = ds.order + voxel_size, offset = _read_voxel_size_offset(ds, order) shape = Coordinate(ds.shape[-len(voxel_size) :]) roi = Roi(offset, voxel_size * shape) From 3adccdd2463724de41a6b50235dbb78e7880ce8b Mon Sep 17 00:00:00 2001 From: Jeff Rhoades <37990507+rhoadesScholar@users.noreply.github.com> Date: Tue, 16 Jan 2024 18:39:51 +0000 Subject: [PATCH 6/9] Black format. --- funlib/persistence/arrays/datasets.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/funlib/persistence/arrays/datasets.py b/funlib/persistence/arrays/datasets.py index fcce619..0edbfef 100644 --- a/funlib/persistence/arrays/datasets.py +++ b/funlib/persistence/arrays/datasets.py @@ -343,7 +343,12 @@ def prepare_ds( root = zarr.open(filename, mode="a") ds = root.create_dataset( - ds_name, shape=shape, chunks=chunk_shape, dtype=dtype, compressor=compressor, overwrite=delete + ds_name, + shape=shape, + chunks=chunk_shape, + dtype=dtype, + compressor=compressor, + overwrite=delete, ) if file_format == "zarr": From 12d1948f9b155abfe0469ca7936cde72d5077c79 Mon Sep 17 00:00:00 2001 From: Jeff Rhoades <37990507+rhoadesScholar@users.noreply.github.com> Date: Tue, 16 Jan 2024 18:53:17 +0000 Subject: [PATCH 7/9] =?UTF-8?q?fix:=20=F0=9F=90=9B=20Typing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- funlib/persistence/arrays/datasets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/funlib/persistence/arrays/datasets.py b/funlib/persistence/arrays/datasets.py index 0edbfef..e77683c 100644 --- a/funlib/persistence/arrays/datasets.py +++ b/funlib/persistence/arrays/datasets.py @@ -211,8 +211,8 @@ def prepare_ds( total_roi: Roi, voxel_size: Coordinate, dtype, - write_roi: Roi = None, - write_size: Coordinate = None, + write_roi: Optional[Roi] = None, + write_size: Optional[Coordinate] = None, num_channels: Optional[int] = None, compressor: Union[str, dict] = "default", delete: bool = False, @@ -314,7 +314,7 @@ def prepare_ds( else: chunk_shape = None - shape = total_roi.shape / voxel_size + shape = tuple(total_roi.shape / voxel_size) if num_channels is not None: shape = (num_channels,) + shape From 6d25e6e709f72db597e20dfb51bc581a36491f82 Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Tue, 16 Jan 2024 17:50:03 -0500 Subject: [PATCH 8/9] fx_bug: get parent path instead of folder name --- funlib/persistence/arrays/datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funlib/persistence/arrays/datasets.py b/funlib/persistence/arrays/datasets.py index e77683c..be15d8f 100644 --- a/funlib/persistence/arrays/datasets.py +++ b/funlib/persistence/arrays/datasets.py @@ -19,7 +19,7 @@ def access_parent(node): """ Get the parent (zarr.Group) of a zarr array or group. """ - return zarr.open(node.store.path, mode="r")[Path(node.path).parent.name] + return zarr.open(node.store.path, mode="r")[str(Path(node.path).parent)] def _read_voxel_size_offset(ds, order="C"): From 9701af162753037326d82d3ff493b55adf80db5d Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Sun, 21 Jan 2024 16:15:02 -0500 Subject: [PATCH 9/9] =?UTF-8?q?fix:=20=E2=9A=A1=EF=B8=8F=20Use=20PyPi=20pr?= =?UTF-8?q?ebuilt=20binary:=20psycopg2-binary=20instead=20of=20building=20?= =?UTF-8?q?from=20source.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8b540c1..04093bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "pymongo", "numpy", "h5py", - "psycopg2", + "psycopg2-binary", ] [tool.setuptools.dynamic]