diff --git a/REQUIREMENTS-STRICT.txt b/REQUIREMENTS-STRICT.txt index 7d87059d9..172880eab 100644 --- a/REQUIREMENTS-STRICT.txt +++ b/REQUIREMENTS-STRICT.txt @@ -61,7 +61,7 @@ semantic-version==2.6.0 Send2Trash==1.5.0 showit==1.1.4 six==1.12.0 -slicedimage==3.1.2 +slicedimage==4.0.0 sympy==1.4 terminado==0.8.2 testpath==0.4.2 diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index 2513372a6..8f25d1f6c 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -10,7 +10,7 @@ scikit-image>=0.14.0 scikit-learn scipy showit >= 1.1.4 -slicedimage==3.1.2 +slicedimage==4.0.0 scikit-learn sympy tqdm diff --git a/starfish/REQUIREMENTS-STRICT.txt b/starfish/REQUIREMENTS-STRICT.txt index 7d87059d9..172880eab 100644 --- a/starfish/REQUIREMENTS-STRICT.txt +++ b/starfish/REQUIREMENTS-STRICT.txt @@ -61,7 +61,7 @@ semantic-version==2.6.0 Send2Trash==1.5.0 showit==1.1.4 six==1.12.0 -slicedimage==3.1.2 +slicedimage==4.0.0 sympy==1.4 terminado==0.8.2 testpath==0.4.2 diff --git a/starfish/core/experiment/builder/__init__.py b/starfish/core/experiment/builder/__init__.py index 49cf96200..738714d1b 100644 --- a/starfish/core/experiment/builder/__init__.py +++ b/starfish/core/experiment/builder/__init__.py @@ -1,6 +1,7 @@ import functools import json import os +import warnings from dataclasses import astuple, dataclass from pathlib import Path from typing import ( @@ -24,6 +25,7 @@ Tile, TileSet, Writer, + WriterContract, ) from starfish import FieldOfView @@ -48,18 +50,6 @@ class TileIdentifier: zplane_label: int -def _tile_opener(toc_path: Path, tile: Tile, file_ext: str) -> BinaryIO: - base = toc_path.parent / toc_path.stem - return open( - f"{os.fspath(base)}-Z{tile.indices[Axes.ZPLANE]}-" - f"R{tile.indices[Axes.ROUND]}-C{tile.indices[Axes.CH]}.{file_ext}", - "wb") - - -def _fov_path_generator(parent_toc_path: Path, toc_name: str) -> Path: - return parent_toc_path.parent / "{}-{}.json".format(parent_toc_path.stem, toc_name) - - def build_irregular_image( tile_identifiers: Iterable[TileIdentifier], image_fetcher: TileFetcher, @@ -212,6 +202,7 @@ def write_irregular_experiment_json( default_shape: Optional[Mapping[Axes, int]]=None, fov_path_generator: Callable[[Path, str], Path] = None, tile_opener: Optional[Callable[[Path, Tile, str], BinaryIO]] = None, + writer_contract: Optional[WriterContract] = None, ) -> None: """ Build and returns a top-level experiment description with the following characteristics: @@ -235,19 +226,28 @@ def write_irregular_experiment_json( fov_path_generator : Optional[Callable[[Path, str], Path]] Generates the path for a FOV's json file. If one is not provided, the default generates the FOV's json file at the same level as the top-level json file for an image. If this is - not provided, a reasonable default will be provided. + not provided, a reasonable default will be provided. If this is provided, writer_contract + should not be provided. tile_opener : Optional[Callable[[Path, Tile, str], BinaryIO]] Callable that gets invoked with the following arguments: 1. the directory of the experiment that is being constructed, 2. the tile that is being written, and 3. the file extension that the tile should be written with. The callable is expected to return an open file - handle. If this is not provided, a reasonable default will be provided. + handle. If this is not provided, a reasonable default will be provided. If this is + provided, writer_contract should not be provided. + writer_contract : Optional[WriterContract] + Contract for specifying how the slicedimage image is to be laid out. If this is provided, + fov_path_generator and tile_opener should not be provided. """ if postprocess_func is None: postprocess_func = lambda doc: doc - if fov_path_generator is None: - fov_path_generator = _fov_path_generator - if tile_opener is None: - tile_opener = _tile_opener + if fov_path_generator is not None or tile_opener is not None: + warnings.warn( + "`fov_path_generator` and `tile_opener` options for writing experiment files is " + "deprecated. Use `writer_contract` instead.", + DeprecationWarning) + if writer_contract is not None: + raise ValueError( + "Cannot specify both `writer_contract` and `fov_path_generator` or `tile_opener`") experiment_doc: Dict[str, Any] = { 'version': str(CURRENT_VERSION), @@ -261,10 +261,11 @@ def write_irregular_experiment_json( Writer.write_to_path( image, - os.path.join(path, f"{image_type}.json"), + Path(path) / f"{image_type}.json", pretty=True, partition_path_generator=fov_path_generator, tile_opener=tile_opener, + writer_contract=writer_contract, tile_format=tile_format, ) experiment_doc['images'][image_type] = f"{image_type}.json" @@ -302,6 +303,7 @@ def write_experiment_json( dimension_order: Sequence[Axes]=(Axes.ZPLANE, Axes.ROUND, Axes.CH), fov_path_generator: Optional[Callable[[Path, str], Path]] = None, tile_opener: Optional[Callable[[Path, Tile, str], BinaryIO]] = None, + writer_contract: Optional[WriterContract] = None, ) -> None: """ Build and returns a top-level experiment description with the following characteristics: @@ -346,15 +348,20 @@ def write_experiment_json( (ROUND=1, CH=0, Z=1) (ROUND=1, CH=1, Z=1) (default = (Axes.Z, Axes.ROUND, Axes.CH)) - fov_path_generator : Callable[[Path, str], Path] + fov_path_generator : Optional[Callable[[Path, str], Path]] Generates the path for a FOV's json file. If one is not provided, the default generates - the FOV's json file at the same level as the top-level json file for an image. If this is - not provided, a reasonable default will be provided. - tile_opener : Callable[[Path, Tile, str], BinaryIO] + the FOV's json file at the same level as the top-level json file for an image. If this is + not provided, a reasonable default will be provided. If this is provided, writer_contract + should not be provided. + tile_opener : Optional[Callable[[Path, Tile, str], BinaryIO]] Callable that gets invoked with the following arguments: 1. the directory of the experiment that is being constructed, 2. the tile that is being written, and 3. the file extension that the tile should be written with. The callable is expected to return an open file - handle. If this is not provided, a reasonable default will be provided. + handle. If this is not provided, a reasonable default will be provided. If this is + provided, writer_contract should not be provided. + writer_contract : Optional[WriterContract] + Contract for specifying how the slicedimage image is to be laid out. If this is provided, + fov_path_generator and tile_opener should not be provided. """ all_tile_fetcher: MutableMapping[str, TileFetcher] = {} if aux_tile_fetcher is not None: @@ -393,4 +400,5 @@ def write_experiment_json( default_shape=default_shape, fov_path_generator=fov_path_generator, tile_opener=tile_opener, + writer_contract=writer_contract, ) diff --git a/starfish/core/experiment/builder/inplace.py b/starfish/core/experiment/builder/inplace.py index 17a71e961..991c980e5 100644 --- a/starfish/core/experiment/builder/inplace.py +++ b/starfish/core/experiment/builder/inplace.py @@ -1,64 +1,39 @@ """To build experiments in-place, a few things must be done: -1. Call enable_inplace_mode(). -2. Call write_experiment_json with tile_opener=inplace_tile_opener. -3. The TileFetcher should return an instance of a InplaceTileFetcher. - -Please note that enabling in-place experiment construction should be only done in an isolated script -dedicated to constructing an experiment, as it modifies some existing code paths. +1. Call write_experiment_json with writer_contract=InplaceWriterContract(). +2. The TileFetcher should return an instance of a InplaceTileFetcher. """ import abc -import io -import sys +import warnings from pathlib import Path -from typing import BinaryIO +from typing import Mapping, Optional import numpy as np -from slicedimage import Tile +from slicedimage import ImageFormat, Tile +from slicedimage.io import WriterContract from starfish.core.types import Axes from .providers import FetchedTile -def sha256_get(tile_self): - return tile_self.provider.sha256 - +class InplaceWriterContract(WriterContract): + def tile_url_generator(self, tileset_url: str, tile: Tile, ext: str) -> str: + return f"file://{tile.provider.filepath}" -def sha256_set(tile_self, value): - pass + def write_tile( + self, + tile_url: str, + tile: Tile, + tile_format: ImageFormat, + backend_config: Optional[Mapping] = None, + ) -> str: + return tile.provider.sha256 def enable_inplace_mode(): - Tile.sha256 = property(sha256_get, sha256_set) - - -def inplace_tile_opener(toc_path: Path, tile: Tile, file_ext: str) -> BinaryIO: - return DevNull(tile.provider.filepath) - - -class DevNull(io.BytesIO): - """A class meant to mimic an open(filepath, 'wb') operation but - prevents any actual writing, reading, or seeking. - This class is an ugly hack to prevent the slicedimage - Writer.generate_partition_document() function from needlessly creating - a copy of image data. - See: https://docs.python.org/3/library/io.html - See also: cpython/Lib/_pyio.py - """ - def __init__(self, filepath: str, *args, **kwargs): - super(DevNull, self).__init__(*args, **kwargs) - self.name = filepath - - def read(self, size=-1): - raise NotImplementedError() - - def write(self, b): - return sys.getsizeof(b) - - def seek(self, pos, whence=0): - raise NotImplementedError() + warnings.warn("`enable_inplace_mode()` is no longer necessary.", DeprecationWarning) class InplaceFetchedTile(FetchedTile): diff --git a/starfish/core/experiment/builder/structured_formatter.py b/starfish/core/experiment/builder/structured_formatter.py index 0f8c4c04f..bcd7fc9c1 100644 --- a/starfish/core/experiment/builder/structured_formatter.py +++ b/starfish/core/experiment/builder/structured_formatter.py @@ -6,8 +6,6 @@ from dataclasses import dataclass from pathlib import Path from typing import ( - BinaryIO, - Callable, cast, FrozenSet, Mapping, @@ -20,11 +18,11 @@ from warnings import warn import numpy as np -from slicedimage import ImageFormat, Tile +from slicedimage import ImageFormat, WriterContract from starfish.core.types import Axes, Coordinates, CoordinateValue, Number from . import TileIdentifier, write_irregular_experiment_json -from .inplace import enable_inplace_mode, inplace_tile_opener, InplaceFetchedTile +from .inplace import InplaceFetchedTile, InplaceWriterContract from .providers import FetchedTile, TileFetcher FILENAME_CRE = re.compile( @@ -130,17 +128,16 @@ def format_structured_dataset( } if in_place: - enable_inplace_mode() - tile_opener: Optional[Callable[[Path, Tile, str], BinaryIO]] = inplace_tile_opener + writer_contract: Optional[WriterContract] = InplaceWriterContract() else: - tile_opener = None + writer_contract = None write_irregular_experiment_json( output_experiment_dir_str, tile_format, image_tile_identifiers=image_tile_identifiers, tile_fetchers=tile_fetchers, - tile_opener=tile_opener, + writer_contract=writer_contract, ) diff --git a/starfish/core/experiment/builder/test/inplace_script.py b/starfish/core/experiment/builder/test/inplace_script.py index a5acec500..0a352238c 100644 --- a/starfish/core/experiment/builder/test/inplace_script.py +++ b/starfish/core/experiment/builder/test/inplace_script.py @@ -10,7 +10,7 @@ from starfish.core.experiment.builder import FetchedTile, TileFetcher, write_experiment_json from starfish.core.experiment.builder.inplace import ( - enable_inplace_mode, inplace_tile_opener, InplaceFetchedTile + InplaceFetchedTile, InplaceWriterContract, ) from starfish.core.experiment.experiment import Experiment, FieldOfView from starfish.core.types import Axes, Coordinates, CoordinateValue @@ -80,8 +80,6 @@ def add_codebook(experiment_json_doc): experiment_json_doc['codebook'] = "codebook.json" return experiment_json_doc - enable_inplace_mode() - write_experiment_json( path=os.fspath(image_dir), fov_count=num_fovs, @@ -95,8 +93,7 @@ def add_codebook(experiment_json_doc): }, postprocess_func=add_codebook, default_shape=SHAPE, - fov_path_generator=fov_path_generator, - tile_opener=inplace_tile_opener, + writer_contract=InplaceWriterContract(), )