From 4e9ccad16a1fc4a8834865341f458cefcb24d01e Mon Sep 17 00:00:00 2001 From: Tony Tung Date: Wed, 11 Dec 2019 10:36:53 -0800 Subject: [PATCH] Make map/reduce APIs more intuitive Right now, specifying a FunctionSource along with function parameters makes for a confusing function call. For instance, `Map('divide', 2, module=FunctionSource.np)` means the FunctionSource comes last, which is not intuitive. This adds the ability for `FunctionSource`s to be called and return a Bundle that includes both the package name and the function name. The call above would then become: `Map(FunctionSource.np('divide'), 2)`. Backwards compatibility with the prior API is maintained, but a warning is generated. If the top-level package is provided twice, it is treated as an error. Test plan: added test cases to cover the new approach, the old approach that should generate the warning, and the ugly combination that should fail. --- .../data_processing_examples/iss_pipeline.py | 2 +- .../tutorials/exec_image_manipulations.py | 2 +- notebooks/BaristaSeq.ipynb | 2 +- notebooks/ISS.ipynb | 2 +- notebooks/MERFISH.ipynb | 2 +- notebooks/STARmap.ipynb | 2 +- notebooks/py/BaristaSeq.py | 2 +- notebooks/py/ISS.py | 2 +- notebooks/py/MERFISH.py | 2 +- starfish/core/image/Filter/map.py | 44 +++++++++---- starfish/core/image/Filter/reduce.py | 56 +++++++++++++---- starfish/core/image/Filter/test/test_map.py | 5 ++ .../core/image/Filter/test/test_reduce.py | 22 +++++++ starfish/core/imagestack/imagestack.py | 9 +-- starfish/core/morphology/Filter/map.py | 46 ++++++++++---- .../DecodeSpots/test/test_trace_builders.py | 10 ++- .../FindSpots/test/test_spot_detection.py | 4 +- .../FindSpots/test/test_synthetic_data.py | 6 +- starfish/core/types/__init__.py | 2 +- starfish/core/types/_functionsource.py | 62 +++++++++++-------- starfish/types.py | 1 + workflows/wdl/iss_published/recipe.py | 3 +- 22 files changed, 198 insertions(+), 90 deletions(-) diff --git a/docs/source/_static/data_processing_examples/iss_pipeline.py b/docs/source/_static/data_processing_examples/iss_pipeline.py index c19653412..b6af8c39f 100644 --- a/docs/source/_static/data_processing_examples/iss_pipeline.py +++ b/docs/source/_static/data_processing_examples/iss_pipeline.py @@ -38,7 +38,7 @@ def iss_pipeline(fov, codebook): ) # detect spots using laplacian of gaussians approach - dots_max = fov.get_image('dots').reduce((Axes.ROUND, Axes.ZPLANE), func="max", module=FunctionSource.np) + dots_max = fov.get_image('dots').reduce((Axes.ROUND, Axes.ZPLANE), func="max") # locate spots in a reference image spots = bd.run(reference_image=dots_max, image_stack=filtered) diff --git a/docs/source/_static/tutorials/exec_image_manipulations.py b/docs/source/_static/tutorials/exec_image_manipulations.py index e2f1c2acf..71e66676d 100644 --- a/docs/source/_static/tutorials/exec_image_manipulations.py +++ b/docs/source/_static/tutorials/exec_image_manipulations.py @@ -48,7 +48,7 @@ # Here, we demonstrate selecting the last 50 pixels of (x, y) for a rounds 2 and 3 using the # :py:meth:`ImageStack.sel` method. -from starfish.types import Axes, FunctionSource +from starfish.types import Axes cropped_image: starfish.ImageStack = image.sel( {Axes.ROUND: (2, 3), Axes.X: (30, 80), Axes.Y: (50, 100)} diff --git a/notebooks/BaristaSeq.ipynb b/notebooks/BaristaSeq.ipynb index 7d7c9afb7..4d47f4b8c 100644 --- a/notebooks/BaristaSeq.ipynb +++ b/notebooks/BaristaSeq.ipynb @@ -124,7 +124,7 @@ "source": [ "from starfish.image import Filter\n", "from starfish.types import FunctionSource\n", - "max_projector = Filter.Reduce((Axes.ZPLANE,), func=\"max\", module=FunctionSource.np)\n", + "max_projector = Filter.Reduce((Axes.ZPLANE,), func=FunctionSource.np(\"max\"))\n", "z_projected_image = max_projector.run(img)\n", "z_projected_nissl = max_projector.run(nissl)\n", "\n", diff --git a/notebooks/ISS.ipynb b/notebooks/ISS.ipynb index 5312d18f5..b0abcaf36 100644 --- a/notebooks/ISS.ipynb +++ b/notebooks/ISS.ipynb @@ -259,7 +259,7 @@ " measurement_type='mean',\n", ")\n", "\n", - "dots_max = dots.reduce((Axes.ROUND, Axes.ZPLANE), func=\"max\", module=FunctionSource.np)\n", + "dots_max = dots.reduce((Axes.ROUND, Axes.ZPLANE), func=FunctionSource.np(\"max\"))\n", "spots = bd.run(image_stack=registered_imgs, reference_image=dots_max)\n", "\n", "decoder = DecodeSpots.PerRoundMaxChannel(codebook=experiment.codebook)\n", diff --git a/notebooks/MERFISH.ipynb b/notebooks/MERFISH.ipynb index 7520b4428..ee73527db 100644 --- a/notebooks/MERFISH.ipynb +++ b/notebooks/MERFISH.ipynb @@ -39,7 +39,7 @@ "\n", "from starfish import display\n", "from starfish import data, FieldOfView\n", - "from starfish.types import Axes, Features, FunctionSource\n", + "from starfish.types import Axes, Features\n", "\n", "from starfish.util.plot import (\n", " imshow_plane, intensity_histogram, overlay_spot_calls\n", diff --git a/notebooks/STARmap.ipynb b/notebooks/STARmap.ipynb index 386647b42..8f06344b3 100644 --- a/notebooks/STARmap.ipynb +++ b/notebooks/STARmap.ipynb @@ -372,4 +372,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/notebooks/py/BaristaSeq.py b/notebooks/py/BaristaSeq.py index 71ae98092..114dee2d8 100644 --- a/notebooks/py/BaristaSeq.py +++ b/notebooks/py/BaristaSeq.py @@ -87,7 +87,7 @@ # EPY: START code from starfish.image import Filter from starfish.types import FunctionSource -max_projector = Filter.Reduce((Axes.ZPLANE,), func="max", module=FunctionSource.np) +max_projector = Filter.Reduce((Axes.ZPLANE,), func=FunctionSource.np("max")) z_projected_image = max_projector.run(img) z_projected_nissl = max_projector.run(nissl) diff --git a/notebooks/py/ISS.py b/notebooks/py/ISS.py index 6bc0263ea..76b7946de 100644 --- a/notebooks/py/ISS.py +++ b/notebooks/py/ISS.py @@ -172,7 +172,7 @@ measurement_type='mean', ) -dots_max = dots.reduce((Axes.ROUND, Axes.ZPLANE), func="max", module=FunctionSource.np) +dots_max = dots.reduce((Axes.ROUND, Axes.ZPLANE), func=FunctionSource.np("max")) spots = bd.run(image_stack=registered_imgs, reference_image=dots_max) decoder = DecodeSpots.PerRoundMaxChannel(codebook=experiment.codebook) diff --git a/notebooks/py/MERFISH.py b/notebooks/py/MERFISH.py index 583dba01a..1da189d3e 100644 --- a/notebooks/py/MERFISH.py +++ b/notebooks/py/MERFISH.py @@ -29,7 +29,7 @@ from starfish import display from starfish import data, FieldOfView -from starfish.types import Axes, Features, FunctionSource +from starfish.types import Axes, Features from starfish.util.plot import ( imshow_plane, intensity_histogram, overlay_spot_calls diff --git a/starfish/core/image/Filter/map.py b/starfish/core/image/Filter/map.py index 5e243a2db..f98564d13 100644 --- a/starfish/core/image/Filter/map.py +++ b/starfish/core/image/Filter/map.py @@ -1,3 +1,4 @@ +import warnings from typing import ( Optional, Set, @@ -5,7 +6,7 @@ ) from starfish.core.imagestack.imagestack import _reconcile_clip_and_level, ImageStack -from starfish.core.types import Axes, Clip, FunctionSource, Levels +from starfish.core.types import Axes, Clip, FunctionSource, FunctionSourceBundle, Levels from ._base import FilterAlgorithm @@ -16,18 +17,24 @@ class Map(FilterAlgorithm): Parameters ---------- - func : str - Name of a function in the module specified by the ``module`` parameter to apply across the - dimension(s) specified by dims. The function is resolved by ``getattr(, func)``, - except in the cases of predefined aliases. See :py:class:`FunctionSource` for more - information about aliases. - module : FunctionSource + func : Union[str, FunctionSourceBundle] + Function to apply across the dimension(s) specified by ``dims``. + + If this value is a string, then the ``module`` parameter is consulted to determine which + python package is used to find the function. If ``module`` is not specified, then the + default is :py:attr:`FunctionSource.np`. + + If this value is a ``FunctionSourceBundle``, then the python package and module name is + obtained from the bundle. + module : Optional[FunctionSource] Python module that serves as the source of the function. It must be listed as one of the members of :py:class:`FunctionSource`. Currently, the supported FunctionSources are: - ``np``: the top-level package of numpy - ``scipy``: the top-level package of scipy + + This is being deprecated in favor of specifying the function as a ``FunctionSourceBundle``. in_place : bool Execute the operation in-place. (default: False) group_by : Set[Axes] @@ -80,16 +87,31 @@ class Map(FilterAlgorithm): def __init__( self, - func: str, + func: Union[str, FunctionSourceBundle], *func_args, - module: FunctionSource = FunctionSource.np, + module: Optional[FunctionSource] = None, in_place: bool = False, group_by: Optional[Set[Union[Axes, str]]] = None, clip_method: Optional[Clip] = None, level_method: Optional[Levels] = None, **func_kwargs, ) -> None: - self.func = module._resolve_method(func) + if isinstance(func, str): + if module is not None: + warnings.warn( + f"The module parameter is being deprecated. Use " + f"`func=FunctionSource.{module.name}{func} instead.", + DeprecationWarning) + else: + module = FunctionSource.np + self.func = module(func) + elif isinstance(func, FunctionSourceBundle): + if module is not None: + raise ValueError( + f"When passing in the function as a `FunctionSourceBundle`, module should not " + f"be set." + ) + self.func = func self.in_place = in_place if group_by is None: group_by = {Axes.ROUND, Axes.CH, Axes.ZPLANE} @@ -122,7 +144,7 @@ def run( # Apply the reducing function return stack.apply( - self.func, + self.func.resolve(), *self.func_args, group_by=self.group_by, in_place=self.in_place, diff --git a/starfish/core/image/Filter/reduce.py b/starfish/core/image/Filter/reduce.py index 84135c6eb..a4aab2ef4 100644 --- a/starfish/core/image/Filter/reduce.py +++ b/starfish/core/image/Filter/reduce.py @@ -1,3 +1,4 @@ +import warnings from typing import ( Iterable, MutableMapping, @@ -8,7 +9,16 @@ import numpy as np from starfish.core.imagestack.imagestack import _reconcile_clip_and_level, ImageStack -from starfish.core.types import ArrayLike, Axes, Clip, Coordinates, FunctionSource, Levels, Number +from starfish.core.types import ( + ArrayLike, + Axes, + Clip, + Coordinates, + FunctionSource, + FunctionSourceBundle, + Levels, + Number, +) from starfish.core.util.levels import levels from ._base import FilterAlgorithm @@ -21,11 +31,15 @@ class Reduce(FilterAlgorithm): ---------- dims : Iterable[Union[Axes, str]] one or more Axes to reduce over - func : str - Name of a function in the module specified by the ``module`` parameter to apply across the - dimension(s) specified by dims. The function is resolved by ``getattr(, func)``, - except in the cases of predefined aliases. See :py:class:`FunctionSource` for more - information about aliases. + func : Union[str, FunctionSourceBundle] + Function to apply across the dimension(s) specified by ``dims``. + + If this value is a string, then the ``module`` parameter is consulted to determine which + python package is used to find the function. If ``module`` is not specified, then the + default is :py:attr:`FunctionSource.np`. + + If this value is a ``FunctionSourceBundle``, then the python package and module name is + obtained from the bundle. Some common examples for the np FunctionSource: @@ -33,13 +47,15 @@ class Reduce(FilterAlgorithm): - max: maximum intensity projection (this is an alias for amax and applies np.amax) - mean: take the mean across the dim(s) (applies np.mean) - sum: sum across the dim(s) (applies np.sum) - module : FunctionSource + module : Optional[FunctionSource] Python module that serves as the source of the function. It must be listed as one of the members of :py:class:`FunctionSource`. Currently, the supported FunctionSources are: - ``np``: the top-level package of numpy - ``scipy``: the top-level package of scipy + + This is being deprecated in favor of specifying the function as a ``FunctionSourceBundle``. clip_method : Optional[Union[str, :py:class:`~starfish.types.Clip`]] Deprecated method to control the way that data are scaled to retain skimage dtype requirements that float data fall in [0, 1]. In all modes, data below 0 are set to 0. @@ -85,8 +101,7 @@ class Reduce(FilterAlgorithm): >>> stack = synthetic_stack() >>> reducer = Filter.Reduce( {Axes.ROUND}, - func="linalg.norm", - module=FunctionSource.scipy, + func=FunctionSource.scipy("linalg.norm"), ord=2, ) >>> norm = reducer.run(stack) @@ -100,14 +115,29 @@ class Reduce(FilterAlgorithm): def __init__( self, dims: Iterable[Union[Axes, str]], - func: str = "max", - module: FunctionSource = FunctionSource.np, + func: Union[str, FunctionSourceBundle] = "max", + module: Optional[FunctionSource] = None, clip_method: Optional[Clip] = None, level_method: Optional[Levels] = None, **kwargs ) -> None: self.dims: Iterable[Axes] = set(Axes(dim) for dim in dims) - self.func = module._resolve_method(func) + if isinstance(func, str): + if module is not None: + warnings.warn( + f"The module parameter is being deprecated. Use " + f"`func=FunctionSource.{module.name}{func} instead.", + DeprecationWarning) + else: + module = FunctionSource.np + self.func = module(func) + elif isinstance(func, FunctionSourceBundle): + if module is not None: + raise ValueError( + f"When passing in the function as a `FunctionSourceBundle`, module should not " + f"be set." + ) + self.func = func self.level_method = _reconcile_clip_and_level(clip_method, level_method) self.kwargs = kwargs @@ -134,7 +164,7 @@ def run( # Apply the reducing function reduced = stack.xarray.reduce( - self.func, dim=[dim.value for dim in self.dims], **self.kwargs) + self.func.resolve(), dim=[dim.value for dim in self.dims], **self.kwargs) # Add the reduced dims back and align with the original stack reduced = reduced.expand_dims(tuple(dim.value for dim in self.dims)) diff --git a/starfish/core/image/Filter/test/test_map.py b/starfish/core/image/Filter/test/test_map.py index 16923b96a..0f6286137 100644 --- a/starfish/core/image/Filter/test/test_map.py +++ b/starfish/core/image/Filter/test/test_map.py @@ -1,4 +1,5 @@ from starfish.core.imagestack.test.factories import synthetic_stack +from starfish.core.types import FunctionSource from .. import Map @@ -9,3 +10,7 @@ def test_map(): mapper = Map("divide", 2) output = mapper.run(stack) assert (output.xarray == 0.5).all() + + mapper = Map(FunctionSource.np("divide"), 2) + output = mapper.run(stack) + assert (output.xarray == 0.5).all() diff --git a/starfish/core/image/Filter/test/test_reduce.py b/starfish/core/image/Filter/test/test_reduce.py index 27a483c67..7fce43ae5 100644 --- a/starfish/core/image/Filter/test/test_reduce.py +++ b/starfish/core/image/Filter/test/test_reduce.py @@ -1,3 +1,4 @@ +import warnings from collections import OrderedDict import numpy as np @@ -103,6 +104,12 @@ def make_expected_image_stack(func): FunctionSource.scipy, {'ord': 2}, ), + ( + make_expected_image_stack('norm'), + FunctionSource.scipy('linalg.norm'), + None, + {'ord': 2}, + ), ] ) def test_image_stack_reduce(expected_result, func, module, kwargs): @@ -123,6 +130,21 @@ def test_image_stack_reduce(expected_result, func, module, kwargs): assert np.allclose(reduced.xarray, expected_result.xarray) +def test_image_stack_module_deprecated(): + """Specifying the function as a string and passing in a module should generate a warning.""" + with warnings.catch_warnings(record=True) as all_warnings: + Reduce(dims=[Axes.ROUND], func="max", module=FunctionSource.np) + + assert DeprecationWarning in (warning.category for warning in all_warnings) + + +def test_image_stack_module_with_functionsourcebundle(): + """Specifying the function as a FunctionSourceBundle and passing in a module should raise an + Exception.""" + with pytest.raises(ValueError): + Reduce(dims=[Axes.ROUND], func=FunctionSource.np("max"), module=FunctionSource.np) + + def test_max_projection_preserves_coordinates(): e = data.ISS(use_test_data=True) nuclei = e.fov().get_image('nuclei') diff --git a/starfish/core/imagestack/imagestack.py b/starfish/core/imagestack/imagestack.py index c7c30feb5..77299971f 100644 --- a/starfish/core/imagestack/imagestack.py +++ b/starfish/core/imagestack/imagestack.py @@ -53,6 +53,7 @@ Coordinates, CoordinateValue, FunctionSource, + FunctionSourceBundle, Levels, Number, STARFISH_EXTRAS_KEY, @@ -1180,8 +1181,8 @@ def _squeezed_numpy(self, *dims: Axes): def reduce( self, dims: Iterable[Union[Axes, str]], - func: str, - module: FunctionSource = FunctionSource.np, + func: Union[str, FunctionSourceBundle], + module: Optional[FunctionSource] = None, clip_method: Optional[Clip] = None, level_method: Optional[Levels] = None, *args, @@ -1203,8 +1204,8 @@ def reduce( def map( self, - func: str, - module: FunctionSource = FunctionSource.np, + func: Union[str, FunctionSourceBundle], + module: Optional[FunctionSource] = None, in_place: bool = False, group_by: Optional[Set[Union[Axes, str]]] = None, clip_method: Optional[Clip] = None, diff --git a/starfish/core/morphology/Filter/map.py b/starfish/core/morphology/Filter/map.py index 6d3b9482c..9db42e67a 100644 --- a/starfish/core/morphology/Filter/map.py +++ b/starfish/core/morphology/Filter/map.py @@ -1,7 +1,8 @@ -from typing import Callable, Optional +import warnings +from typing import Optional, Union from starfish.core.morphology.binary_mask import BinaryMaskCollection -from starfish.core.types import FunctionSource +from starfish.core.types import FunctionSource, FunctionSourceBundle from ._base import FilterAlgorithm @@ -12,12 +13,16 @@ class Map(FilterAlgorithm): Parameters ---------- - func : str - Name of a function in the module specified by the ``module`` parameter to apply across the - dimension(s) specified by dims. The function is resolved by ``getattr(, func)``, - except in the cases of predefined aliases. See :py:class:`FunctionSource` for more - information about aliases. - module : FunctionSource + func : Union[str, FunctionSourceBundle] + Function to apply across to each of the tiles in the input. + + If this value is a string, then the ``module`` parameter is consulted to determine which + python package is used to find the function. If ``module`` is not specified, then the + default is :py:attr:`FunctionSource.np`. + + If this value is a ``FunctionSourceBundle``, then the python package and module name is + obtained from the bundle. + module : Optional[FunctionSource] Python module that serves as the source of the function. It must be listed as one of the members of :py:class:`FunctionSource`. @@ -25,25 +30,42 @@ class Map(FilterAlgorithm): - ``np``: the top-level package of numpy - ``scipy``: the top-level package of scipy + This is being deprecated in favor of specifying the function as a ``FunctionSourceBundle``. Examples -------- Applying a binary opening function. >>> from starfish.core.morphology.binary_mask.test import factories >>> from starfish.morphology import Filter + >>> from starfish.types import FunctionSource >>> from skimage.morphology import disk >>> binary_mask_collection = factories.binary_mask_collection_2d() - >>> opener = Filter.Map("morphology.binary_opening", disk(4)) + >>> opener = Filter.Map(FunctionSource.scipy("morphology.binary_opening"), disk(4)) >>> opened = opener.run(binary_mask_collection) """ def __init__( self, - func: str, + func: Union[str, FunctionSourceBundle], *func_args, module: FunctionSource = FunctionSource.np, **func_kwargs, ) -> None: - self._func: Callable = module._resolve_method(func) + if isinstance(func, str): + if module is not None: + warnings.warn( + f"The module parameter is being deprecated. Use " + f"`func=FunctionSource.{module.name}{func} instead.", + DeprecationWarning) + else: + module = FunctionSource.np + self._func = module(func) + elif isinstance(func, FunctionSourceBundle): + if module is not None: + raise ValueError( + f"When passing in the function as a `FunctionSourceBundle`, module should not " + f"be set." + ) + self._func = func self._func_args = func_args self._func_kwargs = func_kwargs @@ -72,6 +94,6 @@ def run( # Apply the reducing function return binary_mask_collection._apply( - self._func, + self._func.resolve(), *self._func_args, **self._func_kwargs) diff --git a/starfish/core/spots/DecodeSpots/test/test_trace_builders.py b/starfish/core/spots/DecodeSpots/test/test_trace_builders.py index e595b75bc..6066dfb2f 100644 --- a/starfish/core/spots/DecodeSpots/test/test_trace_builders.py +++ b/starfish/core/spots/DecodeSpots/test/test_trace_builders.py @@ -58,7 +58,7 @@ def test_spot_detection_with_reference_image_exact_match( Each method should detect 2 total spots in the max projected reference image then group them into 2 distinct spot traces across the ImageStack. """ - reference_image = data_stack.reduce((Axes.ROUND, Axes.CH), func="max", module=FunctionSource.np) + reference_image = data_stack.reduce((Axes.ROUND, Axes.CH), func=FunctionSource.np("max")) spots = spot_detector.run(image_stack=data_stack, reference_image=reference_image) intensity_table = trace_builders.build_spot_traces_exact_match(spots) assert intensity_table.sizes[Features.AXIS] == 2, "wrong number of spots traces detected" @@ -67,8 +67,7 @@ def test_spot_detection_with_reference_image_exact_match( "wrong spot intensities detected" # verify this execution strategy produces an empty intensitytable when called with a blank image - reference_image = EMPTY_IMAGESTACK.reduce((Axes.ROUND, Axes.CH), func="max", - module=FunctionSource.np) + reference_image = EMPTY_IMAGESTACK.reduce((Axes.ROUND, Axes.CH), func=FunctionSource.np("max")) spots = spot_detector.run(image_stack=EMPTY_IMAGESTACK, reference_image=reference_image) empty_intensity_table = trace_builders.build_spot_traces_exact_match(spots) assert empty_intensity_table.sizes[Features.AXIS] == 0 @@ -134,15 +133,14 @@ def test_spot_finding_reference_image_sequential( we're testing it anyway. """ - reference_image = data_stack.reduce((Axes.ROUND, Axes.CH), func="max", module=FunctionSource.np) + reference_image = data_stack.reduce((Axes.ROUND, Axes.CH), func=FunctionSource.np("max")) spots = spot_detector.run(image_stack=data_stack, reference_image=reference_image) intensity_table = trace_builders.build_traces_sequential(spots) expected_num_traces = (2 * data_stack.num_chs * data_stack.num_rounds) assert intensity_table.sizes[Features.AXIS] == expected_num_traces, "wrong number of " \ "spots traces detected" - reference_image = EMPTY_IMAGESTACK.reduce((Axes.ROUND, Axes.CH), func="max", - module=FunctionSource.np) + reference_image = EMPTY_IMAGESTACK.reduce((Axes.ROUND, Axes.CH), func=FunctionSource.np("max")) spots = spot_detector.run(image_stack=EMPTY_IMAGESTACK, reference_image=reference_image) empty_intensity_table = trace_builders.build_traces_sequential(spots) assert empty_intensity_table.sizes[Features.AXIS] == 0 diff --git a/starfish/core/spots/FindSpots/test/test_spot_detection.py b/starfish/core/spots/FindSpots/test/test_spot_detection.py index 01ec4e54e..3c8b4c590 100644 --- a/starfish/core/spots/FindSpots/test/test_spot_detection.py +++ b/starfish/core/spots/FindSpots/test/test_spot_detection.py @@ -86,7 +86,7 @@ def test_spot_detection_with_reference_image( r/ch pair. The final spot results should represent 2 spots for each r/ch totalling 2*num_rounds*num_ch spots """ - reference_image = data_stack.reduce((Axes.ROUND, Axes.CH), func="max", module=FunctionSource.np) + reference_image = data_stack.reduce((Axes.ROUND, Axes.CH), func=FunctionSource.np("max")) spots_results = spot_detector.run(image_stack=data_stack, reference_image=reference_image) assert spots_results.count_total_spots() == (2 * data_stack.num_chs * data_stack.num_rounds), \ "wrong number of spots detected" @@ -124,7 +124,7 @@ def _make_labeled_image() -> ImageStack: def test_reference_image_spot_detection_with_image_with_labeled_axes(): """This testing method uses a reference image to identify spot locations.""" data_stack = _make_labeled_image() - reference_image = data_stack.reduce((Axes.ROUND, Axes.CH), func="max", module=FunctionSource.np) + reference_image = data_stack.reduce((Axes.ROUND, Axes.CH), func=FunctionSource.np("max")) spot_results = gaussian_spot_detector.run(image_stack=data_stack, reference_image=reference_image) return spot_results diff --git a/starfish/core/spots/FindSpots/test/test_synthetic_data.py b/starfish/core/spots/FindSpots/test/test_synthetic_data.py index f4f06ced8..d3ad88a81 100644 --- a/starfish/core/spots/FindSpots/test/test_synthetic_data.py +++ b/starfish/core/spots/FindSpots/test/test_synthetic_data.py @@ -27,8 +27,7 @@ def test_round_trip_synthetic_data(): codebook = sd.codebook() intensities = sd.intensities(codebook=codebook) spots = sd.spots(intensities=intensities) - spots_max_projector = Filter.Reduce((Axes.ROUND, Axes.CH), func="max", - module=FunctionSource.np) + spots_max_projector = Filter.Reduce((Axes.ROUND, Axes.CH), func=FunctionSource.np("max")) spots_max = spots_max_projector.run(spots) gsd = BlobDetector(min_sigma=1, max_sigma=4, num_sigma=5, threshold=0) @@ -89,8 +88,7 @@ def test_medium_synthetic_stack(): intensities = intensities[np.where(valid_locations)] spots = sd.spots(intensities=intensities) gsd = BlobDetector(min_sigma=1, max_sigma=4, num_sigma=5, threshold=1e-4) - spots_max_projector = Filter.Reduce((Axes.ROUND, Axes.CH), func="max", - module=FunctionSource.np) + spots_max_projector = Filter.Reduce((Axes.ROUND, Axes.CH), func=FunctionSource.np("max")) spots_max = spots_max_projector.run(spots) spot_results = gsd.run(image_stack=spots, reference_image=spots_max) diff --git a/starfish/core/types/__init__.py b/starfish/core/types/__init__.py index 1ba4d5680..2966aff28 100644 --- a/starfish/core/types/__init__.py +++ b/starfish/core/types/__init__.py @@ -19,7 +19,7 @@ TransformType ) from ._decoded_spots import DecodedSpots -from ._functionsource import FunctionSource +from ._functionsource import FunctionSource, FunctionSourceBundle from ._spot_attributes import SpotAttributes from ._spot_finding_results import PerImageSliceSpotResults, SpotFindingResults diff --git a/starfish/core/types/_functionsource.py b/starfish/core/types/_functionsource.py index 7ccfacb41..1583167f2 100644 --- a/starfish/core/types/_functionsource.py +++ b/starfish/core/types/_functionsource.py @@ -1,4 +1,5 @@ import importlib +from dataclasses import dataclass from enum import Enum from typing import ( Callable, @@ -8,28 +9,14 @@ ) -class FunctionSource(Enum): - """Each FunctionSource declares a package from which reduction methods can be obtained. - Generally, the packages should be those that are included as starfish's dependencies for - reproducibility. - - Many packages are broken into subpackages which are not necessarily implicitly imported when - importing the top-level package. For example, ``scipy.linalg`` is not implicitly imported - when one imports ``scipy``. To avoid the complexity of enumerating each scipy subpackage in - FunctionSource, we assemble the fully-qualified method name, and then try all the - permutations of how one could import that method. - - In the example of ``scipy.linalg.norm``, we try the following: - - 1. import ``scipy``, attempt to resolve ``linalg.norm``. - 2. import ``scipy.linalg``, attempt to resolve ``norm``. - """ +@dataclass +class FunctionSourceBundle: + """The combination of a ``FunctionSource`` and the method we are looking for.""" + top_level_package: str + requested_method: str + actual_method: str - def __init__(self, top_level_package: str, aliases: Optional[Mapping[str, str]] = None): - self.top_level_package = top_level_package - self.aliases = aliases or {} - - def _resolve_method(self, method: str) -> Callable: + def resolve(self) -> Callable: """Resolve a method. The method itself might be enclosed in a package, such as subpackage.actual_method. In that case, we will need to attempt to resolve it in the following sequence: @@ -42,10 +29,7 @@ def _resolve_method(self, method: str) -> Callable: This is done instead of just creating a bunch of FunctionSource for libraries that have a lot of packages that are not implicitly imported by importing the top-level package. """ - # first resolve the aliases. - actual_method = self.aliases.get(method, method) - - method_splitted = actual_method.split(".") + method_splitted = self.actual_method.split(".") splitted = [self.top_level_package] splitted.extend(method_splitted) @@ -63,9 +47,35 @@ def _resolve_method(self, method: str) -> Callable: pass raise AttributeError( - f"Unable to resolve the method {actual_method} from package " + f"Unable to resolve the method {self.requested_method} from package " f"{self.top_level_package}") + +class FunctionSource(Enum): + """Each FunctionSource declares a package from which reduction methods can be obtained. + Generally, the packages should be those that are included as starfish's dependencies for + reproducibility. + + Many packages are broken into subpackages which are not necessarily implicitly imported when + importing the top-level package. For example, ``scipy.linalg`` is not implicitly imported + when one imports ``scipy``. To avoid the complexity of enumerating each scipy subpackage in + FunctionSource, we assemble the fully-qualified method name, and then try all the + permutations of how one could import that method. + + In the example of ``scipy.linalg.norm``, we try the following: + + 1. import ``scipy``, attempt to resolve ``linalg.norm``. + 2. import ``scipy.linalg``, attempt to resolve ``norm``. + """ + + def __init__(self, top_level_package: str, aliases: Optional[Mapping[str, str]] = None): + self.top_level_package = top_level_package + self.aliases = aliases or {} + + def __call__(self, method: str) -> FunctionSourceBundle: + return FunctionSourceBundle( + self.top_level_package, method, self.aliases.get(method, method)) + np = ("numpy", {'max': 'amax'}) """Function source for the numpy libraries""" scipy = ("scipy",) diff --git a/starfish/types.py b/starfish/types.py index 8bf4e1046..59cd044f6 100644 --- a/starfish/types.py +++ b/starfish/types.py @@ -8,6 +8,7 @@ CORE_DEPENDENCIES, Features, FunctionSource, + FunctionSourceBundle, Levels, LOG, Number, diff --git a/workflows/wdl/iss_published/recipe.py b/workflows/wdl/iss_published/recipe.py index dd12cfdb9..7f921da25 100644 --- a/workflows/wdl/iss_published/recipe.py +++ b/workflows/wdl/iss_published/recipe.py @@ -57,8 +57,7 @@ def process_fov(field_num: int, experiment_str: str): threshold=0.01, measurement_type='mean', ) - dots_max_projector = Filter.Reduce((Axes.ROUND, Axes.ZPLANE), func="max", - module=FunctionSource.np) + dots_max_projector = Filter.Reduce((Axes.ROUND, Axes.ZPLANE), func=FunctionSource.np("max")) dots_max = dots_max_projector.run(dots) spots = detector.run(image_stack=filtered_imgs, reference_image=dots_max)