Introduce Runnable (#1163)

Runnable is the base unit of a recipe.  Each runnable constitutes a PipelineComponent with a specific algorithm.  Constructor arguments that are FileProviders are loaded into memory and passed in.  Inputs to the `run()` method can also include FileProviders.  In all cases, FileProviders are associated with a file path or url.  The type is inferred from the typing parameters of the method (currently supported: ImageStack, IntensityTable, ExpressionMatrix, and Codebook).

Runnables will be wired together to constitute a pipeline recipe.

Test plan: Added tests to verify a simple Runnable, chained Runnables, and Runnables that has constructor arguments that are FileProviders.

Depends on #1095

docs/source/api/index.rst

@@ -18,6 +18,9 @@ API
 .. toctree::
    spots/index.rst
 
+.. toctree::
+   recipe/index.rst
+
 .. toctree::
    types/index.rst
 

docs/source/api/recipe/index.rst

@@ -0,0 +1,13 @@
+.. _Recipe:
+
+Runnable
+========
+
+.. autoclass:: starfish.recipe.Runnable
+   :members:
+
+FileProvider
+============
+
+.. autoclass:: starfish.recipe.filesystem.FileProvider
+   :members:

starfish/recipe/__init__.py

@@ -0,0 +1,9 @@
+from .errors import (
+    ConstructorError,
+    ConstructorExtraParameterWarning,
+    ExecutionError,
+    RecipeError,
+    RunInsufficientParametersError,
+    TypeInferenceError,
+)
+from .runnable import Runnable

starfish/recipe/errors.py

@@ -0,0 +1,30 @@
+class RecipeWarning(RuntimeWarning):
+    pass
+
+
+class RecipeError(Exception):
+    pass
+
+
+class ConstructorExtraParameterWarning(RecipeWarning):
+    """Raised when a recipe contains parameters that an algorithms constructor does not expect."""
+
+
+class TypeInferenceError(RecipeError):
+    """Raised when we cannot infer the type of object an algorithm expects in its constructor or
+    its run method.  This can be fixed by ensuring all the parameters to the constructor and the run
+    method have type hints."""
+
+
+class ConstructorError(RecipeError):
+    """Raised when there is an error raised during the construction of an algorithm class."""
+    pass
+
+
+class RunInsufficientParametersError(RecipeError):
+    """Raised when the recipe does not provide sufficient parameters for the run method."""
+
+
+class ExecutionError(RecipeError):
+    """Raised when there is an error raised during the execution of an algorithm."""
+    pass

starfish/recipe/filesystem.py

@@ -0,0 +1,108 @@
+import enum
+from typing import Any, Callable, Type
+
+from starfish.codebook.codebook import Codebook
+from starfish.expression_matrix.expression_matrix import ExpressionMatrix
+from starfish.imagestack.imagestack import ImageStack
+from starfish.intensity_table.intensity_table import IntensityTable
+from starfish.util.indirectfile import (
+    convert,
+    GetCodebook,
+    GetCodebookFromExperiment,
+    GetImageStack,
+    GetImageStackFromExperiment,
+)
+
+
+def imagestack_convert(indirect_path_or_url: str) -> ImageStack:
+    """Converts a path or URL to an ImageStack.  This supports the indirect syntax, where a user
+    provides a string like @<url_or_path_of_experiment.json>[fov_name][image_name].  If the indirect
+    syntax is used, the experiment.json is automatically fetched and traversed to find the specified
+    image in the specified field of view."""
+    return convert(
+        indirect_path_or_url,
+        [
+            GetImageStack(),
+            GetImageStackFromExperiment(),
+        ],
+    )
+
+
+def codebook_convert(indirect_path_or_url: str) -> Codebook:
+    """Converts a path or URL to a Codebook.  This supports the indirect syntax, where a user
+    provides a string like @<url_or_path_of_experiment.json>.  If the indirect syntax is used, the
+    experiment.json is automatically fetched to find the codebook."""
+    return convert(
+        indirect_path_or_url,
+        [
+            GetCodebook(),
+            GetCodebookFromExperiment(),
+        ],
+    )
+
+
+class FileTypes(enum.Enum):
+    """These are the filetypes supported as inputs and outputs for recipes.  Each filetype is
+    associated with the implementing class, the method to invoke to load such a filetype, and the
+    method to invoke to save back to the filetype.
+
+    The load method is expected to be called with a string, which is the file or url to load from,
+    and is expected to return an instantiated object.
+
+    The save method is expected to be called with the object and a string, which is the path to
+    write the object to.
+    """
+    IMAGESTACK = (ImageStack, imagestack_convert, ImageStack.export)
+    INTENSITYTABLE = (IntensityTable, IntensityTable.open_netcdf, IntensityTable.to_netcdf)
+    EXPRESSIONMATRIX = (ExpressionMatrix, ExpressionMatrix.load, ExpressionMatrix.save)
+    CODEBOOK = (Codebook, codebook_convert, Codebook.to_json)
+
+    def __init__(self, cls: Type, loader: Callable[[str], Any], saver: Callable[[Any, str], None]):
+        self._cls = cls
+        self._load = loader
+        self._save = saver
+
+    @property
+    def load(self) -> Callable[[str], Any]:
+        return self._load
+
+    @property
+    def save(self) -> Callable[[Any, str], None]:
+        return self._save
+
+    @staticmethod
+    def resolve_by_class(cls: Type) -> "FileTypes":
+        for member in FileTypes.__members__.values():
+            if cls == member.value[0]:
+                return member
+        raise TypeError(f"filetype {cls} not supported.")
+
+    @staticmethod
+    def resolve_by_instance(instance) -> "FileTypes":
+        for member in FileTypes.__members__.values():
+            if isinstance(instance, member.value[0]):
+                return member
+        raise TypeError(f"filetype of {instance.__class__} not supported.")
+
+
+class FileProvider:
+    """This is used to wrap paths or URLs that are passed into Runnables via the `file_inputs` magic
+    variable.  This is so we can differentiate between strings and `file_inputs` values, which must
+    be first constructed into a starfish object via its loader."""
+    def __init__(self, path_or_url: str) -> None:
+        self.path_or_uri = path_or_url
+
+    def __str__(self):
+        return f"FileProvider(\"{self.path_or_uri}\")"
+
+
+class TypedFileProvider:
+    """Like :py:class:`FileProvider`, this is used to wrap paths or URLs that are passed into
+    Runnables via the `file_inputs` magic variable.  In this case, the object type has been
+    resolved by examining the type annotation."""
+    def __init__(self, backing_file_provider: FileProvider, object_class: Type) -> None:
+        self.backing_file_provider = backing_file_provider
+        self.type = FileTypes.resolve_by_class(object_class)
+
+    def load(self) -> Any:
+        return self.type.load(self.backing_file_provider.path_or_uri)