Multi table

.github/workflows/test_and_deploy.yaml

@@ -63,7 +63,7 @@ jobs:
                   PLATFORM: ${{ matrix.os }}
                   DISPLAY: :42
               run: |
-                  pytest -v --cov --color=yes --cov-report=xml
+                  pytest --cov --color=yes --cov-report=xml
             - name: Upload coverage to Codecov
               uses: codecov/codecov-action@v3.1.1
               with:

.gitignore

@@ -45,3 +45,6 @@ spatialdata-sandbox
 
 # version file
 _version.py
+
+# prettier cache
+node_modules/

.pre-commit-config.yaml

@@ -9,25 +9,25 @@ ci:
     skip: []
 repos:
     - repo: https://github.com/psf/black
-      rev: 23.10.1
+      rev: 23.12.1
       hooks:
           - id: black
     - repo: https://github.com/pre-commit/mirrors-prettier
-      rev: v3.0.3
+      rev: v4.0.0-alpha.8
       hooks:
           - id: prettier
     - repo: https://github.com/asottile/blacken-docs
       rev: 1.16.0
       hooks:
           - id: blacken-docs
     - repo: https://github.com/pre-commit/mirrors-mypy
-      rev: v1.6.1
+      rev: v1.8.0
       hooks:
           - id: mypy
             additional_dependencies: [numpy, types-requests]
             exclude: tests/|docs/
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.1.3
+      rev: v0.1.9
       hooks:
           - id: ruff
             args: [--fix, --exit-non-zero-on-fix]

.readthedocs.yaml

@@ -6,7 +6,7 @@ build:
         python: "3.10"
 sphinx:
     configuration: docs/conf.py
-    fail_on_warning: false
+    fail_on_warning: true
 python:
     install:
         - method: pip

CHANGELOG.md

@@ -8,10 +8,53 @@ and this project adheres to [Semantic Versioning][].
 [keep a changelog]: https://keepachangelog.com/en/1.0.0/
 [semantic versioning]: https://semver.org/spec/v2.0.0.html
 
-## [0.0.15] - tbd
+## [0.1.0] - tbd
 
 ### Added
 
+#### Major
+
+-   Implemented support in SpatialData for storing multiple tables. These tables
+    can annotate a SpatialElement but not necessarily so.
+-   Increased in-memory vs on-disk control: changes performed in-memory (e.g. adding a new image) are not automatically performed on-disk.
+
+#### Minor
+
+-   Added public helper function get_table_keys in spatialdata.models to retrieve annotation information of a given table.
+-   Added public helper function check_target_region_column_symmetry in spatialdata.models to check whether annotation
+    metadata in table.uns['spatialdata_attrs'] corresponds with respective columns in table.obs.
+-   Added function validate_table_in_spatialdata in SpatialData to validate the annotation target of a table being
+    present in the SpatialData object.
+-   Added function get_annotated_regions in SpatialData to get the regions annotated by a given table.
+-   Added function get_region_key_column in SpatialData to get the region_key column in table.obs.
+-   Added function get_instance_key_column in SpatialData to get the instance_key column in table.obs.
+-   Added function set_table_annotates_spatialelement in SpatialData to either set or change the annotation metadata of
+    a table in a given SpatialData object.
+-   Added tables property in SpatialData.
+-   Added tables setter in SpatialData.
+-   Added gen_spatial_elements generator in SpatialData to generate the SpatialElements in a given SpatialData object.
+-   Added gen_elements generator in SpatialData to generate elements of a SpatialData object including tables.
+
+### Changed
+
+#### Minor
+
+-   Changed the string representation of SpatialData to reflect the changes in regard to multiple tables.
+
+## [0.0.x] - tbd
+
+### Minor
+
+-   improved usability and robustness of sdata.write() when overwrite=True @aeisenbarth
+
+### Added
+
+### Fixed
+
+#### Minor
+
+-   refactored data loader for deep learning
+
 ## [0.0.14] - 2023-10-11
 
 ### Added

docs/_templates/autosummary/class.rst

@@ -12,11 +12,8 @@ Attributes table
 ~~~~~~~~~~~~~~~~~~
 
 .. autosummary::
-
 {% for item in attributes %}
-
     ~{{ fullname }}.{{ item }}
-
 {%- endfor %}
 {% endif %}
 {% endblock %}
@@ -27,13 +24,10 @@ Methods table
 ~~~~~~~~~~~~~
 
 .. autosummary::
-
 {% for item in methods %}
-
     {%- if item != '__init__' %}
     ~{{ fullname }}.{{ item }}
     {%- endif -%}
-
 {%- endfor %}
 {% endif %}
 {% endblock %}
@@ -46,7 +40,6 @@ Attributes
 {% for item in attributes %}
 
 .. autoattribute:: {{ [objname, item] | join(".") }}
-
 {%- endfor %}
 
 {% endif %}
@@ -61,7 +54,6 @@ Methods
 {%- if item != '__init__' %}
 
 .. automethod:: {{ [objname, item] | join(".") }}
-
 {%- endif -%}
 {%- endfor %}
 

docs/api.md

@@ -29,12 +29,12 @@ Operations on `SpatialData` objects.
     get_extent
     match_table_to_element
     concatenate
-    rasterize
     transform
+    rasterize
     aggregate
 ```
 
-### Utilities
+### Operations Utilities
 
 ```{eval-rst}
 .. autosummary::
@@ -49,6 +49,7 @@ The elements (building-blocks) that consitute `SpatialData`.
 
 ```{eval-rst}
 .. currentmodule:: spatialdata.models
+
 .. autosummary::
     :toctree: generated
 
@@ -61,9 +62,11 @@ The elements (building-blocks) that consitute `SpatialData`.
     TableModel
 ```
 
-### Utilities
+### Models Utilities
 
 ```{eval-rst}
+.. currentmodule:: spatialdata.models
+
 .. autosummary::
     :toctree: generated
 
@@ -94,9 +97,11 @@ The transformations that can be defined between elements and coordinate systems
     Sequence
 ```
 
-### Utilities
+### Transformations Utilities
 
 ```{eval-rst}
+.. currentmodule:: spatialdata.transformations
+
 .. autosummary::
     :toctree: generated
 
@@ -119,7 +124,7 @@ The transformations that can be defined between elements and coordinate systems
     ImageTilesDataset
 ```
 
-## Input/output
+## Input/Output
 
 ```{eval-rst}
 .. currentmodule:: spatialdata
@@ -129,4 +134,5 @@ The transformations that can be defined between elements and coordinate systems
 
     read_zarr
     save_transformations
+    get_dask_backing_files
 ```

pyproject.toml

@@ -50,7 +50,6 @@ dev = [
 docs = [
     "sphinx>=4.5",
     "sphinx-book-theme>=1.0.0",
-    "sphinx_rtd_theme",
     "myst-nb",
     "sphinxcontrib-bibtex>=1.0.0",
     "sphinx-autodoc-typehints",

src/spatialdata/__init__.py

@@ -28,6 +28,7 @@
     "read_zarr",
     "unpad_raster",
     "save_transformations",
+    "get_dask_backing_files",
 ]
 
 from spatialdata import dataloader, models, transformations
@@ -40,6 +41,6 @@
 from spatialdata._core.query.relational_query import get_values, match_table_to_element
 from spatialdata._core.query.spatial_query import bounding_box_query, polygon_query
 from spatialdata._core.spatialdata import SpatialData
-from spatialdata._io._utils import save_transformations
+from spatialdata._io._utils import get_dask_backing_files, save_transformations
 from spatialdata._io.io_zarr import read_zarr
 from spatialdata._utils import unpad_raster

src/spatialdata/_core/_elements.py

@@ -0,0 +1,116 @@
+"""SpatialData elements."""
+from __future__ import annotations
+
+from collections import UserDict
+from collections.abc import Iterable
+from typing import Any
+from warnings import warn
+
+from anndata import AnnData
+from dask.dataframe.core import DataFrame as DaskDataFrame
+from datatree import DataTree
+from geopandas import GeoDataFrame
+
+from spatialdata._types import Raster_T
+from spatialdata._utils import multiscale_spatial_image_from_data_tree
+from spatialdata.models import (
+    Image2DModel,
+    Image3DModel,
+    Labels2DModel,
+    Labels3DModel,
+    PointsModel,
+    ShapesModel,
+    TableModel,
+    get_axes_names,
+    get_model,
+)
+
+
+class Elements(UserDict[str, Any]):
+    def __init__(self, shared_keys: set[str | None]) -> None:
+        self._shared_keys = shared_keys
+        super().__init__()
+
+    @staticmethod
+    def _check_key(key: str, element_keys: Iterable[str], shared_keys: set[str | None]) -> None:
+        if key in element_keys:
+            warn(f"Key `{key}` already exists. Overwriting it.", UserWarning, stacklevel=2)
+        else:
+            if key in shared_keys:
+                raise KeyError(f"Key `{key}` already exists.")
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self._shared_keys.add(key)
+        super().__setitem__(key, value)
+
+    def __delitem__(self, key: str) -> None:
+        self._shared_keys.remove(key)
+        super().__delitem__(key)
+
+
+class Images(Elements):
+    def __setitem__(self, key: str, value: Raster_T) -> None:
+        self._check_key(key, self.keys(), self._shared_keys)
+        if isinstance(value, (DataTree)):
+            value = multiscale_spatial_image_from_data_tree(value)
+        schema = get_model(value)
+        if schema not in (Image2DModel, Image3DModel):
+            raise TypeError(f"Unknown element type with schema: {schema!r}.")
+        ndim = len(get_axes_names(value))
+        if ndim == 3:
+            Image2DModel().validate(value)
+            super().__setitem__(key, value)
+        elif ndim == 4:
+            Image3DModel().validate(value)
+            super().__setitem__(key, value)
+        else:
+            NotImplementedError("TODO: implement for ndim > 4.")
+
+
+class Labels(Elements):
+    def __setitem__(self, key: str, value: Raster_T) -> None:
+        self._check_key(key, self.keys(), self._shared_keys)
+        if isinstance(value, (DataTree)):
+            value = multiscale_spatial_image_from_data_tree(value)
+        schema = get_model(value)
+        if schema not in (Labels2DModel, Labels3DModel):
+            raise TypeError(f"Unknown element type with schema: {schema!r}.")
+        ndim = len(get_axes_names(value))
+        if ndim == 2:
+            Labels2DModel().validate(value)
+            super().__setitem__(key, value)
+        elif ndim == 3:
+            Labels3DModel().validate(value)
+            super().__setitem__(key, value)
+        else:
+            NotImplementedError("TODO: implement for ndim > 3.")
+
+
+class Shapes(Elements):
+    def __setitem__(self, key: str, value: GeoDataFrame) -> None:
+        self._check_key(key, self.keys(), self._shared_keys)
+        schema = get_model(value)
+        if schema != ShapesModel:
+            raise TypeError(f"Unknown element type with schema: {schema!r}.")
+        ShapesModel().validate(value)
+        super().__setitem__(key, value)
+
+
+class Points(Elements):
+    def __setitem__(self, key: str, value: DaskDataFrame) -> None:
+        self._check_key(key, self.keys(), self._shared_keys)
+        schema = get_model(value)
+        if schema != PointsModel:
+            raise TypeError(f"Unknown element type with schema: {schema!r}.")
+        PointsModel().validate(value)
+        super().__setitem__(key, value)
+
+
+class Tables(Elements):
+    def __setitem__(self, key: str, value: AnnData) -> None:
+        self._check_key(key, self.keys(), self._shared_keys)
+        schema = get_model(value)
+        if schema != TableModel:
+            raise TypeError(f"Unknown element type with schema: {schema!r}.")
+        TableModel().validate(value)
+        super().__setitem__(key, value)

src/spatialdata/_core/_utils.py

@@ -0,0 +1,22 @@
+from spatialdata._core.spatialdata import SpatialData
+
+
+def _find_common_table_keys(sdatas: list[SpatialData]) -> set[str]:
+    """
+    Find table keys present in more than one SpatialData object.
+
+    Parameters
+    ----------
+    sdatas
+        A list of SpatialData objects.
+
+    Returns
+    -------
+    A set of common keys that are present in the tables of more than one SpatialData object.
+    """
+    common_keys = set(sdatas[0].tables.keys())
+
+    for sdata in sdatas[1:]:
+        common_keys.intersection_update(sdata.tables.keys())
+
+    return common_keys