Make annotations work

mypy.ini

@@ -2,7 +2,6 @@
 follow_imports = silent
 strict_optional = True
 warn_redundant_casts = True
-warn_unused_ignores = True
 disallow_any_generics = True
 check_untyped_defs = True
 no_implicit_reexport = True

src/ome_autogen.py

@@ -41,11 +41,11 @@ def __post_init__(self) -> None:
 # Maps XSD TypeName to Override configuration, used to control output for that type.
 OVERRIDES = {
     "MetadataOnly": Override(type_="bool", default="False"),
-    "XMLAnnotation": Override(
-        type_="Optional[str]", default="None", imports="from typing import Optional",
-    ),
+    # FIXME: Type should be xml.etree.ElementTree.Element but isinstance checks
+    # with that class often mysteriously fail so the validator fails.
+    "XMLAnnotation/Value": Override(type_="Any", imports="from typing import Any"),
     "BinData/Length": Override(type_="int"),
-    # FIXME: hard-coded LightSource subclass lists
+    # FIXME: hard-coded subclass lists
     "Instrument/LightSourceGroup": Override(
         type_="List[LightSource]",
         default="field(default_factory=list)",
@@ -142,6 +142,60 @@ def validate_union(
                     raise ValueError("invalid type for union values")
         """,
     ),
+    "OME/StructuredAnnotations": Override(
+        type_="List[Annotation]",
+        default="field(default_factory=list)",
+        imports="""
+            from typing import Dict, Union, Any
+            from pydantic import validator
+            from .annotation import Annotation
+            from .boolean_annotation import BooleanAnnotation
+            from .comment_annotation import CommentAnnotation
+            from .double_annotation import DoubleAnnotation
+            from .file_annotation import FileAnnotation
+            from .list_annotation import ListAnnotation
+            from .long_annotation import LongAnnotation
+            from .tag_annotation import TagAnnotation
+            from .term_annotation import TermAnnotation
+            from .timestamp_annotation import TimestampAnnotation
+            from .xml_annotation import XMLAnnotation
+
+            _annotation_types: Dict[str, type] = {
+                "boolean_annotation": BooleanAnnotation,
+                "comment_annotation": CommentAnnotation,
+                "double_annotation": DoubleAnnotation,
+                "file_annotation": FileAnnotation,
+                "list_annotation": ListAnnotation,
+                "long_annotation": LongAnnotation,
+                "tag_annotation": TagAnnotation,
+                "term_annotation": TermAnnotation,
+                "timestamp_annotation": TimestampAnnotation,
+                "xml_annotation": XMLAnnotation,
+            }
+        """,
+        body="""
+            @validator("structured_annotations", pre=True, each_item=True)
+            def validate_structured_annotations(
+                cls, value: Union[Annotation, Dict[Any, Any]]
+            ) -> Annotation:
+                if isinstance(value, Annotation):
+                    return value
+                elif isinstance(value, dict):
+                    try:
+                        _type = value.pop("_type")
+                    except KeyError:
+                        raise ValueError(
+                            "dict initialization requires _type"
+                        ) from None
+                    try:
+                        annotation_cls = _annotation_types[_type]
+                    except KeyError:
+                        raise ValueError(f"unknown Annotation type '{_type}'") from None
+                    return annotation_cls(**value)
+                else:
+                    raise ValueError("invalid type for annotation values")
+        """,
+    ),
     "TiffData/UUID": Override(
         type_="Optional[UUID]",
         default="None",

src/ome_types/schema.py

@@ -1,11 +1,15 @@
 import pickle
 import re
 from os.path import dirname, exists, join
+from xml.etree import ElementTree
 from typing import Any, Dict, Optional
 
 import xmlschema
 from xmlschema.converters import XMLSchemaConverter
 
+
+NS_OME = "{http://www.openmicroscopy.org/Schemas/OME/2016-06}"
+
 __cache__: Dict[str, xmlschema.XMLSchema] = {}
 
 
@@ -33,9 +37,8 @@ def get_schema(xml: str) -> xmlschema.XMLSchema:
 
             # FIXME Hack to work around xmlschema poor support for keyrefs to
             # substitution groups
-            ns = "{http://www.openmicroscopy.org/Schemas/OME/2016-06}"
-            ls_sgs = schema.maps.substitution_groups[f"{ns}LightSourceGroup"]
-            ls_id_maps = schema.maps.identities[f"{ns}LightSourceIDKey"]
+            ls_sgs = schema.maps.substitution_groups[f"{NS_OME}LightSourceGroup"]
+            ls_id_maps = schema.maps.identities[f"{NS_OME}LightSourceIDKey"]
             ls_id_maps.elements = {e: None for e in ls_sgs}
 
             __cache__[version] = schema
@@ -107,6 +110,29 @@ def element_decode(self, data, xsd_element, xsd_type=None, level=0):  # type: ig
                         v["_type"] = _type
                     shapes.extend(values)
             result = shapes
+        elif xsd_element.local_name == "StructuredAnnotations":
+            annotations = []
+            for _type in (
+                "boolean_annotation",
+                "comment_annotation",
+                "double_annotation",
+                "file_annotation",
+                "list_annotation",
+                "long_annotation",
+                "tag_annotation",
+                "term_annotation",
+                "timestamp_annotation",
+                "xml_annotation",
+            ):
+                if _type in result:
+                    values = result.pop(_type)
+                    for v in values:
+                        v["_type"] = _type
+                        # Normalize empty element to zero-length string.
+                        if "value" in v and v["value"] is None:
+                            v["value"] = ""
+                    annotations.extend(values)
+            result = annotations
         return result
 
 
@@ -117,4 +143,16 @@ def to_dict(  # type: ignore
     **kwargs,
 ) -> Dict[str, Any]:
     schema = schema or get_schema(xml)
-    return schema.to_dict(xml, converter=converter, **kwargs)
+    result = schema.to_dict(xml, converter=converter, **kwargs)
+    # xmlschema doesn't provide usable access to mixed XML content, so we'll
+    # fill the XMLAnnotation value attributes ourselves by re-parsing the XML
+    # with ElementTree and using the Element objects as the values.
+    tree = None
+    for annotation in result.get("structured_annotations", []):
+        if annotation["_type"] == "xml_annotation":
+            if tree is None:
+                tree = ElementTree.parse(xml)
+            aid = annotation["id"]
+            elt = tree.find(f".//{NS_OME}XMLAnnotation[@ID='{aid}']/{NS_OME}Value")
+            annotation["value"] = elt
+    return result

testing/test_autogen.py

@@ -30,17 +30,8 @@ def model(tmp_path_factory, request):
 
 
 SHOULD_FAIL = {
-    "commentannotation",
-    "mapannotation",
-    "spim",
-    "tagannotation",
+    # Some timestamps have negative years which datetime doesn't support.
     "timestampannotation",
-    "timestampannotation-posix-only",
-    "transformations-downgrade",
-    "transformations-upgrade",
-    "xmlannotation-body-space",
-    "xmlannotation-multi-value",
-    "xmlannotation-svg",
 }
 SHOULD_RAISE = {"bad"}