feature/mx-1649 add preview models and api method (#347)

### PR Context
- prep for robert-koch-institut/mex-editor#192
- preview models don't care about required fields and length limits for lists
- they will be used by the editor to be able to search even for potentially "broken" items
- also includes some clean-up of the `mex.common.models` module (imports, exports and docs)

### Added
- add preview models for merged items without cardinality validation
- BREAKING: preview models are now part of all `mex.common.fields` lookups
- add `BackendApiConnector.fetch_preview_items` for fetching previews

### Deprecated
- stop using `ExtractedData`, use `AnyExtractedModel` instead
- stop using `MergedItem`, use `AnyMergedModel` instead
- stop using `AdditiveRule`, use `AnyAdditiveRule` instead
- stop using `SubtractiveRule`, use `AnySubtractiveRule` instead
- stop using `PreventiveRule`, use `AnyPreventiveRule` instead
- stop using `BaseEntity`, use a concrete union instead

### Removed
- removed deprecated `BulkInsertResponse` as alias for `IdentifiersResponse`
- removed unused module export of `mex.common.models.generate_entity_filter_schema`
- removed unused module export of `mex.common.models.generate_mapping_schema`
- drop export `models.ExtractedPrimarySourceIdentifier`, import from `types` instead
- drop export `models.MergedPrimarySourceIdentifier`, import from `types` instead

CHANGELOG.md

@@ -9,12 +9,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- add preview models for merged items without cardinality validation
+- BREAKING: preview models are now part of all `mex.common.fields` lookups
+- add `BackendApiConnector.fetch_preview_items` for fetching previews
+
 ### Changes
 
 ### Deprecated
 
+- stop using `ExtractedData`, use `AnyExtractedModel` instead
+- stop using `MergedItem`, use `AnyMergedModel` instead
+- stop using `AdditiveRule`, use `AnyAdditiveRule` instead
+- stop using `SubtractiveRule`, use `AnySubtractiveRule` instead
+- stop using `PreventiveRule`, use `AnyPreventiveRule` instead
+- stop using `BaseEntity`, use a concrete union instead
+
 ### Removed
 
+- removed deprecated `BulkInsertResponse` as alias for `IdentifiersResponse`
+- removed unused module export of `mex.common.models.generate_entity_filter_schema`
+- removed unused module export of `mex.common.models.generate_mapping_schema`
+- drop export `models.ExtractedPrimarySourceIdentifier`, import from `types` instead
+- drop export `models.MergedPrimarySourceIdentifier`, import from `types` instead
+
 ### Fixed
 
 ### Security

mex/common/backend_api/connector.py

@@ -8,6 +8,7 @@
     IdentifiersResponse,
     MergedItemsResponse,
     MergedModelTypeAdapter,
+    PreviewItemsResponse,
     RuleSetResponseTypeAdapter,
 )
 from mex.common.connector import HTTPConnector
@@ -185,6 +186,40 @@ def preview_merged_item(
         )
         return MergedModelTypeAdapter.validate_python(response)
 
+    def fetch_preview_items(
+        self,
+        query_string: str | None,
+        entity_type: list[str] | None,
+        skip: int,
+        limit: int,
+    ) -> PreviewItemsResponse:
+        """Fetch merged item previews that match the given set of filters.
+
+        Args:
+            query_string: Full-text search query
+            entity_type: The item's entityType
+            skip: How many items to skip for pagination
+            limit: How many items to return in one page
+
+        Raises:
+            HTTPError: If search was not accepted, crashes or times out
+
+        Returns:
+            One page of preview items and the total count that was matched
+        """
+        # Note: this is forward-compat for MX-1649, backend might not support this yet!
+        response = self.request(
+            method="GET",
+            endpoint="preview-item",
+            params={
+                "q": query_string,
+                "entityType": entity_type,
+                "skip": str(skip),
+                "limit": str(limit),
+            },
+        )
+        return PreviewItemsResponse.model_validate(response)
+
     def get_rule_set(
         self,
         stable_target_id: str,

mex/common/backend_api/models.py

@@ -5,6 +5,7 @@
 from mex.common.models import (
     AnyExtractedModel,
     AnyMergedModel,
+    AnyPreviewModel,
     AnyRuleSetResponse,
     BaseModel,
 )
@@ -31,6 +32,13 @@ class MergedItemsResponse(BaseModel):
     total: int
 
 
+class PreviewItemsResponse(BaseModel):
+    """Response model for a list of preview items including a total count."""
+
+    items: list[AnyPreviewModel]
+    total: int
+
+
 class IdentifiersResponse(BaseModel):
     """Response models for a list of identifiers."""
 
@@ -43,4 +51,3 @@ class IdentifiersResponse(BaseModel):
 RuleSetResponseTypeAdapter: TypeAdapter[AnyRuleSetResponse] = TypeAdapter(
     Annotated[AnyRuleSetResponse, Field(discriminator="entityType")]
 )
-BulkInsertResponse = IdentifiersResponse  # deprecated

mex/common/fields.py

@@ -3,6 +3,7 @@
     EXTRACTED_MODEL_CLASSES_BY_NAME,
     MERGED_MODEL_CLASSES_BY_NAME,
     PREVENTIVE_MODEL_CLASSES_BY_NAME,
+    PREVIEW_MODEL_CLASSES_BY_NAME,
     SUBTRACTIVE_MODEL_CLASSES_BY_NAME,
 )
 from mex.common.types import (
@@ -21,6 +22,7 @@
     **ADDITIVE_MODEL_CLASSES_BY_NAME,
     **EXTRACTED_MODEL_CLASSES_BY_NAME,
     **MERGED_MODEL_CLASSES_BY_NAME,
+    **PREVIEW_MODEL_CLASSES_BY_NAME,
     **PREVENTIVE_MODEL_CLASSES_BY_NAME,
     **SUBTRACTIVE_MODEL_CLASSES_BY_NAME,
 }

mex/common/models/__init__.py

@@ -21,6 +21,7 @@
 - `BaseT` defines all fields according to `mex-model` except for provenance fields
 - `ExtractedT` defines an automatically extracted metadata item including provenance
 - `MergedT` defines the result of merging extracted items and rules into a single item
+- `PreviewT` defines a preview of a merged item without enforcing cardinality validation
 
 - `AdditiveT` defines a rule to add values to specific fields of a merged item
 - `SubtractiveT` defines a rule to subtract (or block) specific values for specific
@@ -54,6 +55,7 @@
 - BaseT: _OptionalLists, _RequiredLists, _OptionalValues, _RequiredValues
 - ExtractedT: BaseT, ExtractedData
 - MergedT: BaseT, MergedItem
+- PreviewT: _OptionalLists, _SparseLists, _OptionalValues, _SparseValues, PreviewItem
 
 - AdditiveT: _OptionalLists, _SparseLists, _OptionalValues, _SparseValues, AdditiveRule
 - SubtractiveT: _OptionalLists, _SparseLists, _VariadicValues, SubtractiveRule
@@ -78,6 +80,7 @@
     ExtractedAccessPlatform,
     MergedAccessPlatform,
     PreventiveAccessPlatform,
+    PreviewAccessPlatform,
     SubtractiveAccessPlatform,
 )
 from mex.common.models.activity import (
@@ -88,6 +91,7 @@
     ExtractedActivity,
     MergedActivity,
     PreventiveActivity,
+    PreviewActivity,
     SubtractiveActivity,
 )
 from mex.common.models.base.extracted_data import ExtractedData
@@ -105,6 +109,7 @@
     ExtractedBibliographicResource,
     MergedBibliographicResource,
     PreventiveBibliographicResource,
+    PreviewBibliographicResource,
     SubtractiveBibliographicResource,
 )
 from mex.common.models.consent import (
@@ -115,6 +120,7 @@
     ExtractedConsent,
     MergedConsent,
     PreventiveConsent,
+    PreviewConsent,
     SubtractiveConsent,
 )
 from mex.common.models.contact_point import (
@@ -125,6 +131,7 @@
     ExtractedContactPoint,
     MergedContactPoint,
     PreventiveContactPoint,
+    PreviewContactPoint,
     SubtractiveContactPoint,
 )
 from mex.common.models.distribution import (
@@ -135,6 +142,7 @@
     ExtractedDistribution,
     MergedDistribution,
     PreventiveDistribution,
+    PreviewDistribution,
     SubtractiveDistribution,
 )
 from mex.common.models.organization import (
@@ -145,6 +153,7 @@
     OrganizationRuleSetRequest,
     OrganizationRuleSetResponse,
     PreventiveOrganization,
+    PreviewOrganization,
     SubtractiveOrganization,
 )
 from mex.common.models.organizational_unit import (
@@ -155,6 +164,7 @@
     OrganizationalUnitRuleSetRequest,
     OrganizationalUnitRuleSetResponse,
     PreventiveOrganizationalUnit,
+    PreviewOrganizationalUnit,
     SubtractiveOrganizationalUnit,
 )
 from mex.common.models.person import (
@@ -165,6 +175,7 @@
     PersonRuleSetRequest,
     PersonRuleSetResponse,
     PreventivePerson,
+    PreviewPerson,
     SubtractivePerson,
 )
 from mex.common.models.primary_source import (
@@ -173,6 +184,7 @@
     ExtractedPrimarySource,
     MergedPrimarySource,
     PreventivePrimarySource,
+    PreviewPrimarySource,
     PrimarySourceRuleSetRequest,
     PrimarySourceRuleSetResponse,
     SubtractivePrimarySource,
@@ -183,6 +195,7 @@
     ExtractedResource,
     MergedResource,
     PreventiveResource,
+    PreviewResource,
     ResourceRuleSetRequest,
     ResourceRuleSetResponse,
     SubtractiveResource,
@@ -193,6 +206,7 @@
     ExtractedVariable,
     MergedVariable,
     PreventiveVariable,
+    PreviewVariable,
     SubtractiveVariable,
     VariableRuleSetRequest,
     VariableRuleSetResponse,
@@ -203,6 +217,7 @@
     ExtractedVariableGroup,
     MergedVariableGroup,
     PreventiveVariableGroup,
+    PreviewVariableGroup,
     SubtractiveVariableGroup,
     VariableGroupRuleSetRequest,
     VariableGroupRuleSetResponse,
@@ -294,7 +309,6 @@
     "ExtractedOrganizationalUnit",
     "ExtractedPerson",
     "ExtractedPrimarySource",
-    "ExtractedPrimarySourceIdentifier",
     "ExtractedResource",
     "ExtractedVariable",
     "ExtractedVariableGroup",
@@ -310,7 +324,6 @@
     "MergedOrganizationalUnit",
     "MergedPerson",
     "MergedPrimarySource",
-    "MergedPrimarySourceIdentifier",
     "MergedResource",
     "MergedVariable",
     "MergedVariableGroup",
@@ -334,6 +347,19 @@
     "PreventiveRule",
     "PreventiveVariable",
     "PreventiveVariableGroup",
+    "PreviewAccessPlatform",
+    "PreviewActivity",
+    "PreviewBibliographicResource",
+    "PreviewConsent",
+    "PreviewContactPoint",
+    "PreviewDistribution",
+    "PreviewOrganization",
+    "PreviewOrganizationalUnit",
+    "PreviewPerson",
+    "PreviewPrimarySource",
+    "PreviewResource",
+    "PreviewVariable",
+    "PreviewVariableGroup",
     "PrimarySourceRuleSetRequest",
     "PrimarySourceRuleSetResponse",
     "ResourceRuleSetRequest",
@@ -356,8 +382,6 @@
     "VariableGroupRuleSetResponse",
     "VariableRuleSetRequest",
     "VariableRuleSetResponse",
-    "generate_entity_filter_schema",
-    "generate_mapping_schema",
 )
 
 MEX_PRIMARY_SOURCE_IDENTIFIER = ExtractedPrimarySourceIdentifier("00000000000001")
@@ -426,6 +450,28 @@
     cls.__name__: cls for cls in MERGED_MODEL_CLASSES
 }
 
+AnyPreviewModel = (
+    PreviewAccessPlatform
+    | PreviewActivity
+    | PreviewBibliographicResource
+    | PreviewConsent
+    | PreviewContactPoint
+    | PreviewDistribution
+    | PreviewOrganization
+    | PreviewOrganizationalUnit
+    | PreviewPerson
+    | PreviewPrimarySource
+    | PreviewResource
+    | PreviewVariable
+    | PreviewVariableGroup
+)
+PREVIEW_MODEL_CLASSES: Final[list[type[AnyPreviewModel]]] = list(
+    get_args(AnyPreviewModel)
+)
+PREVIEW_MODEL_CLASSES_BY_NAME: Final[dict[str, type[AnyPreviewModel]]] = {
+    cls.__name__: cls for cls in PREVIEW_MODEL_CLASSES
+}
+
 AnyAdditiveModel = (
     AdditiveAccessPlatform
     | AdditiveActivity

mex/common/models/access_platform.py

@@ -7,6 +7,7 @@
 from mex.common.models.base.extracted_data import ExtractedData
 from mex.common.models.base.merged_item import MergedItem
 from mex.common.models.base.model import BaseModel
+from mex.common.models.base.preview_item import PreviewItem
 from mex.common.models.base.rules import (
     AdditiveRule,
     PreventiveRule,
@@ -85,25 +86,36 @@ class ExtractedAccessPlatform(BaseAccessPlatform, ExtractedData):
     @computed_field  # type: ignore[prop-decorator]
     @property
     def identifier(self) -> ExtractedAccessPlatformIdentifier:
-        """Return the computed identifier for this extracted data item."""
+        """Return the computed identifier for this extracted item."""
         return self._get_identifier(ExtractedAccessPlatformIdentifier)
 
     @computed_field  # type: ignore[prop-decorator]
     @property
     def stableTargetId(self) -> MergedAccessPlatformIdentifier:  # noqa: N802
-        """Return the computed stableTargetId for this extracted data item."""
+        """Return the computed stableTargetId for this extracted item."""
         return self._get_stable_target_id(MergedAccessPlatformIdentifier)
 
 
 class MergedAccessPlatform(BaseAccessPlatform, MergedItem):
-    """The result of merging all extracted data and rules for an access platform."""
+    """The result of merging all extracted items and rules for an access platform."""
 
     entityType: Annotated[
         Literal["MergedAccessPlatform"], Field(alias="$type", frozen=True)
     ] = "MergedAccessPlatform"
     identifier: Annotated[MergedAccessPlatformIdentifier, Field(frozen=True)]
 
 
+class PreviewAccessPlatform(
+    _OptionalLists, _OptionalValues, _SparseValues, PreviewItem
+):
+    """Preview for merging all extracted items and rules for an access platform."""
+
+    entityType: Annotated[
+        Literal["PreviewAccessPlatform"], Field(alias="$type", frozen=True)
+    ] = "PreviewAccessPlatform"
+    identifier: Annotated[MergedAccessPlatformIdentifier, Field(frozen=True)]
+
+
 class AdditiveAccessPlatform(
     _OptionalLists, _OptionalValues, _SparseValues, AdditiveRule
 ):