Merge branch 'main' of https://github.com/robert-koch-institut/mex-co…

…mmon into feature/mx-1660-ldap-search-endpoint

.cruft.json

@@ -1,6 +1,6 @@
 {
   "checkout": null,
-  "commit": "bae86448088992cdbd3acde751ad7aa19a79d71b",
+  "commit": "a4f25ab84f9e485ad77eb03663a9cf486f7a5826",
   "context": {
     "cookiecutter": {
       "project_name": "common",

.github/workflows/cookiecutter.yml

@@ -48,20 +48,13 @@ jobs:
 
       - name: Configure git
         env:
+          MEX_BOT_EMAIL: ${{ vars.MEX_BOT_EMAIL }}
+          MEX_BOT_USER: ${{ vars.MEX_BOT_USER }}
           SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
           SIGNING_PUB: ${{ secrets.SIGNING_PUB }}
         run: |
           eval "$(ssh-agent -s)"
-          install --directory ~/.ssh --mode 700
-          base64 -d <<< '${{ secrets.SIGNING_KEY }}' > ~/.ssh/mex
-          base64 -d <<< '${{ secrets.SIGNING_PUB }}' > ~/.ssh/mex.pub
-          chmod 600 ~/.ssh/*
-          ssh-add ~/.ssh/mex
-          git config --local user.email ${{ vars.MEX_BOT_EMAIL }}
-          git config --local user.name ${{ vars.MEX_BOT_USER }}
-          git config --local gpg.format ssh
-          git config --local user.signingkey ~/.ssh/mex.pub
-          git config --local commit.gpgsign true
+          pdm setup-commit-signing
 
       - name: Update template
         env:

.github/workflows/release.yml

@@ -59,20 +59,13 @@ jobs:
 
       - name: Configure git
         env:
+          MEX_BOT_EMAIL: ${{ vars.MEX_BOT_EMAIL }}
+          MEX_BOT_USER: ${{ vars.MEX_BOT_USER }}
           SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
           SIGNING_PUB: ${{ secrets.SIGNING_PUB }}
         run: |
           eval "$(ssh-agent -s)"
-          install --directory ~/.ssh --mode 700
-          base64 -d <<< '${{ secrets.SIGNING_KEY }}' > ~/.ssh/mex
-          base64 -d <<< '${{ secrets.SIGNING_PUB }}' > ~/.ssh/mex.pub
-          chmod 600 ~/.ssh/*
-          ssh-add ~/.ssh/mex
-          git config --local user.email ${{ vars.MEX_BOT_EMAIL }}
-          git config --local user.name ${{ vars.MEX_BOT_USER }}
-          git config --local gpg.format ssh
-          git config --local user.signingkey ~/.ssh/mex.pub
-          git config --local commit.gpgsign true
+          pdm setup-commit-signing
 
       - name: Release new version
         id: release

.github/workflows/renovatebot.yml

@@ -22,7 +22,7 @@ jobs:
           fetch-depth: 1
 
       - name: Run renovatebot
-        uses: renovatebot/github-action@v40.2.10
+        uses: renovatebot/github-action@v41.0.5
         env:
           RENOVATE_GIT_PRIVATE_KEY: ${{ secrets.GPG_SIGNING_KEY }}
           RENOVATE_REPOSITORIES: "robert-koch-institut/mex-common"

.gitignore

@@ -46,8 +46,9 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
+test*.jpeg
+test*.png
 *.cover
-*.jpeg
 *.py,cover
 .hypothesis/
 .pytest_cache/

.pre-commit-config.yaml

@@ -3,13 +3,13 @@ default_language_version:
   python: python3.11
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.4
+    rev: v0.8.0
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: pretty-format-json
         name: json
@@ -25,7 +25,7 @@ repos:
       - id: fix-byte-order-marker
         name: byte-order
   - repo: https://github.com/pdm-project/pdm
-    rev: 2.17.1
+    rev: 2.20.1
     hooks:
       - id: pdm-lock-check
         name: pdm

CHANGELOG.md

@@ -20,11 +20,129 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 
+## [0.43.0] - 2024-12-10
+
+### Added
+
+- add preview models for merged items without cardinality validation
+- BREAKING: preview models are now part of all `mex.common.fields` lookups
+- add `BackendApiConnector.fetch_preview_items` for fetching previews
+
+### Deprecated
+
+- stop using `ExtractedData`, use `AnyExtractedModel` instead
+- stop using `MergedItem`, use `AnyMergedModel` instead
+- stop using `AdditiveRule`, use `AnyAdditiveRule` instead
+- stop using `SubtractiveRule`, use `AnySubtractiveRule` instead
+- stop using `PreventiveRule`, use `AnyPreventiveRule` instead
+- stop using `BaseEntity`, use a concrete union instead
+
+### Removed
+
+- removed deprecated `BulkInsertResponse` as alias for `IdentifiersResponse`
+- removed unused module export of `mex.common.models.generate_entity_filter_schema`
+- removed unused module export of `mex.common.models.generate_mapping_schema`
+- drop export `models.ExtractedPrimarySourceIdentifier`, import from `types` instead
+- drop export `models.MergedPrimarySourceIdentifier`, import from `types` instead
+
+## [0.42.0] - 2024-12-02
+
+### Added
+
+- add vocabulary and temporal unions and lookups to `mex.common.types`
+- add `mex.common.fields` with field type by class name lookups
+
+### Changes
+
+- wikidata helper now optionally accepts wikidata primary source
+- set default empty rules to all of the rule-set models
+- pin pydantic to sub 2.10 (for now) because of breaking changes
+
+### Fixed
+
+- switch HTTP method for preview endpoint to `POST`
+- add optional values to variadic values for distribution models
+- make `endpointDescription` optional for variadic access platform models
+
+## [0.41.0] - 2024-11-18
+
+### Added
+
+- organigram extraction checks for duplicate emails/labels in different organigram units
+
+### Changes
+
+- upgrade mex-model dependency to version 3.2
+
+## [0.40.0] - 2024-10-28
+
+### Changes
+
+- upgrade mex-model dependency to version 3.1
+
+### Fixed
+
+- fix typo in `repositoryURL` of bibliographic resources
+- make identifier and stableTargetId of ExtractedBibliographicResource computed fields
+
+## [0.39.0] - 2024-10-28
+
+### Added
+
+- added new consent and bibliography reference models and vocabs
+- added doi field to resource models
+- helper function for primary source look up
+
+### Changes
+
+- upgrade mex-model dependency to version 3
+- make ruff linter config opt-out, instead of opt-in
+- make instances of extracted data hashable
+- BREAKING: Wikidata convenience function refactored and renamed to 'helper'
+- wikidata helper function split between mex-common and mex-extractors
+- code de-duplication: fixture extracted_primary_sources uses function-part of helper
+- split up YearMonth and Year temporal types and improved patterns
+- applied all changes to model fields according to model v3
+- update LOINC pattern
+
+### Fixed
+
+- fix temporal entity schemas
+
+## [0.38.0] - 2024-10-11
+
+### Added
+
+- add pattern constants for vocabs, emails, urls and ids to types module
+- add regex pattern to json schema of identifier fields
+- automatically add examples and useScheme to json schema of enum fields
+
+### Changes
+
+- BREAKING: use `identifier` instead of `stableTargetId` to get merged item from backend
+- ensure identifier unions are typed to generic `Identifier` instead of the first match
+  to signal that we don't actually know which of the union types is correct
+- unify pydantic schema configuration for all types
+- consistently parse emails, identifiers and temporals in models to their type, not str
+- consistently serialize emails, ids and temporals in models to str, not their type
+- make instances of Link type hashable, to harmonize them with Text models
+
+### Removed
+
+- drop manual examples from enum fields, because they are autogenerated now
+- BREAKING: remove `MEX_ID_PATTERN` from types, in favor of `IDENTIFIER_PATTERN`
+- BREAKING: make public `MEX_ID_ALPHABET` constant from identifier module private
+- BREAKING: remove `__str__` methods from Text and Link classes
+- BREAKING: drop support for parsing UUIDs as Identifiers, this was unused
+- BREAKING: drop support for parsing Links from markdown syntax, this was unused
+- BREAKING: remove pydantic1-style `validate` methods from all type models
+- BREAKING: `BackendApiConnector.post_models` in favor of `post_extracted_items`
+
 ## [0.37.0] - 2024-10-01
 
 ### Added
-- added methods for extracting persons by name or ID from ldap
 
+- added methods for extracting persons by name or ID from ldap
 - `contains_only_types` to check if fields are annotated as desired
 - `group_fields_by_class_name` utility to simplify filtered model/field lookups
 - new parameters to `get_inner_types` to customize what to unpack

README.md

@@ -9,7 +9,7 @@ Common library for MEx python projects.
 [![open-code](https://github.com/robert-koch-institut/mex-common/actions/workflows/open-code.yml/badge.svg)](https://gitlab.opencode.de/robert-koch-institut/mex/mex-common)
 [![testing](https://github.com/robert-koch-institut/mex-common/actions/workflows/testing.yml/badge.svg)](https://github.com/robert-koch-institut/mex-common/actions/workflows/testing.yml)
 
-## project
+## Project
 
 The Metadata Exchange (MEx) project is committed to improve the retrieval of RKI
 research data and projects. How? By focusing on metadata: instead of providing the
@@ -39,28 +39,29 @@ data Findable, Accessible, Interoperable and Reusable.
 **Contact** \
 For more information, please feel free to email us at [mex@rki.de](mailto:mex@rki.de).
 
-### Publisher of this document
+### Publisher
+
 **Robert Koch-Institut** \
 Nordufer 20 \
 13353 Berlin \
 Germany
 
-## package
+## Package
 
 The `mex-common` library is a software development toolkit that is used by multiple
 components within the MEx project. It contains utilities for building pipelines like a
 common commandline interface, logging and configuration setup. It also provides common
 auxiliary connectors that can be used to fetch data from external services and a
 re-usable implementation of the MEx metadata schema as pydantic models.
 
-## license
+## License
 
 This package is licensed under the [MIT license](/LICENSE). All other software
 components of the MEx project are open-sourced under the same license as well.
 
-## development
+## Development
 
-### installation
+### Installation
 
 - on unix, consider using pyenv https://github.com/pyenv/pyenv
   - get pyenv `curl https://pyenv.run | bash`
@@ -73,21 +74,21 @@ components of the MEx project are open-sourced under the same license as well.
   - switch version `pyenv global 3.11`
   - run `.\mex.bat install`
 
-### linting and testing
+### Linting and testing
 
 - run all linters with `pdm lint`
 - run only unit tests with `pdm unit`
 - run unit and integration tests with `pdm test`
 
-### updating dependencies
+### Updating dependencies
 
 - update boilerplate files with `cruft update`
 - update global requirements in `requirements.txt` manually
 - update git hooks with `pre-commit autoupdate`
 - update package dependencies using `pdm update-all`
 - update github actions in `.github/workflows/*.yml` manually
 
-### creating release
+### Creating release
 
 - run `pdm release RULE` to release a new version where RULE determines which part of
   the version to update and is one of `major`, `minor`, `patch`.