Pydantic v2 support

.github/workflows/checks.yml

@@ -11,13 +11,16 @@ jobs:
         include:
         - python-version: 3.13
           env:
-            TOXENV: pylint
+            TOXENV: typing
         - python-version: 3.13
           env:
-            TOXENV: typing
+            TOXENV: docs
         - python-version: 3.13
           env:
             TOXENV: twinecheck
+        - python-version: 3.13
+          env:
+            TOXENV: pylint
 
     steps:
     - uses: actions/checkout@v4

.github/workflows/tests.yml

@@ -9,7 +9,55 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "pypy3.10"]
+        include:
+        - python-version: "3.9"
+          env:
+            TOXENV: min-attrs
+        - python-version: "3.9"
+          env:
+            TOXENV: min-pydantic
+        - python-version: "3.9"
+          env:
+            TOXENV: min-scrapy
+        - python-version: "3.9"
+          env:
+            TOXENV: min-extra
+        - python-version: "3.9"
+          env:
+            TOXENV: py
+        - python-version: "3.10"
+          env:
+            TOXENV: py
+        - python-version: "pypy3.10"
+          env:
+            TOXENV: py
+        - python-version: "3.11"
+          env:
+            TOXENV: py
+        - python-version: "3.12"
+          env:
+            TOXENV: py
+        - python-version: "3.13"
+          env:
+            TOXENV: py
+        - python-version: "3.13"
+          env:
+            TOXENV: attrs
+        - python-version: "3.13"
+          env:
+            TOXENV: pydantic1
+        - python-version: "3.13"
+          env:
+            TOXENV: pydantic
+        - python-version: "3.13"
+          env:
+            TOXENV: scrapy
+        - python-version: "3.13"
+          env:
+            TOXENV: extra
+        - python-version: "3.13"
+          env:
+            TOXENV: extra-pydantic1
 
     steps:
     - uses: actions/checkout@v4
@@ -23,31 +71,8 @@ jobs:
       run: pip install tox
 
     - name: Run tests
-      run: tox -e py
-
-    - name: Upload coverage report
-      run: |
-        curl -Os https://uploader.codecov.io/latest/linux/codecov
-        chmod +x codecov
-        ./codecov
-
-  tests-other:
-    name: "Test: py39-scrapy22, Ubuntu"
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
-
-    - name: Install tox
-      run: pip install tox
-
-    - name: Run tests
-      run: tox -e py39-scrapy22
+      env: ${{ matrix.env }}
+      run: tox
 
     - name: Upload coverage report
       run: |

.gitignore

@@ -7,3 +7,4 @@
 .coverage
 htmlcov/
 coverage.xml
+/dist/

README.md

@@ -15,7 +15,7 @@ Currently supported types are:
 * [`dict`](https://docs.python.org/3/library/stdtypes.html#dict)
 * [`dataclass`](https://docs.python.org/3/library/dataclasses.html)-based classes
 * [`attrs`](https://www.attrs.org)-based classes
-* [`pydantic`](https://pydantic-docs.helpmanual.io/)-based classes (`pydantic>=2` not yet supported)
+* [`pydantic`](https://pydantic-docs.helpmanual.io/)-based classes
 
 Additionally, interaction with arbitrary types is supported, by implementing
 a pre-defined interface (see [extending `itemadapter`](#extending-itemadapter)).
@@ -24,11 +24,14 @@ a pre-defined interface (see [extending `itemadapter`](#extending-itemadapter)).
 
 ## Requirements
 
-* Python 3.9+, either the CPython implementation (default) or the PyPy implementation
-* [`scrapy`](https://scrapy.org/): optional, needed to interact with `scrapy` items
-* [`attrs`](https://pypi.org/project/attrs/): optional, needed to interact with `attrs`-based items
-* [`pydantic`](https://pypi.org/project/pydantic/): optional, needed to interact with
-  `pydantic`-based items (`pydantic>=2` not yet supported)
+* Python 3.9+, either the CPython implementation (default) or the PyPy
+  implementation
+* [`scrapy`](https://scrapy.org/) 2.2+: optional, needed to interact with
+  `scrapy` items
+* [`attrs`](https://pypi.org/project/attrs/) 18.1.0+: optional, needed to
+  interact with `attrs`-based items
+* [`pydantic`](https://pypi.org/project/pydantic/) 1.8+: optional, needed to
+  interact with `pydantic`-based items
 
 ---
 
@@ -40,6 +43,20 @@ a pre-defined interface (see [extending `itemadapter`](#extending-itemadapter)).
 pip install itemadapter
 ```
 
+For `attrs`, `pydantic` and `scrapy` support, install the corresponding extra
+to ensure that a supported version of the corresponding dependencies is
+installed. For example:
+
+```
+pip install itemadapter[scrapy]
+```
+
+Mind that you can install multiple extras as needed. For example:
+
+```
+pip install itemadapter[attrs,pydantic,scrapy]
+```
+
 ---
 
 ## License
@@ -306,9 +323,9 @@ mappingproxy({'serializer': <class 'int'>, 'limit': 100})
 ...
 >>> adapter = ItemAdapter(InventoryItem(name="foo", value=10))
 >>> adapter.get_field_meta("name")
-mappingproxy({'serializer': <class 'str'>})
+mappingproxy({'default': PydanticUndefined, 'json_schema_extra': {'serializer': <class 'str'>}, 'repr': True})
 >>> adapter.get_field_meta("value")
-mappingproxy({'serializer': <class 'int'>, 'limit': 100})
+mappingproxy({'default': PydanticUndefined, 'json_schema_extra': {'serializer': <class 'int'>, 'limit': 100}, 'repr': True})
 >>>
 ```
 
@@ -361,19 +378,23 @@ so all methods from the `MutableMapping` interface must be implemented as well.
 
 ### Registering an adapter
 
-Add your custom adapter class to the `itemadapter.adapter.ItemAdapter.ADAPTER_CLASSES`
-class attribute in order to handle custom item classes:
+Add your custom adapter class to the
+`itemadapter.adapter.ItemAdapter.ADAPTER_CLASSES` class attribute in order to
+handle custom item classes.
 
 **Example**
+```
+pip install zyte-common-items
+```
 ```python
 >>> from itemadapter.adapter import ItemAdapter
->>> from tests.test_interface import BaseFakeItemAdapter, FakeItemClass
+>>> from zyte_common_items import Item, ZyteItemAdapter
 >>>
->>> ItemAdapter.ADAPTER_CLASSES.appendleft(BaseFakeItemAdapter)
->>> item = FakeItemClass()
+>>> ItemAdapter.ADAPTER_CLASSES.appendleft(ZyteItemAdapter)
+>>> item = Item()
 >>> adapter = ItemAdapter(item)
 >>> adapter
-<ItemAdapter for FakeItemClass()>
+<ItemAdapter for Item()>
 >>>
 ```
 

itemadapter/_imports.py

@@ -1,32 +1,49 @@
 # attempt the following imports only once,
 # to be imported from itemadapter's submodules
 
+from typing import Any
+
 _scrapy_item_classes: tuple
+scrapy: Any
 
 try:
     import scrapy  # pylint: disable=W0611 (unused-import)
 except ImportError:
-    scrapy = None  # type: ignore[assignment]
     _scrapy_item_classes = ()
+    scrapy = None
 else:
     try:
         # handle deprecated base classes
         _base_item_cls = getattr(
             scrapy.item,
             "_BaseItem",
-            scrapy.item.BaseItem,  # type: ignore[attr-defined]
+            scrapy.item.BaseItem,
         )
     except AttributeError:
         _scrapy_item_classes = (scrapy.item.Item,)
     else:
         _scrapy_item_classes = (scrapy.item.Item, _base_item_cls)
 
+attr: Any
 try:
     import attr  # pylint: disable=W0611 (unused-import)
 except ImportError:
-    attr = None  # type: ignore[assignment]
+    attr = None
+
+pydantic_v1: Any = None
+pydantic: Any = None
 
 try:
     import pydantic  # pylint: disable=W0611 (unused-import)
-except ImportError:
-    pydantic = None  # type: ignore[assignment]
+except ImportError:  # No pydantic
+    pass
+else:
+    try:
+        import pydantic.v1 as pydantic_v1  # pylint: disable=W0611 (unused-import)
+    except ImportError:  # Pydantic <1.10.17
+        pydantic_v1 = pydantic
+        pydantic = None  # pylint: disable=C0103 (invalid-name)
+    else:  # Pydantic 1.10.17+
+        if not hasattr(pydantic.BaseModel, "model_fields"):  # Pydantic <2
+            pydantic_v1 = pydantic
+            pydantic = None  # pylint: disable=C0103 (invalid-name)

itemadapter/adapter.py

@@ -8,8 +8,10 @@
 from itemadapter._imports import _scrapy_item_classes, attr
 from itemadapter.utils import (
     _get_pydantic_model_metadata,
+    _get_pydantic_v1_model_metadata,
     _is_attrs_class,
     _is_pydantic_model,
+    _is_pydantic_v1_model,
 )
 
 __all__ = [
@@ -167,47 +169,83 @@ class PydanticAdapter(AdapterInterface):
 
     @classmethod
     def is_item_class(cls, item_class: type) -> bool:
-        return _is_pydantic_model(item_class)
+        return _is_pydantic_model(item_class) or _is_pydantic_v1_model(item_class)
 
     @classmethod
     def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType:
         try:
-            return _get_pydantic_model_metadata(item_class, field_name)
+            try:
+                return _get_pydantic_model_metadata(item_class, field_name)
+            except AttributeError:
+                return _get_pydantic_v1_model_metadata(item_class, field_name)
         except KeyError:
             raise KeyError(f"{item_class.__name__} does not support field: {field_name}")
 
     @classmethod
     def get_field_names_from_class(cls, item_class: type) -> Optional[List[str]]:
-        return list(item_class.__fields__.keys())  # type: ignore[attr-defined]
+        try:
+            return list(item_class.model_fields.keys())  # type: ignore[attr-defined]
+        except AttributeError:
+            return list(item_class.__fields__.keys())  # type: ignore[attr-defined]
 
     def field_names(self) -> KeysView:
-        return KeysView(self.item.__fields__)
+        try:
+            return KeysView(self.item.model_fields)
+        except AttributeError:
+            return KeysView(self.item.__fields__)
 
     def __getitem__(self, field_name: str) -> Any:
-        if field_name in self.item.__fields__:
-            return getattr(self.item, field_name)
+        try:
+            self.item.model_fields
+        except AttributeError:
+            if field_name in self.item.__fields__:
+                return getattr(self.item, field_name)
+        else:
+            if field_name in self.item.model_fields:
+                return getattr(self.item, field_name)
         raise KeyError(field_name)
 
     def __setitem__(self, field_name: str, value: Any) -> None:
-        if field_name in self.item.__fields__:
-            setattr(self.item, field_name, value)
+        try:
+            self.item.model_fields
+        except AttributeError:
+            if field_name in self.item.__fields__:
+                setattr(self.item, field_name, value)
+                return
         else:
-            raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}")
+            if field_name in self.item.model_fields:
+                setattr(self.item, field_name, value)
+                return
+        raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}")
 
     def __delitem__(self, field_name: str) -> None:
-        if field_name in self.item.__fields__:
-            try:
-                if hasattr(self.item, field_name):
-                    delattr(self.item, field_name)
-                else:
+        try:
+            self.item.model_fields
+        except AttributeError:
+            if field_name in self.item.__fields__:
+                try:
+                    if hasattr(self.item, field_name):
+                        delattr(self.item, field_name)
+                        return
                     raise AttributeError
-            except AttributeError:
-                raise KeyError(field_name)
+                except AttributeError:
+                    raise KeyError(field_name)
         else:
-            raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}")
+            if field_name in self.item.model_fields:
+                try:
+                    if hasattr(self.item, field_name):
+                        delattr(self.item, field_name)
+                        return
+                    raise AttributeError
+                except AttributeError:
+                    raise KeyError(field_name)
+        raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}")
 
     def __iter__(self) -> Iterator:
-        return iter(attr for attr in self.item.__fields__ if hasattr(self.item, attr))
+        try:
+            return iter(attr for attr in self.item.model_fields if hasattr(self.item, attr))
+        except AttributeError:
+            return iter(attr for attr in self.item.__fields__ if hasattr(self.item, attr))
 
     def __len__(self) -> int:
         return len(list(iter(self)))