From 0b558d500ca8dfeccc23d3b599088022872b4a33 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Aug 2021 19:18:12 -0300 Subject: [PATCH 01/12] ItemAdapter.is_item_class and ItemAdapter.get_field_meta_from_class --- itemadapter/adapter.py | 85 +++++++++++++++++++++++++++++++++++++++-- itemadapter/utils.py | 27 ++----------- tests/test_interface.py | 4 ++ tests/test_utils.py | 4 ++ 4 files changed, 93 insertions(+), 27 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index cdfb279..864fd0e 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -6,6 +6,10 @@ from itemadapter.utils import ( _get_pydantic_model_metadata, + _get_scrapy_item_classes, + _is_attrs_class, + _is_dataclass, + _is_pydantic_model, is_attrs_instance, is_dataclass_instance, is_item, @@ -42,6 +46,15 @@ def is_item(cls, item: Any) -> bool: """Return True if the adapter can handle the given item, False otherwise""" raise NotImplementedError() + @classmethod + def is_item_class(cls, item_class: type) -> bool: + """Return True if the adapter can handle the given item class, False otherwise""" + raise NotImplementedError() + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + return MappingProxyType({}) + def get_field_meta(self, field_name: str) -> MappingProxyType: """Return metadata for the given field name, if available.""" return MappingProxyType({}) @@ -101,6 +114,19 @@ def __init__(self, item: Any) -> None: def is_item(cls, item: Any) -> bool: return is_attrs_instance(item) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return _is_attrs_class(item_class) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + from attr import fields_dict + + try: + return fields_dict(item_class)[field_name].metadata # type: ignore + except KeyError: + raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + class DataclassAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): def __init__(self, item: Any) -> None: @@ -114,18 +140,42 @@ def __init__(self, item: Any) -> None: def is_item(cls, item: Any) -> bool: return is_dataclass_instance(item) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return _is_dataclass(item_class) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + from dataclasses import fields + + for field in fields(item_class): + if field.name == field_name: + return field.metadata # type: ignore + raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + class PydanticAdapter(AdapterInterface): item: Any - def get_field_meta(self, field_name: str) -> MappingProxyType: - return _get_pydantic_model_metadata(type(self.item), field_name) - @classmethod def is_item(cls, item: Any) -> bool: return is_pydantic_instance(item) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return _is_pydantic_model(item_class) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + try: + return _get_pydantic_model_metadata(item_class, field_name) + except KeyError: + raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + + def get_field_meta(self, field_name: str) -> MappingProxyType: + return self.__class__.get_field_meta_from_class(type(self.item), field_name) + def field_names(self) -> KeysView: return KeysView(self.item.__fields__) @@ -182,6 +232,14 @@ class DictAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): def is_item(cls, item: Any) -> bool: return isinstance(item, dict) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return issubclass(item_class, dict) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + return MappingProxyType({}) + def get_field_meta(self, field_name: str) -> MappingProxyType: return MappingProxyType({}) @@ -194,6 +252,14 @@ class ScrapyItemAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): def is_item(cls, item: Any) -> bool: return is_scrapy_item(item) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return issubclass(item_class, _get_scrapy_item_classes()) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + return MappingProxyType(item_class.fields[field_name]) # type: ignore + def get_field_meta(self, field_name: str) -> MappingProxyType: return MappingProxyType(self.item.fields[field_name]) @@ -228,6 +294,19 @@ def __init__(self, item: Any) -> None: def is_item(cls, item: Any) -> bool: return any(adapter_class.is_item(item) for adapter_class in cls.ADAPTER_CLASSES) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return any( + adapter_class.is_item_class(item_class) for adapter_class in cls.ADAPTER_CLASSES + ) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + for adapter_class in cls.ADAPTER_CLASSES: + if adapter_class.is_item_class(item_class): + return adapter_class.get_field_meta_from_class(item_class, field_name) + raise TypeError("%s is not a valid item class" % (item_class,)) + @property def item(self) -> Any: return self.adapter.item diff --git a/itemadapter/utils.py b/itemadapter/utils.py index 37dd19c..342ed39 100644 --- a/itemadapter/utils.py +++ b/itemadapter/utils.py @@ -129,28 +129,7 @@ def get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxy The returned value is an instance of types.MappingProxyType, i.e. a dynamic read-only view of the original mapping, which gets automatically updated if the original mapping changes. """ - if issubclass(item_class, _get_scrapy_item_classes()): - return MappingProxyType(item_class.fields[field_name]) # type: ignore - elif _is_dataclass(item_class): - from dataclasses import fields - - for field in fields(item_class): - if field.name == field_name: - return field.metadata # type: ignore - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) - elif _is_attrs_class(item_class): - from attr import fields_dict - try: - return fields_dict(item_class)[field_name].metadata # type: ignore - except KeyError: - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) - elif _is_pydantic_model(item_class): - try: - return _get_pydantic_model_metadata(item_class, field_name) - except KeyError: - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) - elif issubclass(item_class, dict): - return MappingProxyType({}) - else: - raise TypeError("%s is not a valid item class" % (item_class,)) + from itemadapter.adapter import ItemAdapter + + return ItemAdapter.get_field_meta_from_class(item_class, field_name) diff --git a/tests/test_interface.py b/tests/test_interface.py index 4e4e849..40562c4 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -30,6 +30,10 @@ class BaseFakeItemAdapter(AdapterInterface): def is_item(cls, item: Any) -> bool: return isinstance(item, FakeItemClass) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return issubclass(item_class, FakeItemClass) + def __getitem__(self, field_name: str) -> Any: if field_name in self.item._fields: return self.item._values[field_name] diff --git a/tests/test_utils.py b/tests/test_utils.py index 7a5609f..7df99a5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,4 @@ +import importlib import unittest from unittest import mock from types import MappingProxyType @@ -22,6 +23,9 @@ def mocked_import(name, *args, **kwargs): + """Allow only internal itemadapter imports.""" + if name.split(".")[0] == "itemadapter": + return importlib.__import__(name, *args, **kwargs) raise ImportError(name) From 6cdd92da61a84169aaee819f3695788823618e28 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 10 Aug 2021 08:45:22 -0300 Subject: [PATCH 02/12] f-string formatting --- itemadapter/adapter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 864fd0e..ed6c3eb 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -125,7 +125,7 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping try: return fields_dict(item_class)[field_name].metadata # type: ignore except KeyError: - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + raise KeyError(f"{item_class.__name__} does not support field: {field_name}") class DataclassAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): @@ -151,7 +151,7 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping for field in fields(item_class): if field.name == field_name: return field.metadata # type: ignore - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + raise KeyError(f"{item_class.__name__} does not support field: {field_name}") class PydanticAdapter(AdapterInterface): @@ -171,7 +171,7 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping try: return _get_pydantic_model_metadata(item_class, field_name) except KeyError: - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + raise KeyError(f"{item_class.__name__} does not support field: {field_name}") def get_field_meta(self, field_name: str) -> MappingProxyType: return self.__class__.get_field_meta_from_class(type(self.item), field_name) @@ -305,14 +305,14 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping for adapter_class in cls.ADAPTER_CLASSES: if adapter_class.is_item_class(item_class): return adapter_class.get_field_meta_from_class(item_class, field_name) - raise TypeError("%s is not a valid item class" % (item_class,)) + raise TypeError(f"{item_class} is not a valid item class") @property def item(self) -> Any: return self.adapter.item def __repr__(self) -> str: - values = ", ".join(["%s=%r" % (key, value) for key, value in self.items()]) + values = ", ".join([f"{key}={value!r}" for key, value in self.items()]) return f"" def __getitem__(self, field_name: str) -> Any: From 8467f34f50a3062a69d411a90c966764f7462085 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 10 Aug 2021 12:38:47 -0300 Subject: [PATCH 03/12] Make is_item_class an abstract method, improve test coverage --- itemadapter/adapter.py | 1 + tests/test_interface.py | 39 ++++++++++++++++++++++++++++++++++++--- tests/test_utils.py | 10 ++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index ed6c3eb..6c53a85 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -47,6 +47,7 @@ def is_item(cls, item: Any) -> bool: raise NotImplementedError() @classmethod + @abstractmethod def is_item_class(cls, item_class: type) -> bool: """Return True if the adapter can handle the given item class, False otherwise""" raise NotImplementedError() diff --git a/tests/test_interface.py b/tests/test_interface.py index 40562c4..1264940 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -11,6 +11,8 @@ class AdapterInterfaceTest(unittest.TestCase): def test_interface_class_methods(self): with self.assertRaises(NotImplementedError): AdapterInterface.is_item(object()) + with self.assertRaises(NotImplementedError): + AdapterInterface.is_item_class(object) class FakeItemClass: @@ -67,7 +69,11 @@ def field_names(self) -> KeysView: class MetadataFakeItemAdapter(BaseFakeItemAdapter): - """An adapter that also implements the get_field_meta method.""" + """An adapter that also implements the metadata-related methods.""" + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + return MappingProxyType(item_class._fields.get(field_name) or {}) def get_field_meta(self, field_name: str) -> MappingProxyType: if field_name in self.item._fields: @@ -159,13 +165,26 @@ def test_get_value_keyerror_item_dict(self): with self.assertRaises(KeyError): adapter["name"] - def test_get_field_meta_defined_fields(self): + def test_get_field_meta(self): """Metadata is always empty for the default implementation.""" adapter = ItemAdapter(self.item_class()) self.assertEqual(adapter.get_field_meta("_undefined_"), MappingProxyType({})) self.assertEqual(adapter.get_field_meta("name"), MappingProxyType({})) self.assertEqual(adapter.get_field_meta("value"), MappingProxyType({})) + def test_get_field_meta_from_class(self): + """Metadata is always empty for the default implementation.""" + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "_undefined_"), + MappingProxyType({}), + ) + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "name"), MappingProxyType({}) + ) + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "value"), MappingProxyType({}) + ) + def test_field_names(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) @@ -178,12 +197,26 @@ class MetadataFakeItemAdapterTest(BaseFakeItemAdapterTest): item_class = FakeItemClass adapter_class = MetadataFakeItemAdapter - def test_get_field_meta_defined_fields(self): + def test_get_field_meta(self): adapter = ItemAdapter(self.item_class()) self.assertEqual(adapter.get_field_meta("_undefined_"), MappingProxyType({})) self.assertEqual(adapter.get_field_meta("name"), MappingProxyType({"serializer": str})) self.assertEqual(adapter.get_field_meta("value"), MappingProxyType({"serializer": int})) + def test_get_field_meta_from_class(self): + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "_undefined_"), + MappingProxyType({}), + ) + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "name"), + MappingProxyType({"serializer": str}), + ) + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "value"), + MappingProxyType({"serializer": int}), + ) + class FieldNamesFakeItemAdapterTest(BaseFakeItemAdapterTest): diff --git a/tests/test_utils.py b/tests/test_utils.py index 7df99a5..136e3a7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -11,6 +11,7 @@ is_pydantic_instance, is_scrapy_item, ) +from itemadapter import ItemAdapter from tests import ( AttrsItem, @@ -61,26 +62,35 @@ def test_false(self): self.assertFalse(is_item(ScrapySubclassedItem)) self.assertFalse(is_item(AttrsItem)) self.assertFalse(is_item(PydanticModel)) + self.assertFalse(ItemAdapter.is_item_class(list)) + self.assertFalse(ItemAdapter.is_item_class(int)) + self.assertFalse(ItemAdapter.is_item_class(tuple)) def test_true_dict(self): self.assertTrue(is_item({"a": "dict"})) + self.assertTrue(ItemAdapter.is_item_class(dict)) @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") def test_true_scrapy(self): self.assertTrue(is_item(ScrapyItem())) self.assertTrue(is_item(ScrapySubclassedItem(name="asdf", value=1234))) + self.assertTrue(ItemAdapter.is_item_class(ScrapyItem)) + self.assertTrue(ItemAdapter.is_item_class(ScrapySubclassedItem)) @unittest.skipIf(not DataClassItem, "dataclasses module is not available") def test_true_dataclass(self): self.assertTrue(is_item(DataClassItem(name="asdf", value=1234))) + self.assertTrue(ItemAdapter.is_item_class(DataClassItem)) @unittest.skipIf(not AttrsItem, "attrs module is not available") def test_true_attrs(self): self.assertTrue(is_item(AttrsItem(name="asdf", value=1234))) + self.assertTrue(ItemAdapter.is_item_class(AttrsItem)) @unittest.skipIf(not PydanticModel, "pydantic module is not available") def test_true_pydantic(self): self.assertTrue(is_item(PydanticModel(name="asdf", value=1234))) + self.assertTrue(ItemAdapter.is_item_class(PydanticModel)) class AttrsTestCase(unittest.TestCase): From ca05f4aa1d13d876d3e55c89f2be2bc0226ad114 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 13 Aug 2021 12:06:16 -0300 Subject: [PATCH 04/12] simplify get_field_meta methods --- itemadapter/adapter.py | 24 ++---------------------- tests/test_interface.py | 8 +------- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 6c53a85..1a3dd66 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -58,7 +58,7 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping def get_field_meta(self, field_name: str) -> MappingProxyType: """Return metadata for the given field name, if available.""" - return MappingProxyType({}) + return self.get_field_meta_from_class(self.item.__class__, field_name) def field_names(self) -> KeysView: """Return a dynamic view of the item's field names.""" @@ -174,9 +174,6 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping except KeyError: raise KeyError(f"{item_class.__name__} does not support field: {field_name}") - def get_field_meta(self, field_name: str) -> MappingProxyType: - return self.__class__.get_field_meta_from_class(type(self.item), field_name) - def field_names(self) -> KeysView: return KeysView(self.item.__fields__) @@ -241,9 +238,6 @@ def is_item_class(cls, item_class: type) -> bool: def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: return MappingProxyType({}) - def get_field_meta(self, field_name: str) -> MappingProxyType: - return MappingProxyType({}) - def field_names(self) -> KeysView: return KeysView(self.item) @@ -261,9 +255,6 @@ def is_item_class(cls, item_class: type) -> bool: def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: return MappingProxyType(item_class.fields[field_name]) # type: ignore - def get_field_meta(self, field_name: str) -> MappingProxyType: - return MappingProxyType(self.item.fields[field_name]) - def field_names(self) -> KeysView: return KeysView(self.item.fields) @@ -332,18 +323,7 @@ def __len__(self) -> int: return self.adapter.__len__() def get_field_meta(self, field_name: str) -> MappingProxyType: - """Return a read-only mapping with metadata for the given field name. If there is no metadata - for the field, or the wrapped item does not support field metadata, an empty object is - returned. - - Field metadata is taken from different sources, depending on the item type: - * scrapy.item.Item: corresponding scrapy.item.Field object - * dataclass items: "metadata" attribute for the corresponding field - * attrs items: "metadata" attribute for the corresponding field - - The returned value is an instance of types.MappingProxyType, i.e. a dynamic read-only view - of the original mapping, which gets automatically updated if the original mapping changes. - """ + """Return metadata for the given field name.""" return self.adapter.get_field_meta(field_name) def field_names(self) -> KeysView: diff --git a/tests/test_interface.py b/tests/test_interface.py index 1264940..a64d9e0 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -69,18 +69,12 @@ def field_names(self) -> KeysView: class MetadataFakeItemAdapter(BaseFakeItemAdapter): - """An adapter that also implements the metadata-related methods.""" + """An adapter that also implements metadata-related methods.""" @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: return MappingProxyType(item_class._fields.get(field_name) or {}) - def get_field_meta(self, field_name: str) -> MappingProxyType: - if field_name in self.item._fields: - return MappingProxyType(self.item._fields[field_name]) - else: - return super().get_field_meta(field_name) - class BaseFakeItemAdapterTest(unittest.TestCase): From aac1aa093b0ffdacda283a7ff481b1c930d5cfe3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 13 Aug 2021 12:07:10 -0300 Subject: [PATCH 05/12] Document ItemAdapter.is_item_class --- Changelog.md | 6 ++++++ README.md | 22 +++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/Changelog.md b/Changelog.md index c835ae7..82f793e 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,11 @@ # Changelog +### 0.4.0 (2021-MM-DD) + +Added `ItemAdapter.is_item_class` and `ItemAdapter.get_field_meta_from_class` +([#54](https://github.com/scrapy/itemadapter/pull/54)) + + ### 0.3.0 (2021-07-15) Added suport for `pydantic` models ([#53](https://github.com/scrapy/itemadapter/pull/53)) diff --git a/README.md b/README.md index 552f939..5b82a8c 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,13 @@ Return `True` if any of the registed adapters can handle the item (i.e. if any of them returns `True` for its `is_item` method with `item` as argument), `False` otherwise. -#### `get_field_meta(field_name: str) -> MappingProxyType` +#### class method `is_item_class(item_class: type) -> bool` + +Return `True` if any of the registed adapters can handle the item class +(i.e. if any of them returns `True` for its `is_item_class` method with +`item_class` as argument), `False` otherwise. + +#### class method `get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxyType` Return a [`types.MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) object, which is a read-only mapping with metadata about the given field. If the item class does not @@ -185,12 +191,18 @@ support field metadata, or there is no metadata for the given field, an empty ob The returned value is taken from the following sources, depending on the item type: * [`scrapy.item.Field`](https://docs.scrapy.org/en/latest/topics/items.html#item-fields) - for `scrapy.item.Item`s + for `scrapy.item.Item`s * [`dataclasses.field.metadata`](https://docs.python.org/3/library/dataclasses.html#dataclasses.field) for `dataclass`-based items * [`attr.Attribute.metadata`](https://www.attrs.org/en/stable/examples.html#metadata) for `attrs`-based items - * [`pydantic.fields.FieldInfo`](https://pydantic-docs.helpmanual.io/usage/schema/#field-customisation) for `pydantic`-based items + * [`pydantic.fields.FieldInfo`](https://pydantic-docs.helpmanual.io/usage/schema/#field-customisation) + for `pydantic`-based items + +#### `get_field_meta(field_name: str) -> MappingProxyType` + +Return metadata for the given field, if available. Unless overriden in a custom adapter class, by default +this method calls the adapter's `get_field_meta_from_class` method, passing the stored item's class. #### `field_names() -> collections.abc.KeysView` @@ -312,6 +324,10 @@ so all methods from the `MutableMapping` class must be implemented as well. Return `True` if the adapter can handle the given item, `False` otherwise. Abstract (mandatory). +* _class method `is_item_class(cls, item_class: type) -> bool`_ + + Return `True` if the adapter can handle the given item class, `False` otherwise. Abstract (mandatory). + * _method `get_field_meta(self, field_name: str) -> types.MappingProxyType`_ Return metadata for the given field name, if available. From fb176101cc247e685c8f3fde24cc5c8ffd1fe6ad Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 13 Aug 2021 18:14:00 -0300 Subject: [PATCH 06/12] Document ItemAdapter.get_field_meta_from_class --- README.md | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 5b82a8c..57d9119 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Consider the following type definition: >>> ``` -The `ItemAdapter` object can be treated much like a dictionary: +An `ItemAdapter` object can be treated much like a dictionary: ```python >>> obj = InventoryItem(name='foo', price=20.5, stock=10) @@ -202,7 +202,7 @@ The returned value is taken from the following sources, depending on the item ty #### `get_field_meta(field_name: str) -> MappingProxyType` Return metadata for the given field, if available. Unless overriden in a custom adapter class, by default -this method calls the adapter's `get_field_meta_from_class` method, passing the stored item's class. +this method calls the adapter's `get_field_meta_from_class` method, passing the wrapped item's class. #### `field_names() -> collections.abc.KeysView` @@ -223,10 +223,7 @@ Return `True` if the given object belongs to (at least) one of the supported typ ### function `itemadapter.utils.get_field_meta_from_class(item_class: type, field_name: str) -> types.MappingProxyType` -Given an item class and a field name, return a -[`MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) -object, which is a read-only mapping with metadata about the given field. If the item class does not -support field metadata, or there is no metadata for the given field, an empty object is returned. +Alias for `itemadapter.adapter.ItemAdapter.get_field_meta_from_class` --- @@ -235,10 +232,12 @@ support field metadata, or there is no metadata for the given field, an empty ob `scrapy.item.Item`, `dataclass`, `attrs`, and `pydantic` objects allow the definition of arbitrary field metadata. This can be accessed through a [`MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) -object, which can be retrieved from an item instance with the -`itemadapter.adapter.ItemAdapter.get_field_meta` method, or from an item class -with the `itemadapter.utils.get_field_meta_from_class` function. -The definition procedure depends on the underlying type. +object, which can be retrieved from an item instance with +`itemadapter.adapter.ItemAdapter.get_field_meta`, or from an item class +with the `itemadapter.adapter.ItemAdapter.get_field_meta.get_field_meta_from_class` +method (or its alias `itemadapter.utils.get_field_meta_from_class`). +The source of the data depends on the underlying type (see the docs for +`ItemAdapter.get_field_meta_from_class` above). #### `scrapy.item.Item` objects @@ -318,7 +317,7 @@ _class `itemadapter.adapter.AdapterInterface(item: Any)`_ Abstract Base Class for adapters. An adapter that handles a specific type of item must inherit from this class and implement the abstract methods defined on it. `AdapterInterface` inherits from [`collections.abc.MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping), -so all methods from the `MutableMapping` class must be implemented as well. +so all methods from the `MutableMapping` interface must be implemented as well. * _class method `is_item(cls, item: Any) -> bool`_ @@ -328,13 +327,21 @@ so all methods from the `MutableMapping` class must be implemented as well. Return `True` if the adapter can handle the given item class, `False` otherwise. Abstract (mandatory). -* _method `get_field_meta(self, field_name: str) -> types.MappingProxyType`_ +* _class method `get_field_meta_from_class(cls, item_class: type) -> bool`_ - Return metadata for the given field name, if available. + Return metadata for the given item class and field name, if available. By default, this method returns an empty `MappingProxyType` object. Please supply your own method definition if you want to handle field metadata based on custom logic. See the [section on metadata support](#metadata-support) for additional information. +* _method `get_field_meta(self, field_name: str) -> types.MappingProxyType`_ + + Return metadata for the given field name, if available. It's usually not necessary to + override this method, since the `itemadapter.adapter.AdapterInterface` base class + provides a default implementation that calls `ItemAdapter.get_field_meta_from_class` + with the wrapped item's class as argument. + See the [section on metadata support](#metadata-support) for additional information. + * _method `field_names(self) -> collections.abc.KeysView`_: Return a [dynamic view](https://docs.python.org/3/library/collections.abc.html#collections.abc.KeysView) From 8ca798b8fff7f4de4c26326bd2ccddd8915bef88 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 16 Aug 2021 07:55:29 -0300 Subject: [PATCH 07/12] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57d9119..1246dde 100644 --- a/README.md +++ b/README.md @@ -234,7 +234,7 @@ arbitrary field metadata. This can be accessed through a [`MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) object, which can be retrieved from an item instance with `itemadapter.adapter.ItemAdapter.get_field_meta`, or from an item class -with the `itemadapter.adapter.ItemAdapter.get_field_meta.get_field_meta_from_class` +with the `itemadapter.adapter.ItemAdapter.get_field_meta_from_class` method (or its alias `itemadapter.utils.get_field_meta_from_class`). The source of the data depends on the underlying type (see the docs for `ItemAdapter.get_field_meta_from_class` above). From bc4726140b7a26c79cbbe4d2e86c434434abaafc Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 16 Aug 2021 07:56:31 -0300 Subject: [PATCH 08/12] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1246dde..585719a 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ Return `True` if any of the registed adapters can handle the item #### class method `is_item_class(item_class: type) -> bool` -Return `True` if any of the registed adapters can handle the item class +Return `True` if any of the registered adapters can handle the item class (i.e. if any of them returns `True` for its `is_item_class` method with `item_class` as argument), `False` otherwise. From fdcd266d96f67fe33cfc924d70f3213e1629ee0e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 16 Aug 2021 08:00:04 -0300 Subject: [PATCH 09/12] Remove unnecessary stuff --- README.md | 2 +- itemadapter/adapter.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 585719a..5e5038a 100644 --- a/README.md +++ b/README.md @@ -237,7 +237,7 @@ object, which can be retrieved from an item instance with with the `itemadapter.adapter.ItemAdapter.get_field_meta_from_class` method (or its alias `itemadapter.utils.get_field_meta_from_class`). The source of the data depends on the underlying type (see the docs for -`ItemAdapter.get_field_meta_from_class` above). +`ItemAdapter.get_field_meta_from_class`). #### `scrapy.item.Item` objects diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 1a3dd66..74a9749 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -234,10 +234,6 @@ def is_item(cls, item: Any) -> bool: def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, dict) - @classmethod - def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: - return MappingProxyType({}) - def field_names(self) -> KeysView: return KeysView(self.item) From 758584b0e94a5fb2408c266e9595a2cffe6262d3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Aug 2021 12:12:51 -0300 Subject: [PATCH 10/12] Default implementation for is_item based on is_item_class --- Changelog.md | 2 +- README.md | 9 +++++---- itemadapter/adapter.py | 15 +++++---------- tests/test_interface.py | 6 +++--- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/Changelog.md b/Changelog.md index 82f793e..c3f30c4 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,7 +8,7 @@ Added `ItemAdapter.is_item_class` and `ItemAdapter.get_field_meta_from_class` ### 0.3.0 (2021-07-15) -Added suport for `pydantic` models ([#53](https://github.com/scrapy/itemadapter/pull/53)) +Added built-in support for `pydantic` models ([#53](https://github.com/scrapy/itemadapter/pull/53)) ### 0.2.0 (2020-11-06) diff --git a/README.md b/README.md index 5e5038a..8e4bbf4 100644 --- a/README.md +++ b/README.md @@ -319,14 +319,15 @@ inherit from this class and implement the abstract methods defined on it. `Adapt inherits from [`collections.abc.MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping), so all methods from the `MutableMapping` interface must be implemented as well. -* _class method `is_item(cls, item: Any) -> bool`_ - - Return `True` if the adapter can handle the given item, `False` otherwise. Abstract (mandatory). - * _class method `is_item_class(cls, item_class: type) -> bool`_ Return `True` if the adapter can handle the given item class, `False` otherwise. Abstract (mandatory). +* _class method `is_item(cls, item: Any) -> bool`_ + + Return `True` if the adapter can handle the given item, `False` otherwise. + The default implementation calls `cls.is_item_class(item.__class__)`. + * _class method `get_field_meta_from_class(cls, item_class: type) -> bool`_ Return metadata for the given item class and field name, if available. diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 74a9749..24c0d2e 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -42,15 +42,14 @@ def __init__(self, item: Any) -> None: @classmethod @abstractmethod - def is_item(cls, item: Any) -> bool: - """Return True if the adapter can handle the given item, False otherwise""" + def is_item_class(cls, item_class: type) -> bool: + """Return True if the adapter can handle the given item class, False otherwise.""" raise NotImplementedError() @classmethod - @abstractmethod - def is_item_class(cls, item_class: type) -> bool: - """Return True if the adapter can handle the given item class, False otherwise""" - raise NotImplementedError() + def is_item(cls, item: Any) -> bool: + """Return True if the adapter can handle the given item, False otherwise.""" + return cls.is_item_class(item.__class__) @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: @@ -226,10 +225,6 @@ def __len__(self) -> int: class DictAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): - @classmethod - def is_item(cls, item: Any) -> bool: - return isinstance(item, dict) - @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, dict) diff --git a/tests/test_interface.py b/tests/test_interface.py index a64d9e0..7e22264 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -28,9 +28,9 @@ def __init__(self, **kwargs) -> None: class BaseFakeItemAdapter(AdapterInterface): """An adapter that only implements the required methods.""" - @classmethod - def is_item(cls, item: Any) -> bool: - return isinstance(item, FakeItemClass) + # @classmethod + # def is_item(cls, item: Any) -> bool: + # return isinstance(item, FakeItemClass) @classmethod def is_item_class(cls, item_class: type) -> bool: From 4c0a2bf5985df303dcd14a8657950da9cbc1d7c7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Aug 2021 12:16:34 -0300 Subject: [PATCH 11/12] Remove commented code in test --- tests/test_interface.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_interface.py b/tests/test_interface.py index 7e22264..21c0783 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -28,10 +28,6 @@ def __init__(self, **kwargs) -> None: class BaseFakeItemAdapter(AdapterInterface): """An adapter that only implements the required methods.""" - # @classmethod - # def is_item(cls, item: Any) -> bool: - # return isinstance(item, FakeItemClass) - @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, FakeItemClass) From 383a89d0bc9d5b83d520b5ca7f9d7f0f300f399b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Aug 2021 13:54:30 -0300 Subject: [PATCH 12/12] use ItemAdapter.is_item instead of is_item --- itemadapter/adapter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 24c0d2e..0d8895b 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -12,7 +12,6 @@ _is_pydantic_model, is_attrs_instance, is_dataclass_instance, - is_item, is_pydantic_instance, is_scrapy_item, ) @@ -336,7 +335,7 @@ def _asdict(obj: Any) -> Any: return obj.__class__(_asdict(x) for x in obj) elif isinstance(obj, ItemAdapter): return obj.asdict() - elif is_item(obj): + elif ItemAdapter.is_item(obj): return ItemAdapter(obj).asdict() else: return obj