diff --git a/python/pydantic_core/_pydantic_core.pyi b/python/pydantic_core/_pydantic_core.pyi index aeec227f8..eb30a1d44 100644 --- a/python/pydantic_core/_pydantic_core.pyi +++ b/python/pydantic_core/_pydantic_core.pyi @@ -266,6 +266,7 @@ class SchemaSerializer: fallback: Callable[[Any], Any] | None = None, serialize_as_any: bool = False, context: Any | None = None, + serialize_generators: bool = True, ) -> Any: """ Serialize/marshal a Python object to a Python object including transforming and filtering data. @@ -289,6 +290,7 @@ class SchemaSerializer: serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. context: The context to use for serialization, this is passed to functional serializers as [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. + serialize_generators: Whether to consume and wrap generators. If this is False, generators are treated as an unknown type. Raises: PydanticSerializationError: If serialization fails and no `fallback` function is provided. @@ -312,6 +314,7 @@ class SchemaSerializer: fallback: Callable[[Any], Any] | None = None, serialize_as_any: bool = False, context: Any | None = None, + serialize_generators: bool = True, ) -> bytes: """ Serialize a Python object to JSON including transforming and filtering data. @@ -334,6 +337,7 @@ class SchemaSerializer: serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. context: The context to use for serialization, this is passed to functional serializers as [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. + serialize_generators: Whether to consume and wrap generators. If this is False, generators are treated as an unknown type. Raises: PydanticSerializationError: If serialization fails and no `fallback` function is provided. @@ -358,6 +362,7 @@ def to_json( fallback: Callable[[Any], Any] | None = None, serialize_as_any: bool = False, context: Any | None = None, + serialize_generators: bool = True, ) -> bytes: """ Serialize a Python object to JSON including transforming and filtering data. @@ -382,6 +387,7 @@ def to_json( serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. context: The context to use for serialization, this is passed to functional serializers as [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. + serialize_generators: Whether to consume and wrap generators. If this is False, generators are treated as an unknown type. Raises: PydanticSerializationError: If serialization fails and no `fallback` function is provided. @@ -433,6 +439,7 @@ def to_jsonable_python( fallback: Callable[[Any], Any] | None = None, serialize_as_any: bool = False, context: Any | None = None, + serialize_generators: bool = True, ) -> Any: """ Serialize/marshal a Python object to a JSON-serializable Python object including transforming and filtering data. @@ -457,6 +464,7 @@ def to_jsonable_python( serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. context: The context to use for serialization, this is passed to functional serializers as [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. + serialize_generators: Whether to consume and wrap generators. If this is False, generators are treated as an unknown type. Raises: PydanticSerializationError: If serialization fails and no `fallback` function is provided. diff --git a/src/errors/validation_exception.rs b/src/errors/validation_exception.rs index e7861fa9b..d3cfa62ba 100644 --- a/src/errors/validation_exception.rs +++ b/src/errors/validation_exception.rs @@ -333,6 +333,7 @@ impl ValidationError { None, DuckTypingSerMode::SchemaBased, None, + true, ); let serializer = ValidationErrorSerializer { py, diff --git a/src/serializers/extra.rs b/src/serializers/extra.rs index a62b55304..9e69ce2e1 100644 --- a/src/serializers/extra.rs +++ b/src/serializers/extra.rs @@ -89,6 +89,7 @@ impl SerializationState { fallback: Option<&'py Bound<'_, PyAny>>, duck_typing_ser_mode: DuckTypingSerMode, context: Option<&'py Bound<'_, PyAny>>, + serialize_generators: bool, ) -> Extra<'py> { Extra::new( py, @@ -105,6 +106,7 @@ impl SerializationState { fallback, duck_typing_ser_mode, context, + serialize_generators, ) } @@ -138,6 +140,7 @@ pub(crate) struct Extra<'a> { pub fallback: Option<&'a Bound<'a, PyAny>>, pub duck_typing_ser_mode: DuckTypingSerMode, pub context: Option<&'a Bound<'a, PyAny>>, + pub serialize_generators: bool, } impl<'a> Extra<'a> { @@ -157,6 +160,7 @@ impl<'a> Extra<'a> { fallback: Option<&'a Bound<'a, PyAny>>, duck_typing_ser_mode: DuckTypingSerMode, context: Option<&'a Bound<'a, PyAny>>, + serialize_generators: bool, ) -> Self { Self { mode, @@ -176,6 +180,7 @@ impl<'a> Extra<'a> { fallback, duck_typing_ser_mode, context, + serialize_generators, } } @@ -236,6 +241,7 @@ pub(crate) struct ExtraOwned { pub fallback: Option, duck_typing_ser_mode: DuckTypingSerMode, pub context: Option, + serialize_generators: bool, } impl ExtraOwned { @@ -257,6 +263,7 @@ impl ExtraOwned { fallback: extra.fallback.map(|model| model.clone().into()), duck_typing_ser_mode: extra.duck_typing_ser_mode, context: extra.context.map(|model| model.clone().into()), + serialize_generators: extra.serialize_generators, } } @@ -279,6 +286,7 @@ impl ExtraOwned { fallback: self.fallback.as_ref().map(|m| m.bind(py)), duck_typing_ser_mode: self.duck_typing_ser_mode, context: self.context.as_ref().map(|m| m.bind(py)), + serialize_generators: self.serialize_generators, } } } diff --git a/src/serializers/infer.rs b/src/serializers/infer.rs index 45d8957bb..7b62b1c22 100644 --- a/src/serializers/infer.rs +++ b/src/serializers/infer.rs @@ -106,6 +106,7 @@ pub(crate) fn infer_to_python_known( extra.fallback, extra.duck_typing_ser_mode, extra.context, + extra.serialize_generators, ); serializer.serializer.to_python(value, include, exclude, &extra) }; @@ -207,7 +208,7 @@ pub(crate) fn infer_to_python_known( let v = value.getattr(intern!(py, "value"))?; infer_to_python(&v, include, exclude, extra)?.into_py(py) } - ObType::Generator => { + ObType::Generator if extra.serialize_generators => { let py_seq = value.downcast::()?; let mut items = Vec::new(); let filter = AnyFilter::new(); @@ -228,7 +229,7 @@ pub(crate) fn infer_to_python_known( } ObType::Path => value.str()?.into_py(py), ObType::Pattern => value.getattr(intern!(py, "pattern"))?.into_py(py), - ObType::Unknown => { + _ => { if let Some(fallback) = extra.fallback { let next_value = fallback.call1((value,))?; let next_result = infer_to_python(&next_value, include, exclude, extra); @@ -263,7 +264,7 @@ pub(crate) fn infer_to_python_known( } ObType::PydanticSerializable => serialize_with_serializer()?, ObType::Dataclass => serialize_pairs_python(py, any_dataclass_iter(value)?.0, include, exclude, extra, Ok)?, - ObType::Generator => { + ObType::Generator if extra.serialize_generators => { let iter = super::type_serializers::generator::SerializationIterator::new( value.downcast()?, super::type_serializers::any::AnySerializer.into(), @@ -274,7 +275,8 @@ pub(crate) fn infer_to_python_known( ); iter.into_py(py) } - ObType::Unknown => { + ObType::Unknown | ObType::Generator => { + // if !extra.serialize_generators if let Some(fallback) = extra.fallback { let next_value = fallback.call1((value,))?; let next_result = infer_to_python(&next_value, include, exclude, extra); @@ -482,6 +484,7 @@ pub(crate) fn infer_serialize_known( extra.fallback, extra.duck_typing_ser_mode, extra.context, + extra.serialize_generators, ); let pydantic_serializer = PydanticSerializer::new(value, &extracted_serializer.serializer, include, exclude, &extra); @@ -499,7 +502,7 @@ pub(crate) fn infer_serialize_known( let v = value.getattr(intern!(value.py(), "value")).map_err(py_err_se_err)?; infer_serialize(&v, serializer, include, exclude, extra) } - ObType::Generator => { + ObType::Generator if extra.serialize_generators => { let py_seq = value.downcast::().map_err(py_err_se_err)?; let mut seq = serializer.serialize_seq(None)?; let filter = AnyFilter::new(); @@ -530,7 +533,7 @@ pub(crate) fn infer_serialize_known( .map_err(py_err_se_err)?; serializer.serialize_str(&s) } - ObType::Unknown => { + ObType::Unknown | ObType::Generator => { if let Some(fallback) = extra.fallback { let next_value = fallback.call1((value,)).map_err(py_err_se_err)?; let next_result = infer_serialize(&next_value, serializer, include, exclude, extra); @@ -550,14 +553,14 @@ pub(crate) fn infer_serialize_known( ser_result } -fn unknown_type_error(value: &Bound<'_, PyAny>) -> PyErr { +pub(crate) fn unknown_type_error(value: &Bound<'_, PyAny>) -> PyErr { PydanticSerializationError::new_err(format!( "Unable to serialize unknown type: {}", safe_repr(&value.get_type()) )) } -fn serialize_unknown<'py>(value: &Bound<'py, PyAny>) -> Cow<'py, str> { +pub(crate) fn serialize_unknown<'py>(value: &Bound<'py, PyAny>) -> Cow<'py, str> { if let Ok(s) = value.str() { s.to_string_lossy().into_owned().into() } else if let Ok(name) = value.get_type().qualname() { diff --git a/src/serializers/mod.rs b/src/serializers/mod.rs index 1a0405e2c..20dd1369d 100644 --- a/src/serializers/mod.rs +++ b/src/serializers/mod.rs @@ -64,6 +64,7 @@ impl SchemaSerializer { fallback: Option<&'a Bound<'a, PyAny>>, duck_typing_ser_mode: DuckTypingSerMode, context: Option<&'a Bound<'a, PyAny>>, + serialize_generators: bool, ) -> Extra<'b> { Extra::new( py, @@ -80,6 +81,7 @@ impl SchemaSerializer { fallback, duck_typing_ser_mode, context, + serialize_generators, ) } } @@ -107,7 +109,7 @@ impl SchemaSerializer { #[allow(clippy::too_many_arguments)] #[pyo3(signature = (value, *, mode = None, include = None, exclude = None, by_alias = true, exclude_unset = false, exclude_defaults = false, exclude_none = false, round_trip = false, warnings = WarningsArg::Bool(true), - fallback = None, serialize_as_any = false, context = None))] + fallback = None, serialize_as_any = false, context = None, serialize_generators = true))] pub fn to_python( &self, py: Python, @@ -124,6 +126,7 @@ impl SchemaSerializer { fallback: Option<&Bound<'_, PyAny>>, serialize_as_any: bool, context: Option<&Bound<'_, PyAny>>, + serialize_generators: bool, ) -> PyResult { let mode: SerMode = mode.into(); let warnings_mode = match warnings { @@ -147,6 +150,7 @@ impl SchemaSerializer { fallback, duck_typing_ser_mode, context, + serialize_generators, ); let v = self.serializer.to_python(value, include, exclude, &extra)?; warnings.final_check(py)?; @@ -156,7 +160,7 @@ impl SchemaSerializer { #[allow(clippy::too_many_arguments)] #[pyo3(signature = (value, *, indent = None, include = None, exclude = None, by_alias = true, exclude_unset = false, exclude_defaults = false, exclude_none = false, round_trip = false, warnings = WarningsArg::Bool(true), - fallback = None, serialize_as_any = false, context = None))] + fallback = None, serialize_as_any = false, context = None, serialize_generators = true))] pub fn to_json( &self, py: Python, @@ -173,6 +177,7 @@ impl SchemaSerializer { fallback: Option<&Bound<'_, PyAny>>, serialize_as_any: bool, context: Option<&Bound<'_, PyAny>>, + serialize_generators: bool, ) -> PyResult { let warnings_mode = match warnings { WarningsArg::Bool(b) => b.into(), @@ -195,6 +200,7 @@ impl SchemaSerializer { fallback, duck_typing_ser_mode, context, + serialize_generators, ); let bytes = to_json_bytes( value, @@ -244,7 +250,7 @@ impl SchemaSerializer { #[pyo3(signature = (value, *, indent = None, include = None, exclude = None, by_alias = true, exclude_none = false, round_trip = false, timedelta_mode = "iso8601", bytes_mode = "utf8", inf_nan_mode = "constants", serialize_unknown = false, fallback = None, serialize_as_any = false, - context = None))] + context = None, serialize_generators = true))] pub fn to_json( py: Python, value: &Bound<'_, PyAny>, @@ -261,6 +267,7 @@ pub fn to_json( fallback: Option<&Bound<'_, PyAny>>, serialize_as_any: bool, context: Option<&Bound<'_, PyAny>>, + serialize_generators: bool, ) -> PyResult { let state = SerializationState::new(timedelta_mode, bytes_mode, inf_nan_mode)?; let duck_typing_ser_mode = DuckTypingSerMode::from_bool(serialize_as_any); @@ -274,6 +281,7 @@ pub fn to_json( fallback, duck_typing_ser_mode, context, + serialize_generators, ); let serializer = type_serializers::any::AnySerializer.into(); let bytes = to_json_bytes(value, &serializer, include, exclude, &extra, indent, 1024)?; @@ -286,7 +294,7 @@ pub fn to_json( #[pyfunction] #[pyo3(signature = (value, *, include = None, exclude = None, by_alias = true, exclude_none = false, round_trip = false, timedelta_mode = "iso8601", bytes_mode = "utf8", inf_nan_mode = "constants", serialize_unknown = false, fallback = None, - serialize_as_any = false, context = None))] + serialize_as_any = false, context = None, serialize_generators = true))] pub fn to_jsonable_python( py: Python, value: &Bound<'_, PyAny>, @@ -302,6 +310,7 @@ pub fn to_jsonable_python( fallback: Option<&Bound<'_, PyAny>>, serialize_as_any: bool, context: Option<&Bound<'_, PyAny>>, + serialize_generators: bool, ) -> PyResult { let state = SerializationState::new(timedelta_mode, bytes_mode, inf_nan_mode)?; let duck_typing_ser_mode = DuckTypingSerMode::from_bool(serialize_as_any); @@ -315,6 +324,7 @@ pub fn to_jsonable_python( fallback, duck_typing_ser_mode, context, + serialize_generators, ); let v = infer::infer_to_python(value, include, exclude, &extra)?; state.final_check(py)?; diff --git a/src/serializers/type_serializers/generator.rs b/src/serializers/type_serializers/generator.rs index d7d211456..52a6129a5 100644 --- a/src/serializers/type_serializers/generator.rs +++ b/src/serializers/type_serializers/generator.rs @@ -16,6 +16,7 @@ use super::{ infer_serialize, infer_to_python, py_err_se_err, BuildSerializer, CombinedSerializer, Extra, ExtraOwned, PydanticSerializer, SchemaFilter, SerMode, TypeSerializer, }; +use crate::serializers::infer::{serialize_unknown, unknown_type_error}; #[derive(Debug, Clone)] pub struct GeneratorSerializer { @@ -57,8 +58,8 @@ impl TypeSerializer for GeneratorSerializer { match value.downcast::() { Ok(py_iter) => { let py = value.py(); - match extra.mode { - SerMode::Json => { + match (extra.serialize_generators, extra.mode) { + (true, SerMode::Json) => { let item_serializer = self.item_serializer.as_ref(); let mut items = match value.len() { @@ -79,7 +80,7 @@ impl TypeSerializer for GeneratorSerializer { } Ok(items.into_py(py)) } - _ => { + (true, _) => { let iter = SerializationIterator::new( py_iter, self.item_serializer.as_ref().clone(), @@ -90,6 +91,24 @@ impl TypeSerializer for GeneratorSerializer { ); Ok(iter.into_py(py)) } + (false, SerMode::Json) => { + if let Some(fallback) = extra.fallback { + let next_value = fallback.call1((value,))?; + infer_to_python(&next_value, include, exclude, extra) + } else if extra.serialize_unknown { + Ok(serialize_unknown(value).into_py(py)) + } else { + return Err(unknown_type_error(value)); + } + } + (false, _) => { + if let Some(fallback) = extra.fallback { + let next_value = fallback.call1((value,))?; + let next_result = infer_to_python(&next_value, include, exclude, extra); + return next_result; + } + Ok(value.into_py(py)) + } } } Err(_) => { @@ -111,6 +130,12 @@ impl TypeSerializer for GeneratorSerializer { exclude: Option<&Bound<'_, PyAny>>, extra: &Extra, ) -> Result { + if !extra.serialize_generators { + if extra.fallback.is_none() { + extra.warnings.on_fallback_ser::(self.get_name(), value, extra)?; + } + return infer_serialize(value, serializer, include, exclude, extra); + } match value.downcast::() { Ok(py_iter) => { let len = match value.len() { diff --git a/tests/serializers/test_generator.py b/tests/serializers/test_generator.py index cbd72ee8e..3163bf264 100644 --- a/tests/serializers/test_generator.py +++ b/tests/serializers/test_generator.py @@ -1,6 +1,7 @@ import pytest from dirty_equals import IsStr +import pydantic_core from pydantic_core import SchemaSerializer, core_schema @@ -137,3 +138,77 @@ def test_custom_serializer(): s = SchemaSerializer(core_schema.any_schema(serialization=core_schema.simple_ser_schema('generator'))) assert s.to_python(gen_ok(1, 2), mode='json') == [1, 2] assert s.to_json(gen_ok(1, 2)) == b'[1,2]' + + +def test_generator_no_ser_any_iter(): + s = SchemaSerializer(core_schema.generator_schema(core_schema.any_schema())) + gen_inner = gen_ok('a', b'b', 3) + gen = s.to_python(gen_inner, serialize_generators=False) + assert gen_inner is gen + + +def test_any_no_ser_iter(): + s = SchemaSerializer(core_schema.any_schema()) + gen_inner = gen_ok('a', b'b', 3) + gen = s.to_python(gen_inner, serialize_generators=False) + assert gen is gen_inner + + with pytest.raises(ValueError, match='Unable to serialize unknown type'): + assert s.to_json(iter(['a', b'b', 3]), serialize_generators=False) == b'["a","b",3]' + + +def test_generator_no_ser_any(): + # todo + s = SchemaSerializer(core_schema.generator_schema(core_schema.any_schema())) + assert list(s.to_python(iter(['a', b'b', 3]), serialize_generators=False)) == ['a', b'b', 3] + assert list(s.to_python(gen_ok('a', b'b', 3), serialize_generators=False)) == ['a', b'b', 3] + + with pytest.raises(ValueError, match='Unable to serialize unknown type'): + assert s.to_python(iter(['a', b'b', 3]), mode='json', serialize_generators=False) == ['a', 'b', 3] + + with pytest.raises(ValueError, match='Unable to serialize unknown type'): + assert s.to_json(iter(['a', b'b', 3]), serialize_generators=False) == b'["a","b",3]' + assert s.to_json(iter(['a', b'b', 3]), serialize_generators=False, fallback=list) == b'["a","b",3]' + assert s.to_json(gen_ok('a', b'b', 3), serialize_generators=False, fallback=list) == b'["a","b",3]' + assert s.to_python(gen_ok('a', b'b', 3), serialize_generators=False, fallback=list) == ['a', b'b', 3] + + msg = 'Expected `generator` but got `int` with value `4` - serialized value may not be as expected' + with pytest.warns(UserWarning, match=msg): + assert s.to_python(4, serialize_generators=False) == 4 + with pytest.warns(UserWarning, match="Expected `generator` but got `tuple` with value `\\('a', b'b', 3\\)`"): + assert s.to_python(('a', b'b', 3), serialize_generators=False) == ('a', b'b', 3) + with pytest.warns(UserWarning, match="Expected `generator` but got `str` with value `'abc'`"): + assert s.to_python('abc', serialize_generators=False) == 'abc' + + with pytest.raises(ValueError, match='oops'): + list(s.to_python(gen_error(1, 2), serialize_generators=False)) + + with pytest.raises(ValueError, match='Unable to serialize unknown type'): + s.to_python(gen_error(1, 2), mode='json', serialize_generators=False) + + with pytest.raises(ValueError, match='Unable to serialize unknown type'): + s.to_json(gen_error(1, 2), serialize_generators=False) + + +def test_no_ser_custom_serializer(): + s = SchemaSerializer(core_schema.any_schema(serialization=core_schema.simple_ser_schema('generator'))) + with pytest.raises(ValueError, match='Unable to serialize unknown type'): + assert s.to_python(gen_ok(1, 2), mode='json', serialize_generators=False) == [1, 2] + with pytest.raises(ValueError, match='Unable to serialize unknown type'): + assert s.to_json(gen_ok(1, 2), serialize_generators=False) == b'[1,2]' + assert s.to_python(gen_ok(1, 2), mode='json', serialize_generators=False, fallback=list) == [1, 2] + assert s.to_json(gen_ok(1, 2), serialize_generators=False, fallback=list) == b'[1,2]' + + +def test_no_ser_to_json(): + with pytest.raises(ValueError, match='Unable to serialize unknown type'): + assert pydantic_core.to_jsonable_python(iter([]), serialize_generators=False) + assert pydantic_core.to_jsonable_python(iter([]), serialize_generators=False, serialize_unknown=True) == IsStr( + regex=r'<(list_|sequence)iterator object at 0x\w+>' + ) + assert pydantic_core.to_json(iter([]), serialize_generators=False, serialize_unknown=True).decode('utf8') == IsStr( + regex=r'"<(list_|sequence)iterator object at 0x\w+>"' + ) + assert pydantic_core.to_json(iter([]), serialize_generators=False, serialize_unknown=True).decode('utf8') == IsStr( + regex=r'"<(list_|sequence)iterator object at 0x\w+>"' + ) diff --git a/tests/test.rs b/tests/test.rs index 1a4a87ef3..1d2354452 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -99,6 +99,7 @@ a = A() None, false, None, + true, ) .unwrap(); let serialized: &[u8] = serialized.extract(py).unwrap(); @@ -203,6 +204,7 @@ dump_json_input_2 = {'a': 'something'} None, false, None, + true, ) .unwrap(); let repr = format!("{}", serialization_result.bind(py).repr().unwrap()); @@ -224,6 +226,7 @@ dump_json_input_2 = {'a': 'something'} None, false, None, + true, ) .unwrap(); let repr = format!("{}", serialization_result.bind(py).repr().unwrap());