From c4c2ee2428c0132c8a05957eb74525b274fead28 Mon Sep 17 00:00:00 2001 From: sydney-runkle Date: Fri, 25 Oct 2024 12:44:51 -0400 Subject: [PATCH] Revert "feat: add 'millisecond' option to ser_json_timedelta config parameter (#1427)" This reverts commit e0b4c94054b4bca649d63a8f18843c3baadee1c4. --- python/pydantic_core/_pydantic_core.pyi | 8 +- python/pydantic_core/core_schema.py | 2 +- src/input/datetime.rs | 87 ---------- src/serializers/config.rs | 42 +++-- src/serializers/infer.rs | 4 +- src/serializers/type_serializers/timedelta.rs | 6 +- tests/serializers/test_any.py | 161 +++--------------- tests/serializers/test_pickling.py | 2 +- tests/serializers/test_timedelta.py | 2 +- tests/test_garbage_collection.py | 2 +- 10 files changed, 57 insertions(+), 259 deletions(-) diff --git a/python/pydantic_core/_pydantic_core.pyi b/python/pydantic_core/_pydantic_core.pyi index 2b06ee54c..d696ac407 100644 --- a/python/pydantic_core/_pydantic_core.pyi +++ b/python/pydantic_core/_pydantic_core.pyi @@ -357,7 +357,7 @@ def to_json( by_alias: bool = True, exclude_none: bool = False, round_trip: bool = False, - timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601', + timedelta_mode: Literal['iso8601', 'float'] = 'iso8601', bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8', inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants', serialize_unknown: bool = False, @@ -378,7 +378,7 @@ def to_json( by_alias: Whether to use the alias names of fields. exclude_none: Whether to exclude fields that have a value of `None`. round_trip: Whether to enable serialization and validation round-trip support. - timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'` or `'milliseconds_float'`. + timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`. bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`. inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`. serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails @@ -432,7 +432,7 @@ def to_jsonable_python( by_alias: bool = True, exclude_none: bool = False, round_trip: bool = False, - timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601', + timedelta_mode: Literal['iso8601', 'float'] = 'iso8601', bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8', inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants', serialize_unknown: bool = False, @@ -453,7 +453,7 @@ def to_jsonable_python( by_alias: Whether to use the alias names of fields. exclude_none: Whether to exclude fields that have a value of `None`. round_trip: Whether to enable serialization and validation round-trip support. - timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'`, or`'milliseconds_float'`. + timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`. bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`. inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`. serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails diff --git a/python/pydantic_core/core_schema.py b/python/pydantic_core/core_schema.py index 98ff0c61d..5a8646fca 100644 --- a/python/pydantic_core/core_schema.py +++ b/python/pydantic_core/core_schema.py @@ -105,7 +105,7 @@ class CoreConfig(TypedDict, total=False): # fields related to float fields only allow_inf_nan: bool # default: True # the config options are used to customise serialization to JSON - ser_json_timedelta: Literal['iso8601', 'seconds_float', 'milliseconds_float'] # default: 'iso8601' + ser_json_timedelta: Literal['iso8601', 'float'] # default: 'iso8601' ser_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8' ser_json_inf_nan: Literal['null', 'constants', 'strings'] # default: 'null' val_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8' diff --git a/src/input/datetime.rs b/src/input/datetime.rs index db5daff40..577a91014 100644 --- a/src/input/datetime.rs +++ b/src/input/datetime.rs @@ -109,93 +109,6 @@ impl<'a> EitherTimedelta<'a> { Self::Raw(duration) => duration_as_pytimedelta(py, duration), } } - - pub fn total_seconds(&self) -> PyResult { - match self { - Self::Raw(timedelta) => { - let mut days: i64 = i64::from(timedelta.day); - let mut seconds: i64 = i64::from(timedelta.second); - let mut microseconds = i64::from(timedelta.microsecond); - if !timedelta.positive { - days = -days; - seconds = -seconds; - microseconds = -microseconds; - } - - let days_seconds = (86_400 * days) + seconds; - if let Some(days_seconds_as_micros) = days_seconds.checked_mul(1_000_000) { - let total_microseconds = days_seconds_as_micros + microseconds; - Ok(total_microseconds as f64 / 1_000_000.0) - } else { - // Fall back to floating-point operations if the multiplication overflows - let total_seconds = days_seconds as f64 + microseconds as f64 / 1_000_000.0; - Ok(total_seconds) - } - } - Self::PyExact(py_timedelta) => { - let days: i64 = py_timedelta.get_days().into(); // -999999999 to 999999999 - let seconds: i64 = py_timedelta.get_seconds().into(); // 0 through 86399 - let microseconds = py_timedelta.get_microseconds(); // 0 through 999999 - let days_seconds = (86_400 * days) + seconds; - if let Some(days_seconds_as_micros) = days_seconds.checked_mul(1_000_000) { - let total_microseconds = days_seconds_as_micros + i64::from(microseconds); - Ok(total_microseconds as f64 / 1_000_000.0) - } else { - // Fall back to floating-point operations if the multiplication overflows - let total_seconds = days_seconds as f64 + f64::from(microseconds) / 1_000_000.0; - Ok(total_seconds) - } - } - Self::PySubclass(py_timedelta) => py_timedelta - .call_method0(intern!(py_timedelta.py(), "total_seconds"))? - .extract(), - } - } - - pub fn total_milliseconds(&self) -> PyResult { - match self { - Self::Raw(timedelta) => { - let mut days: i64 = i64::from(timedelta.day); - let mut seconds: i64 = i64::from(timedelta.second); - let mut microseconds = i64::from(timedelta.microsecond); - if !timedelta.positive { - days = -days; - seconds = -seconds; - microseconds = -microseconds; - } - - let days_seconds = (86_400 * days) + seconds; - if let Some(days_seconds_as_micros) = days_seconds.checked_mul(1_000_000) { - let total_microseconds = days_seconds_as_micros + microseconds; - Ok(total_microseconds as f64 / 1_000.0) - } else { - // Fall back to floating-point operations if the multiplication overflows - let total_seconds = days_seconds as f64 + microseconds as f64 / 1_000.0; - Ok(total_seconds) - } - } - Self::PyExact(py_timedelta) => { - let days: i64 = py_timedelta.get_days().into(); // -999999999 to 999999999 - let seconds: i64 = py_timedelta.get_seconds().into(); // 0 through 86399 - let microseconds = py_timedelta.get_microseconds(); // 0 through 999999 - let days_seconds = (86_400 * days) + seconds; - if let Some(days_seconds_as_micros) = days_seconds.checked_mul(1_000_000) { - let total_microseconds = days_seconds_as_micros + i64::from(microseconds); - Ok(total_microseconds as f64 / 1_000.0) - } else { - // Fall back to floating-point operations if the multiplication overflows - let total_milliseconds = days_seconds as f64 * 1_000.0 + f64::from(microseconds) / 1_000.0; - Ok(total_milliseconds) - } - } - Self::PySubclass(py_timedelta) => { - let total_seconds: f64 = py_timedelta - .call_method0(intern!(py_timedelta.py(), "total_seconds"))? - .extract()?; - Ok(total_seconds / 1000.0) - } - } - } } impl<'a> TryFrom<&'_ Bound<'a, PyAny>> for EitherTimedelta<'a> { diff --git a/src/serializers/config.rs b/src/serializers/config.rs index 8b1367958..13a833176 100644 --- a/src/serializers/config.rs +++ b/src/serializers/config.rs @@ -4,7 +4,7 @@ use std::str::{from_utf8, FromStr, Utf8Error}; use base64::Engine; use pyo3::intern; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyString}; +use pyo3::types::{PyDelta, PyDict, PyString}; use serde::ser::Error; @@ -88,8 +88,7 @@ serialization_mode! { TimedeltaMode, "ser_json_timedelta", Iso8601 => "iso8601", - SecondsFloat => "seconds_float", - MillisecondsFloat => "milliseconds_float" + Float => "float", } serialization_mode! { @@ -109,42 +108,43 @@ serialization_mode! { } impl TimedeltaMode { + fn total_seconds<'py>(py_timedelta: &Bound<'py, PyDelta>) -> PyResult> { + py_timedelta.call_method0(intern!(py_timedelta.py(), "total_seconds")) + } + pub fn either_delta_to_json(self, py: Python, either_delta: &EitherTimedelta) -> PyResult { match self { Self::Iso8601 => { let d = either_delta.to_duration()?; Ok(d.to_string().into_py(py)) } - Self::SecondsFloat => { - let seconds: f64 = either_delta.total_seconds()?; + Self::Float => { + // convert to int via a py timedelta not duration since we know this this case the input would have + // been a py timedelta + let py_timedelta = either_delta.try_into_py(py)?; + let seconds = Self::total_seconds(&py_timedelta)?; Ok(seconds.into_py(py)) } - Self::MillisecondsFloat => { - let milliseconds: f64 = either_delta.total_milliseconds()?; - Ok(milliseconds.into_py(py)) - } } } - pub fn json_key<'py>(self, either_delta: &EitherTimedelta) -> PyResult> { + pub fn json_key<'py>(self, py: Python, either_delta: &EitherTimedelta) -> PyResult> { match self { Self::Iso8601 => { let d = either_delta.to_duration()?; Ok(d.to_string().into()) } - Self::SecondsFloat => { - let seconds: f64 = either_delta.total_seconds()?; + Self::Float => { + let py_timedelta = either_delta.try_into_py(py)?; + let seconds: f64 = Self::total_seconds(&py_timedelta)?.extract()?; Ok(seconds.to_string().into()) } - Self::MillisecondsFloat => { - let milliseconds: f64 = either_delta.total_milliseconds()?; - Ok(milliseconds.to_string().into()) - } } } pub fn timedelta_serialize( self, + py: Python, either_delta: &EitherTimedelta, serializer: S, ) -> Result { @@ -153,14 +153,12 @@ impl TimedeltaMode { let d = either_delta.to_duration().map_err(py_err_se_err)?; serializer.serialize_str(&d.to_string()) } - Self::SecondsFloat => { - let seconds: f64 = either_delta.total_seconds().map_err(py_err_se_err)?; + Self::Float => { + let py_timedelta = either_delta.try_into_py(py).map_err(py_err_se_err)?; + let seconds = Self::total_seconds(&py_timedelta).map_err(py_err_se_err)?; + let seconds: f64 = seconds.extract().map_err(py_err_se_err)?; serializer.serialize_f64(seconds) } - Self::MillisecondsFloat => { - let milliseconds: f64 = either_delta.total_milliseconds().map_err(py_err_se_err)?; - serializer.serialize_f64(milliseconds) - } } } } diff --git a/src/serializers/infer.rs b/src/serializers/infer.rs index f03e890d5..4e6bb504f 100644 --- a/src/serializers/infer.rs +++ b/src/serializers/infer.rs @@ -477,7 +477,7 @@ pub(crate) fn infer_serialize_known( extra .config .timedelta_mode - .timedelta_serialize(&either_delta, serializer) + .timedelta_serialize(value.py(), &either_delta, serializer) } ObType::Url => { let py_url: PyUrl = value.extract().map_err(py_err_se_err)?; @@ -655,7 +655,7 @@ pub(crate) fn infer_json_key_known<'a>( } ObType::Timedelta => { let either_delta = EitherTimedelta::try_from(key)?; - extra.config.timedelta_mode.json_key(&either_delta) + extra.config.timedelta_mode.json_key(key.py(), &either_delta) } ObType::Url => { let py_url: PyUrl = key.extract()?; diff --git a/src/serializers/type_serializers/timedelta.rs b/src/serializers/type_serializers/timedelta.rs index acc604732..a62b595d0 100644 --- a/src/serializers/type_serializers/timedelta.rs +++ b/src/serializers/type_serializers/timedelta.rs @@ -54,7 +54,7 @@ impl TypeSerializer for TimeDeltaSerializer { fn json_key<'a>(&self, key: &'a Bound<'_, PyAny>, extra: &Extra) -> PyResult> { match EitherTimedelta::try_from(key) { - Ok(either_timedelta) => self.timedelta_mode.json_key(&either_timedelta), + Ok(either_timedelta) => self.timedelta_mode.json_key(key.py(), &either_timedelta), Err(_) => { extra.warnings.on_fallback_py(self.get_name(), key, extra)?; infer_json_key(key, extra) @@ -71,7 +71,9 @@ impl TypeSerializer for TimeDeltaSerializer { extra: &Extra, ) -> Result { match EitherTimedelta::try_from(value) { - Ok(either_timedelta) => self.timedelta_mode.timedelta_serialize(&either_timedelta, serializer), + Ok(either_timedelta) => self + .timedelta_mode + .timedelta_serialize(value.py(), &either_timedelta, serializer), Err(_) => { extra.warnings.on_fallback_ser::(self.get_name(), value, extra)?; infer_serialize(value, serializer, include, exclude, extra) diff --git a/tests/serializers/test_any.py b/tests/serializers/test_any.py index 9fbc8ac03..344e0ff02 100644 --- a/tests/serializers/test_any.py +++ b/tests/serializers/test_any.py @@ -177,139 +177,28 @@ def test_any_with_timedelta_serializer(): ] -@pytest.mark.parametrize( - 'td,expected_to_python,expected_to_json,expected_to_python_dict,expected_to_json_dict,mode', - [ - (timedelta(hours=2), 7200000.0, b'7200000.0', {'7200000': 'foo'}, b'{"7200000":"foo"}', 'milliseconds_float'), - ( - timedelta(hours=-2), - -7200000.0, - b'-7200000.0', - {'-7200000': 'foo'}, - b'{"-7200000":"foo"}', - 'milliseconds_float', - ), - (timedelta(seconds=1.5), 1500.0, b'1500.0', {'1500': 'foo'}, b'{"1500":"foo"}', 'milliseconds_float'), - (timedelta(seconds=-1.5), -1500.0, b'-1500.0', {'-1500': 'foo'}, b'{"-1500":"foo"}', 'milliseconds_float'), - (timedelta(microseconds=1), 0.001, b'0.001', {'0.001': 'foo'}, b'{"0.001":"foo"}', 'milliseconds_float'), - ( - timedelta(microseconds=-1), - -0.001, - b'-0.001', - {'-0.001': 'foo'}, - b'{"-0.001":"foo"}', - 'milliseconds_float', - ), - ( - timedelta(days=1), - 86400000.0, - b'86400000.0', - {'86400000': 'foo'}, - b'{"86400000":"foo"}', - 'milliseconds_float', - ), - ( - timedelta(days=-1), - -86400000.0, - b'-86400000.0', - {'-86400000': 'foo'}, - b'{"-86400000":"foo"}', - 'milliseconds_float', - ), - ( - timedelta(days=1, seconds=1), - 86401000.0, - b'86401000.0', - {'86401000': 'foo'}, - b'{"86401000":"foo"}', - 'milliseconds_float', - ), - ( - timedelta(days=-1, seconds=-1), - -86401000.0, - b'-86401000.0', - {'-86401000': 'foo'}, - b'{"-86401000":"foo"}', - 'milliseconds_float', - ), - ( - timedelta(days=1, seconds=-1), - 86399000.0, - b'86399000.0', - {'86399000': 'foo'}, - b'{"86399000":"foo"}', - 'milliseconds_float', - ), - ( - timedelta(days=1, seconds=1, microseconds=1), - 86401000.001, - b'86401000.001', - {'86401000.001': 'foo'}, - b'{"86401000.001":"foo"}', - 'milliseconds_float', - ), - ( - timedelta(days=-1, seconds=-1, microseconds=-1), - -86401000.001, - b'-86401000.001', - {'-86401000.001': 'foo'}, - b'{"-86401000.001":"foo"}', - 'milliseconds_float', - ), - (timedelta(hours=2), 7200.0, b'7200.0', {'7200': 'foo'}, b'{"7200":"foo"}', 'seconds_float'), - (timedelta(hours=-2), -7200.0, b'-7200.0', {'-7200': 'foo'}, b'{"-7200":"foo"}', 'seconds_float'), - (timedelta(seconds=1.5), 1.5, b'1.5', {'1.5': 'foo'}, b'{"1.5":"foo"}', 'seconds_float'), - (timedelta(seconds=-1.5), -1.5, b'-1.5', {'-1.5': 'foo'}, b'{"-1.5":"foo"}', 'seconds_float'), - (timedelta(microseconds=1), 1e-6, b'1e-6', {'0.000001': 'foo'}, b'{"0.000001":"foo"}', 'seconds_float'), - ( - timedelta(microseconds=-1), - -1e-6, - b'-1e-6', - {'-0.000001': 'foo'}, - b'{"-0.000001":"foo"}', - 'seconds_float', - ), - (timedelta(days=1), 86400.0, b'86400.0', {'86400': 'foo'}, b'{"86400":"foo"}', 'seconds_float'), - (timedelta(days=-1), -86400.0, b'-86400.0', {'-86400': 'foo'}, b'{"-86400":"foo"}', 'seconds_float'), - (timedelta(days=1, seconds=1), 86401.0, b'86401.0', {'86401': 'foo'}, b'{"86401":"foo"}', 'seconds_float'), - ( - timedelta(days=-1, seconds=-1), - -86401.0, - b'-86401.0', - {'-86401': 'foo'}, - b'{"-86401":"foo"}', - 'seconds_float', - ), - (timedelta(days=1, seconds=-1), 86399.0, b'86399.0', {'86399': 'foo'}, b'{"86399":"foo"}', 'seconds_float'), - ( - timedelta(days=1, seconds=1, microseconds=1), - 86401.000001, - b'86401.000001', - {'86401.000001': 'foo'}, - b'{"86401.000001":"foo"}', - 'seconds_float', - ), - ( - timedelta(days=-1, seconds=-1, microseconds=-1), - -86401.000001, - b'-86401.000001', - {'-86401.000001': 'foo'}, - b'{"-86401.000001":"foo"}', - 'seconds_float', - ), - ], -) -def test_any_config_timedelta( - td: timedelta, expected_to_python, expected_to_json, expected_to_python_dict, expected_to_json_dict, mode -): - s = SchemaSerializer(core_schema.any_schema(), config={'ser_json_timedelta': mode}) - assert s.to_python(td) == td - assert s.to_python(td, mode='json') == expected_to_python - assert s.to_json(td) == expected_to_json +def test_any_config_timedelta_float(): + s = SchemaSerializer(core_schema.any_schema(), config={'ser_json_timedelta': 'float'}) + h2 = timedelta(hours=2) + assert s.to_python(h2) == h2 + assert s.to_python(h2, mode='json') == 7200.0 + assert s.to_json(h2) == b'7200.0' + + assert s.to_python({h2: 'foo'}) == {h2: 'foo'} + assert s.to_python({h2: 'foo'}, mode='json') == {'7200': 'foo'} + assert s.to_json({h2: 'foo'}) == b'{"7200":"foo"}' + - assert s.to_python({td: 'foo'}) == {td: 'foo'} - assert s.to_python({td: 'foo'}, mode='json') == expected_to_python_dict - assert s.to_json({td: 'foo'}) == expected_to_json_dict +def test_any_config_timedelta_float_faction(): + s = SchemaSerializer(core_schema.any_schema(), config={'ser_json_timedelta': 'float'}) + one_half_s = timedelta(seconds=1.5) + assert s.to_python(one_half_s) == one_half_s + assert s.to_python(one_half_s, mode='json') == 1.5 + assert s.to_json(one_half_s) == b'1.5' + + assert s.to_python({one_half_s: 'foo'}) == {one_half_s: 'foo'} + assert s.to_python({one_half_s: 'foo'}, mode='json') == {'1.5': 'foo'} + assert s.to_json({one_half_s: 'foo'}) == b'{"1.5":"foo"}' def test_recursion(any_serializer): @@ -533,13 +422,9 @@ def test_base64(): (lambda: datetime(2032, 1, 1), {}, b'"2032-01-01T00:00:00"'), (lambda: time(12, 34, 56), {}, b'"12:34:56"'), (lambda: timedelta(days=12, seconds=34, microseconds=56), {}, b'"P12DT34.000056S"'), - ( - lambda: timedelta(days=12, seconds=34, microseconds=56), - dict(timedelta_mode='seconds_float'), - b'1036834.000056', - ), + (lambda: timedelta(days=12, seconds=34, microseconds=56), dict(timedelta_mode='float'), b'1036834.000056'), (lambda: timedelta(seconds=-1), {}, b'"-PT1S"'), - (lambda: timedelta(seconds=-1), dict(timedelta_mode='seconds_float'), b'-1.0'), + (lambda: timedelta(seconds=-1), dict(timedelta_mode='float'), b'-1.0'), (lambda: {1, 2, 3}, {}, b'[1,2,3]'), (lambda: frozenset([1, 2, 3]), {}, b'[1,2,3]'), (lambda: (v for v in range(4)), {}, b'[0,1,2,3]'), diff --git a/tests/serializers/test_pickling.py b/tests/serializers/test_pickling.py index b6bc105d5..2ca230313 100644 --- a/tests/serializers/test_pickling.py +++ b/tests/serializers/test_pickling.py @@ -42,7 +42,7 @@ def test_schema_serializer_capturing_function(value, expected_python, expected_j def test_schema_serializer_containing_config(): - s = SchemaSerializer(core_schema.timedelta_schema(), config={'ser_json_timedelta': 'seconds_float'}) + s = SchemaSerializer(core_schema.timedelta_schema(), config={'ser_json_timedelta': 'float'}) s = pickle.loads(pickle.dumps(s)) assert s.to_python(timedelta(seconds=4, microseconds=500_000)) == timedelta(seconds=4, microseconds=500_000) diff --git a/tests/serializers/test_timedelta.py b/tests/serializers/test_timedelta.py index 9fcc9d362..19abb673d 100644 --- a/tests/serializers/test_timedelta.py +++ b/tests/serializers/test_timedelta.py @@ -31,7 +31,7 @@ def test_timedelta(): def test_timedelta_float(): - v = SchemaSerializer(core_schema.timedelta_schema(), config={'ser_json_timedelta': 'seconds_float'}) + v = SchemaSerializer(core_schema.timedelta_schema(), config={'ser_json_timedelta': 'float'}) assert v.to_python(timedelta(seconds=4, microseconds=500_000)) == timedelta(seconds=4, microseconds=500_000) assert v.to_python(timedelta(seconds=4, microseconds=500_000), mode='json') == 4.5 diff --git a/tests/test_garbage_collection.py b/tests/test_garbage_collection.py index 803979aa9..f4e178f26 100644 --- a/tests/test_garbage_collection.py +++ b/tests/test_garbage_collection.py @@ -28,7 +28,7 @@ class BaseModel: def __init_subclass__(cls) -> None: cls.__schema__ = SchemaSerializer( - core_schema.model_schema(cls, GC_TEST_SCHEMA_INNER), config={'ser_json_timedelta': 'seconds_float'} + core_schema.model_schema(cls, GC_TEST_SCHEMA_INNER), config={'ser_json_timedelta': 'float'} ) cache: 'WeakValueDictionary[int, Any]' = WeakValueDictionary()