Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add 'millisecond' option to ser_json_timedelta config parameter #1427

Merged
merged 23 commits into from
Sep 18, 2024
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions python/pydantic_core/_pydantic_core.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ def to_json(
by_alias: bool = True,
exclude_none: bool = False,
round_trip: bool = False,
timedelta_mode: Literal['iso8601', 'float'] = 'iso8601',
timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601',
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sydney-runkle as requested in pydantic pr, i've removed float from the type hints in various places.

bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8',
inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants',
serialize_unknown: bool = False,
Expand All @@ -377,7 +377,7 @@ def to_json(
by_alias: Whether to use the alias names of fields.
exclude_none: Whether to exclude fields that have a value of `None`.
round_trip: Whether to enable serialization and validation round-trip support.
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`.
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'` or `'milliseconds_float'`.
bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`.
inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`.
serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails
Expand Down Expand Up @@ -431,7 +431,7 @@ def to_jsonable_python(
by_alias: bool = True,
exclude_none: bool = False,
round_trip: bool = False,
timedelta_mode: Literal['iso8601', 'float'] = 'iso8601',
timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601',
bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8',
inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants',
serialize_unknown: bool = False,
Expand All @@ -452,7 +452,7 @@ def to_jsonable_python(
by_alias: Whether to use the alias names of fields.
exclude_none: Whether to exclude fields that have a value of `None`.
round_trip: Whether to enable serialization and validation round-trip support.
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`.
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'`, or`'milliseconds_float'`.
bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`.
inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`.
serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails
Expand Down
2 changes: 1 addition & 1 deletion python/pydantic_core/core_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ class CoreConfig(TypedDict, total=False):
# fields related to float fields only
allow_inf_nan: bool # default: True
# the config options are used to customise serialization to JSON
ser_json_timedelta: Literal['iso8601', 'float'] # default: 'iso8601'
ser_json_timedelta: Literal['iso8601', 'float', 'seconds_float', 'milliseconds_float'] # default: 'iso8601'
ser_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8'
ser_json_inf_nan: Literal['null', 'constants', 'strings'] # default: 'null'
val_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8'
Expand Down
28 changes: 28 additions & 0 deletions src/input/datetime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,34 @@ impl<'a> EitherTimedelta<'a> {
Self::Raw(duration) => duration_as_pytimedelta(py, duration),
}
}

pub fn total_seconds(&self) -> PyResult<f64> {
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
match self {
Self::Raw(timedelta) => {
let days: f64 = f64::from(timedelta.day);
let seconds: f64 = f64::from(timedelta.second);
let microseconds: f64 = f64::from(timedelta.microsecond);
let total_seconds: f64 = if !timedelta.positive {
-1.0 * (86400.0 * days + seconds + microseconds / 1_000_000.0)
} else {
86400.0 * days + seconds + microseconds / 1_000_000.0
};
Ok(total_seconds)
}
Self::PyExact(py_timedelta) => {
let days: f64 = f64::from(py_timedelta.get_days()); // -999999999 to 999999999
let seconds: f64 = f64::from(py_timedelta.get_seconds()); // 0 through 86399
let microseconds: f64 = f64::from(py_timedelta.get_microseconds()); // 0 through 999999
Ok(86400.0 * days + seconds + microseconds / 1_000_000.0)
}
Self::PySubclass(py_timedelta) => {
let total_seconds: f64 = py_timedelta
.call_method0(intern!(py_timedelta.py(), "total_seconds"))?
.extract()?;
Ok(total_seconds)
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
}
}
}
}

impl<'a> TryFrom<&'_ Bound<'a, PyAny>> for EitherTimedelta<'a> {
Expand Down
45 changes: 27 additions & 18 deletions src/serializers/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use std::str::{from_utf8, FromStr, Utf8Error};
use base64::Engine;
use pyo3::intern;
use pyo3::prelude::*;
use pyo3::types::{PyDelta, PyDict, PyString};
use pyo3::types::{PyDict, PyFloat, PyString};

use serde::ser::Error;

Expand Down Expand Up @@ -88,7 +88,8 @@ serialization_mode! {
TimedeltaMode,
"ser_json_timedelta",
Iso8601 => "iso8601",
Float => "float",
SecondsFloat => "seconds_float",
MillisecondsFloat => "milliseconds_float"
}

serialization_mode! {
Expand All @@ -108,43 +109,48 @@ serialization_mode! {
}

impl TimedeltaMode {
fn total_seconds<'py>(py_timedelta: &Bound<'py, PyDelta>) -> PyResult<Bound<'py, PyAny>> {
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
py_timedelta.call_method0(intern!(py_timedelta.py(), "total_seconds"))
}

pub fn either_delta_to_json(self, py: Python, either_delta: &EitherTimedelta) -> PyResult<PyObject> {
match self {
Self::Iso8601 => {
let d = either_delta.to_duration()?;
Ok(d.to_string().into_py(py))
}
Self::Float => {
Self::SecondsFloat => {
// convert to int via a py timedelta not duration since we know this this case the input would have
// been a py timedelta
let py_timedelta = either_delta.try_into_py(py)?;
let seconds = Self::total_seconds(&py_timedelta)?;
let seconds: f64 = either_delta.total_seconds()?;
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
Ok(seconds.into_py(py))
}
Self::MillisecondsFloat => {
// convert to int via a py timedelta not duration since we know this this case the input would have
// been a py timedelta
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
let seconds: f64 = either_delta.total_seconds()?;
let object: Bound<PyFloat> = PyFloat::new_bound(py, seconds * 1000.0);
Ok(object.into_py(py))
}
}
}

pub fn json_key<'py>(self, py: Python, either_delta: &EitherTimedelta) -> PyResult<Cow<'py, str>> {
pub fn json_key<'py>(self, either_delta: &EitherTimedelta) -> PyResult<Cow<'py, str>> {
match self {
Self::Iso8601 => {
let d = either_delta.to_duration()?;
Ok(d.to_string().into())
}
Self::Float => {
let py_timedelta = either_delta.try_into_py(py)?;
let seconds: f64 = Self::total_seconds(&py_timedelta)?.extract()?;
Self::SecondsFloat => {
let seconds: f64 = either_delta.total_seconds()?;
Ok(seconds.to_string().into())
}
Self::MillisecondsFloat => {
let seconds: f64 = either_delta.total_seconds()?;
let milliseconds: f64 = seconds * 1000.0;
Ok(milliseconds.to_string().into())
}
}
}

pub fn timedelta_serialize<S: serde::ser::Serializer>(
self,
py: Python,
either_delta: &EitherTimedelta,
serializer: S,
) -> Result<S::Ok, S::Error> {
Expand All @@ -153,12 +159,15 @@ impl TimedeltaMode {
let d = either_delta.to_duration().map_err(py_err_se_err)?;
serializer.serialize_str(&d.to_string())
}
Self::Float => {
let py_timedelta = either_delta.try_into_py(py).map_err(py_err_se_err)?;
let seconds = Self::total_seconds(&py_timedelta).map_err(py_err_se_err)?;
let seconds: f64 = seconds.extract().map_err(py_err_se_err)?;
Self::SecondsFloat => {
let seconds: f64 = either_delta.total_seconds().map_err(py_err_se_err)?;
serializer.serialize_f64(seconds)
}
Self::MillisecondsFloat => {
let seconds: f64 = either_delta.total_seconds().map_err(py_err_se_err)?;
let milliseconds: f64 = seconds * 1000.0;
serializer.serialize_f64(milliseconds)
}
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/serializers/infer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,7 @@ pub(crate) fn infer_serialize_known<S: Serializer>(
extra
.config
.timedelta_mode
.timedelta_serialize(value.py(), &either_delta, serializer)
.timedelta_serialize(&either_delta, serializer)
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
}
ObType::Url => {
let py_url: PyUrl = value.extract().map_err(py_err_se_err)?;
Expand Down Expand Up @@ -655,7 +655,7 @@ pub(crate) fn infer_json_key_known<'a>(
}
ObType::Timedelta => {
let either_delta = EitherTimedelta::try_from(key)?;
extra.config.timedelta_mode.json_key(key.py(), &either_delta)
extra.config.timedelta_mode.json_key(&either_delta)
}
ObType::Url => {
let py_url: PyUrl = key.extract()?;
Expand Down
6 changes: 2 additions & 4 deletions src/serializers/type_serializers/timedelta.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ impl TypeSerializer for TimeDeltaSerializer {

fn json_key<'a>(&self, key: &'a Bound<'_, PyAny>, extra: &Extra) -> PyResult<Cow<'a, str>> {
match EitherTimedelta::try_from(key) {
Ok(either_timedelta) => self.timedelta_mode.json_key(key.py(), &either_timedelta),
Ok(either_timedelta) => self.timedelta_mode.json_key(&either_timedelta),
Err(_) => {
extra.warnings.on_fallback_py(self.get_name(), key, extra)?;
infer_json_key(key, extra)
Expand All @@ -71,9 +71,7 @@ impl TypeSerializer for TimeDeltaSerializer {
extra: &Extra,
) -> Result<S::Ok, S::Error> {
match EitherTimedelta::try_from(value) {
Ok(either_timedelta) => self
.timedelta_mode
.timedelta_serialize(value.py(), &either_timedelta, serializer),
Ok(either_timedelta) => self.timedelta_mode.timedelta_serialize(&either_timedelta, serializer),
Err(_) => {
extra.warnings.on_fallback_ser::<S>(self.get_name(), value, extra)?;
infer_serialize(value, serializer, include, exclude, extra)
Expand Down
161 changes: 138 additions & 23 deletions tests/serializers/test_any.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,28 +177,139 @@ def test_any_with_timedelta_serializer():
]


def test_any_config_timedelta_float():
s = SchemaSerializer(core_schema.any_schema(), config={'ser_json_timedelta': 'float'})
h2 = timedelta(hours=2)
assert s.to_python(h2) == h2
assert s.to_python(h2, mode='json') == 7200.0
assert s.to_json(h2) == b'7200.0'

assert s.to_python({h2: 'foo'}) == {h2: 'foo'}
assert s.to_python({h2: 'foo'}, mode='json') == {'7200': 'foo'}
assert s.to_json({h2: 'foo'}) == b'{"7200":"foo"}'


def test_any_config_timedelta_float_faction():
s = SchemaSerializer(core_schema.any_schema(), config={'ser_json_timedelta': 'float'})
one_half_s = timedelta(seconds=1.5)
assert s.to_python(one_half_s) == one_half_s
assert s.to_python(one_half_s, mode='json') == 1.5
assert s.to_json(one_half_s) == b'1.5'
@pytest.mark.parametrize(
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
'td,expected_to_python,expected_to_json,expected_to_python_dict,expected_to_json_dict,mode',
[
(timedelta(hours=2), 7200000.0, b'7200000.0', {'7200000': 'foo'}, b'{"7200000":"foo"}', 'milliseconds_float'),
(
timedelta(hours=-2),
-7200000.0,
b'-7200000.0',
{'-7200000': 'foo'},
b'{"-7200000":"foo"}',
'milliseconds_float',
),
(timedelta(seconds=1.5), 1500.0, b'1500.0', {'1500': 'foo'}, b'{"1500":"foo"}', 'milliseconds_float'),
(timedelta(seconds=-1.5), -1500.0, b'-1500.0', {'-1500': 'foo'}, b'{"-1500":"foo"}', 'milliseconds_float'),
(timedelta(microseconds=1), 0.001, b'0.001', {'0.001': 'foo'}, b'{"0.001":"foo"}', 'milliseconds_float'),
(
timedelta(microseconds=-1),
-0.0010000000000287557,
b'-0.0010000000000287557',
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
{'-0.0010000000000287557': 'foo'},
b'{"-0.0010000000000287557":"foo"}',
'milliseconds_float',
),
(
timedelta(days=1),
86400000.0,
b'86400000.0',
{'86400000': 'foo'},
b'{"86400000":"foo"}',
'milliseconds_float',
),
(
timedelta(days=-1),
-86400000.0,
b'-86400000.0',
{'-86400000': 'foo'},
b'{"-86400000":"foo"}',
'milliseconds_float',
),
(
timedelta(days=1, seconds=1),
86401000.0,
b'86401000.0',
{'86401000': 'foo'},
b'{"86401000":"foo"}',
'milliseconds_float',
),
(
timedelta(days=-1, seconds=-1),
-86401000.0,
b'-86401000.0',
{'-86401000': 'foo'},
b'{"-86401000":"foo"}',
'milliseconds_float',
),
(
timedelta(days=1, seconds=-1),
86399000.0,
b'86399000.0',
{'86399000': 'foo'},
b'{"86399000":"foo"}',
'milliseconds_float',
),
(
timedelta(days=1, seconds=1, microseconds=1),
86401000.00099999,
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
b'86401000.00099999',
{'86401000.00099999': 'foo'},
b'{"86401000.00099999":"foo"}',
'milliseconds_float',
),
(
timedelta(days=-1, seconds=-1, microseconds=-1),
-86401000.00099999,
b'-86401000.00099999',
{'-86401000.00099999': 'foo'},
b'{"-86401000.00099999":"foo"}',
'milliseconds_float',
),
(timedelta(hours=2), 7200.0, b'7200.0', {'7200': 'foo'}, b'{"7200":"foo"}', 'seconds_float'),
(timedelta(hours=-2), -7200.0, b'-7200.0', {'-7200': 'foo'}, b'{"-7200":"foo"}', 'seconds_float'),
(timedelta(seconds=1.5), 1.5, b'1.5', {'1.5': 'foo'}, b'{"1.5":"foo"}', 'seconds_float'),
(timedelta(seconds=-1.5), -1.5, b'-1.5', {'-1.5': 'foo'}, b'{"-1.5":"foo"}', 'seconds_float'),
(timedelta(microseconds=1), 1e-6, b'1e-6', {'0.000001': 'foo'}, b'{"0.000001":"foo"}', 'seconds_float'),
(
timedelta(microseconds=-1),
-1.0000000000287557e-6,
b'-1.0000000000287557e-6',
{'-0.0000010000000000287557': 'foo'},
b'{"-0.0000010000000000287557":"foo"}',
'seconds_float',
),
(timedelta(days=1), 86400.0, b'86400.0', {'86400': 'foo'}, b'{"86400":"foo"}', 'seconds_float'),
(timedelta(days=-1), -86400.0, b'-86400.0', {'-86400': 'foo'}, b'{"-86400":"foo"}', 'seconds_float'),
(timedelta(days=1, seconds=1), 86401.0, b'86401.0', {'86401': 'foo'}, b'{"86401":"foo"}', 'seconds_float'),
(
timedelta(days=-1, seconds=-1),
-86401.0,
b'-86401.0',
{'-86401': 'foo'},
b'{"-86401":"foo"}',
'seconds_float',
),
(timedelta(days=1, seconds=-1), 86399.0, b'86399.0', {'86399': 'foo'}, b'{"86399":"foo"}', 'seconds_float'),
(
timedelta(days=1, seconds=1, microseconds=1),
86401.000001,
b'86401.000001',
{'86401.000001': 'foo'},
b'{"86401.000001":"foo"}',
'seconds_float',
),
(
timedelta(days=-1, seconds=-1, microseconds=-1),
-86401.000001,
b'-86401.000001',
{'-86401.000001': 'foo'},
b'{"-86401.000001":"foo"}',
'seconds_float',
),
],
)
def test_any_config_timedelta(
td: timedelta, expected_to_python, expected_to_json, expected_to_python_dict, expected_to_json_dict, mode
):
s = SchemaSerializer(core_schema.any_schema(), config={'ser_json_timedelta': mode})
assert s.to_python(td) == td
assert s.to_python(td, mode='json') == expected_to_python
assert s.to_json(td) == expected_to_json

assert s.to_python({one_half_s: 'foo'}) == {one_half_s: 'foo'}
assert s.to_python({one_half_s: 'foo'}, mode='json') == {'1.5': 'foo'}
assert s.to_json({one_half_s: 'foo'}) == b'{"1.5":"foo"}'
assert s.to_python({td: 'foo'}) == {td: 'foo'}
assert s.to_python({td: 'foo'}, mode='json') == expected_to_python_dict
assert s.to_json({td: 'foo'}) == expected_to_json_dict


def test_recursion(any_serializer):
Expand Down Expand Up @@ -422,9 +533,13 @@ def test_base64():
(lambda: datetime(2032, 1, 1), {}, b'"2032-01-01T00:00:00"'),
(lambda: time(12, 34, 56), {}, b'"12:34:56"'),
(lambda: timedelta(days=12, seconds=34, microseconds=56), {}, b'"P12DT34.000056S"'),
(lambda: timedelta(days=12, seconds=34, microseconds=56), dict(timedelta_mode='float'), b'1036834.000056'),
(
lambda: timedelta(days=12, seconds=34, microseconds=56),
dict(timedelta_mode='seconds_float'),
b'1036834.000056',
),
(lambda: timedelta(seconds=-1), {}, b'"-PT1S"'),
(lambda: timedelta(seconds=-1), dict(timedelta_mode='float'), b'-1.0'),
(lambda: timedelta(seconds=-1), dict(timedelta_mode='seconds_float'), b'-1.0'),
(lambda: {1, 2, 3}, {}, b'[1,2,3]'),
(lambda: frozenset([1, 2, 3]), {}, b'[1,2,3]'),
(lambda: (v for v in range(4)), {}, b'[0,1,2,3]'),
Expand Down
Loading
Loading