Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add 'millisecond' option to ser_json_timedelta config parameter #1427

Merged
merged 23 commits into from
Sep 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions python/pydantic_core/_pydantic_core.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ def to_json(
by_alias: bool = True,
exclude_none: bool = False,
round_trip: bool = False,
timedelta_mode: Literal['iso8601', 'float'] = 'iso8601',
timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601',
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sydney-runkle as requested in pydantic pr, i've removed float from the type hints in various places.

bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8',
inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants',
serialize_unknown: bool = False,
Expand All @@ -378,7 +378,7 @@ def to_json(
by_alias: Whether to use the alias names of fields.
exclude_none: Whether to exclude fields that have a value of `None`.
round_trip: Whether to enable serialization and validation round-trip support.
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`.
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'` or `'milliseconds_float'`.
bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`.
inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`.
serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails
Expand Down Expand Up @@ -432,7 +432,7 @@ def to_jsonable_python(
by_alias: bool = True,
exclude_none: bool = False,
round_trip: bool = False,
timedelta_mode: Literal['iso8601', 'float'] = 'iso8601',
timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601',
bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8',
inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants',
serialize_unknown: bool = False,
Expand All @@ -453,7 +453,7 @@ def to_jsonable_python(
by_alias: Whether to use the alias names of fields.
exclude_none: Whether to exclude fields that have a value of `None`.
round_trip: Whether to enable serialization and validation round-trip support.
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`.
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'`, or`'milliseconds_float'`.
bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`.
inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`.
serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails
Expand Down
2 changes: 1 addition & 1 deletion python/pydantic_core/core_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ class CoreConfig(TypedDict, total=False):
# fields related to float fields only
allow_inf_nan: bool # default: True
# the config options are used to customise serialization to JSON
ser_json_timedelta: Literal['iso8601', 'float'] # default: 'iso8601'
ser_json_timedelta: Literal['iso8601', 'seconds_float', 'milliseconds_float'] # default: 'iso8601'
ser_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8'
ser_json_inf_nan: Literal['null', 'constants', 'strings'] # default: 'null'
val_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8'
Expand Down
87 changes: 87 additions & 0 deletions src/input/datetime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,93 @@ impl<'a> EitherTimedelta<'a> {
Self::Raw(duration) => duration_as_pytimedelta(py, duration),
}
}

pub fn total_seconds(&self) -> PyResult<f64> {
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
match self {
Self::Raw(timedelta) => {
let mut days: i64 = i64::from(timedelta.day);
let mut seconds: i64 = i64::from(timedelta.second);
let mut microseconds = i64::from(timedelta.microsecond);
if !timedelta.positive {
days = -days;
seconds = -seconds;
microseconds = -microseconds;
}

let days_seconds = (86_400 * days) + seconds;
if let Some(days_seconds_as_micros) = days_seconds.checked_mul(1_000_000) {
let total_microseconds = days_seconds_as_micros + microseconds;
Ok(total_microseconds as f64 / 1_000_000.0)
} else {
// Fall back to floating-point operations if the multiplication overflows
let total_seconds = days_seconds as f64 + microseconds as f64 / 1_000_000.0;
Ok(total_seconds)
}
}
Self::PyExact(py_timedelta) => {
let days: i64 = py_timedelta.get_days().into(); // -999999999 to 999999999
let seconds: i64 = py_timedelta.get_seconds().into(); // 0 through 86399
let microseconds = py_timedelta.get_microseconds(); // 0 through 999999
let days_seconds = (86_400 * days) + seconds;
if let Some(days_seconds_as_micros) = days_seconds.checked_mul(1_000_000) {
let total_microseconds = days_seconds_as_micros + i64::from(microseconds);
Ok(total_microseconds as f64 / 1_000_000.0)
} else {
// Fall back to floating-point operations if the multiplication overflows
let total_seconds = days_seconds as f64 + f64::from(microseconds) / 1_000_000.0;
Ok(total_seconds)
}
}
Self::PySubclass(py_timedelta) => py_timedelta
.call_method0(intern!(py_timedelta.py(), "total_seconds"))?
.extract(),
}
}

pub fn total_milliseconds(&self) -> PyResult<f64> {
match self {
Self::Raw(timedelta) => {
let mut days: i64 = i64::from(timedelta.day);
let mut seconds: i64 = i64::from(timedelta.second);
let mut microseconds = i64::from(timedelta.microsecond);
if !timedelta.positive {
days = -days;
seconds = -seconds;
microseconds = -microseconds;
}

let days_seconds = (86_400 * days) + seconds;
if let Some(days_seconds_as_micros) = days_seconds.checked_mul(1_000_000) {
let total_microseconds = days_seconds_as_micros + microseconds;
Ok(total_microseconds as f64 / 1_000.0)
} else {
// Fall back to floating-point operations if the multiplication overflows
let total_seconds = days_seconds as f64 + microseconds as f64 / 1_000.0;
Ok(total_seconds)
}
}
Self::PyExact(py_timedelta) => {
let days: i64 = py_timedelta.get_days().into(); // -999999999 to 999999999
let seconds: i64 = py_timedelta.get_seconds().into(); // 0 through 86399
let microseconds = py_timedelta.get_microseconds(); // 0 through 999999
let days_seconds = (86_400 * days) + seconds;
if let Some(days_seconds_as_micros) = days_seconds.checked_mul(1_000_000) {
let total_microseconds = days_seconds_as_micros + i64::from(microseconds);
Ok(total_microseconds as f64 / 1_000.0)
} else {
// Fall back to floating-point operations if the multiplication overflows
let total_milliseconds = days_seconds as f64 * 1_000.0 + f64::from(microseconds) / 1_000.0;
Ok(total_milliseconds)
}
}
Self::PySubclass(py_timedelta) => {
let total_seconds: f64 = py_timedelta
.call_method0(intern!(py_timedelta.py(), "total_seconds"))?
.extract()?;
Ok(total_seconds / 1000.0)
}
}
}
}

impl<'a> TryFrom<&'_ Bound<'a, PyAny>> for EitherTimedelta<'a> {
Expand Down
42 changes: 22 additions & 20 deletions src/serializers/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use std::str::{from_utf8, FromStr, Utf8Error};
use base64::Engine;
use pyo3::intern;
use pyo3::prelude::*;
use pyo3::types::{PyDelta, PyDict, PyString};
use pyo3::types::{PyDict, PyString};

use serde::ser::Error;

Expand Down Expand Up @@ -88,7 +88,8 @@ serialization_mode! {
TimedeltaMode,
"ser_json_timedelta",
Iso8601 => "iso8601",
Float => "float",
SecondsFloat => "seconds_float",
MillisecondsFloat => "milliseconds_float"
}

serialization_mode! {
Expand All @@ -108,43 +109,42 @@ serialization_mode! {
}

impl TimedeltaMode {
fn total_seconds<'py>(py_timedelta: &Bound<'py, PyDelta>) -> PyResult<Bound<'py, PyAny>> {
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
py_timedelta.call_method0(intern!(py_timedelta.py(), "total_seconds"))
}

pub fn either_delta_to_json(self, py: Python, either_delta: &EitherTimedelta) -> PyResult<PyObject> {
match self {
Self::Iso8601 => {
let d = either_delta.to_duration()?;
Ok(d.to_string().into_py(py))
}
Self::Float => {
// convert to int via a py timedelta not duration since we know this this case the input would have
// been a py timedelta
let py_timedelta = either_delta.try_into_py(py)?;
let seconds = Self::total_seconds(&py_timedelta)?;
Self::SecondsFloat => {
let seconds: f64 = either_delta.total_seconds()?;
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
Ok(seconds.into_py(py))
}
Self::MillisecondsFloat => {
let milliseconds: f64 = either_delta.total_milliseconds()?;
Ok(milliseconds.into_py(py))
}
}
}

pub fn json_key<'py>(self, py: Python, either_delta: &EitherTimedelta) -> PyResult<Cow<'py, str>> {
pub fn json_key<'py>(self, either_delta: &EitherTimedelta) -> PyResult<Cow<'py, str>> {
match self {
Self::Iso8601 => {
let d = either_delta.to_duration()?;
Ok(d.to_string().into())
}
Self::Float => {
let py_timedelta = either_delta.try_into_py(py)?;
let seconds: f64 = Self::total_seconds(&py_timedelta)?.extract()?;
Self::SecondsFloat => {
let seconds: f64 = either_delta.total_seconds()?;
Ok(seconds.to_string().into())
}
Self::MillisecondsFloat => {
let milliseconds: f64 = either_delta.total_milliseconds()?;
Ok(milliseconds.to_string().into())
}
}
}

pub fn timedelta_serialize<S: serde::ser::Serializer>(
self,
py: Python,
either_delta: &EitherTimedelta,
serializer: S,
) -> Result<S::Ok, S::Error> {
Expand All @@ -153,12 +153,14 @@ impl TimedeltaMode {
let d = either_delta.to_duration().map_err(py_err_se_err)?;
serializer.serialize_str(&d.to_string())
}
Self::Float => {
let py_timedelta = either_delta.try_into_py(py).map_err(py_err_se_err)?;
let seconds = Self::total_seconds(&py_timedelta).map_err(py_err_se_err)?;
let seconds: f64 = seconds.extract().map_err(py_err_se_err)?;
Self::SecondsFloat => {
let seconds: f64 = either_delta.total_seconds().map_err(py_err_se_err)?;
serializer.serialize_f64(seconds)
}
Self::MillisecondsFloat => {
let milliseconds: f64 = either_delta.total_milliseconds().map_err(py_err_se_err)?;
serializer.serialize_f64(milliseconds)
}
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/serializers/infer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,7 @@ pub(crate) fn infer_serialize_known<S: Serializer>(
extra
.config
.timedelta_mode
.timedelta_serialize(value.py(), &either_delta, serializer)
.timedelta_serialize(&either_delta, serializer)
ollz272 marked this conversation as resolved.
Show resolved Hide resolved
}
ObType::Url => {
let py_url: PyUrl = value.extract().map_err(py_err_se_err)?;
Expand Down Expand Up @@ -655,7 +655,7 @@ pub(crate) fn infer_json_key_known<'a>(
}
ObType::Timedelta => {
let either_delta = EitherTimedelta::try_from(key)?;
extra.config.timedelta_mode.json_key(key.py(), &either_delta)
extra.config.timedelta_mode.json_key(&either_delta)
}
ObType::Url => {
let py_url: PyUrl = key.extract()?;
Expand Down
6 changes: 2 additions & 4 deletions src/serializers/type_serializers/timedelta.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ impl TypeSerializer for TimeDeltaSerializer {

fn json_key<'a>(&self, key: &'a Bound<'_, PyAny>, extra: &Extra) -> PyResult<Cow<'a, str>> {
match EitherTimedelta::try_from(key) {
Ok(either_timedelta) => self.timedelta_mode.json_key(key.py(), &either_timedelta),
Ok(either_timedelta) => self.timedelta_mode.json_key(&either_timedelta),
Err(_) => {
extra.warnings.on_fallback_py(self.get_name(), key, extra)?;
infer_json_key(key, extra)
Expand All @@ -71,9 +71,7 @@ impl TypeSerializer for TimeDeltaSerializer {
extra: &Extra,
) -> Result<S::Ok, S::Error> {
match EitherTimedelta::try_from(value) {
Ok(either_timedelta) => self
.timedelta_mode
.timedelta_serialize(value.py(), &either_timedelta, serializer),
Ok(either_timedelta) => self.timedelta_mode.timedelta_serialize(&either_timedelta, serializer),
Err(_) => {
extra.warnings.on_fallback_ser::<S>(self.get_name(), value, extra)?;
infer_serialize(value, serializer, include, exclude, extra)
Expand Down
Loading
Loading