From 61993fdc42621efdd9f17158638937cd95d3690b Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Tue, 19 Sep 2023 13:06:59 +0100 Subject: [PATCH] jiter compiling --- Cargo.lock | 15 ++++ Cargo.toml | 1 + src/errors/line_error.rs | 6 +- src/input/input_abstract.rs | 6 +- src/input/input_json.rs | 136 +++++++++++++++++------------------- src/input/input_python.rs | 19 ++--- src/input/mod.rs | 2 - src/input/return_enums.rs | 16 ++--- src/input/shared.rs | 8 ++- src/lazy_index_map.rs | 63 ----------------- src/lib.rs | 1 - src/lookup_key.rs | 18 ++--- 12 files changed, 124 insertions(+), 167 deletions(-) delete mode 100644 src/lazy_index_map.rs diff --git a/Cargo.lock b/Cargo.lock index 55224a954..b36b3d33d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -135,6 +135,20 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" +[[package]] +name = "jiter" +version = "0.0.2" +dependencies = [ + "ahash", + "indexmap", + "num-bigint", + "num-traits", + "pyo3", + "smallvec", + "strum", + "strum_macros", +] + [[package]] name = "libc" version = "0.2.147" @@ -248,6 +262,7 @@ dependencies = [ "base64", "enum_dispatch", "idna", + "jiter", "num-bigint", "pyo3", "pyo3-build-config", diff --git a/Cargo.toml b/Cargo.toml index a4dfda154..4dc2549f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ base64 = "0.21.4" num-bigint = "0.4.4" python3-dll-a = "0.2.7" uuid = "1.4.1" +jiter = {path = "../jiter", features = ["python"]} [lib] name = "_pydantic_core" diff --git a/src/errors/line_error.rs b/src/errors/line_error.rs index e5d3c7bac..3ee4c7894 100644 --- a/src/errors/line_error.rs +++ b/src/errors/line_error.rs @@ -2,7 +2,9 @@ use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; use pyo3::PyDowncastError; -use crate::input::{Input, JsonInput}; +use jiter::JsonValue; + +use crate::input::Input; use super::location::{LocItem, Location}; use super::types::ErrorType; @@ -147,7 +149,7 @@ impl<'a> ValLineError<'a> { #[derive(Clone)] pub enum InputValue<'a> { PyAny(&'a PyAny), - JsonInput(JsonInput), + JsonInput(JsonValue), String(&'a str), } diff --git a/src/input/input_abstract.rs b/src/input/input_abstract.rs index d799da473..0044445d7 100644 --- a/src/input/input_abstract.rs +++ b/src/input/input_abstract.rs @@ -3,12 +3,14 @@ use std::fmt; use pyo3::types::{PyDict, PyType}; use pyo3::{intern, prelude::*}; +use jiter::JsonValue; + use crate::errors::{InputValue, LocItem, ValResult}; use crate::{PyMultiHostUrl, PyUrl}; use super::datetime::{EitherDate, EitherDateTime, EitherTime, EitherTimedelta}; use super::return_enums::{EitherBytes, EitherInt, EitherString}; -use super::{EitherFloat, GenericArguments, GenericIterable, GenericIterator, GenericMapping, JsonInput}; +use super::{EitherFloat, GenericArguments, GenericIterable, GenericIterator, GenericMapping}; #[derive(Debug, Clone, Copy)] pub enum InputType { @@ -70,7 +72,7 @@ pub trait Input<'a>: fmt::Debug + ToPyObject { fn validate_dataclass_args(&'a self, dataclass_name: &str) -> ValResult<'a, GenericArguments<'a>>; - fn parse_json(&'a self) -> ValResult<'a, JsonInput>; + fn parse_json(&'a self) -> ValResult<'a, JsonValue>; fn validate_str(&'a self, strict: bool) -> ValResult> { if strict { diff --git a/src/input/input_json.rs b/src/input/input_json.rs index d948e2493..5bc997067 100644 --- a/src/input/input_json.rs +++ b/src/input/input_json.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; +use jiter::{JsonArray, JsonValue}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyString}; use speedate::MicrosecondsPrecisionOverflowBehavior; @@ -12,36 +13,35 @@ use super::datetime::{ bytes_as_date, bytes_as_datetime, bytes_as_time, bytes_as_timedelta, float_as_datetime, float_as_duration, float_as_time, int_as_datetime, int_as_duration, int_as_time, EitherDate, EitherDateTime, EitherTime, }; -use super::parse_json::JsonArray; use super::shared::{float_as_int, int_as_bool, map_json_err, str_as_bool, str_as_float, str_as_int}; use super::{ EitherBytes, EitherFloat, EitherInt, EitherString, EitherTimedelta, GenericArguments, GenericIterable, - GenericIterator, GenericMapping, Input, JsonArgs, JsonInput, + GenericIterator, GenericMapping, Input, JsonArgs, }; -impl<'a> Input<'a> for JsonInput { +impl<'a> Input<'a> for JsonValue { /// This is required by since JSON object keys are always strings, I don't think it can be called #[cfg_attr(has_coverage_attribute, coverage(off))] fn as_loc_item(&self) -> LocItem { match self { - JsonInput::Int(i) => (*i).into(), - JsonInput::String(s) => s.as_str().into(), + JsonValue::Int(i) => (*i).into(), + JsonValue::String(s) => s.as_str().into(), v => format!("{v:?}").into(), } } fn as_error_value(&'a self) -> InputValue<'a> { - // cloning JsonInput is cheap due to use of Arc + // cloning JsonValue is cheap due to use of Arc InputValue::JsonInput(self.clone()) } fn is_none(&self) -> bool { - matches!(self, JsonInput::Null) + matches!(self, JsonValue::Null) } fn as_kwargs(&'a self, py: Python<'a>) -> Option<&'a PyDict> { match self { - JsonInput::Object(object) => { + JsonValue::Object(object) => { let dict = PyDict::new(py); for (k, v) in object.iter() { dict.set_item(k, v.to_object(py)).unwrap(); @@ -54,15 +54,15 @@ impl<'a> Input<'a> for JsonInput { fn validate_args(&'a self) -> ValResult<'a, GenericArguments<'a>> { match self { - JsonInput::Object(object) => Ok(JsonArgs::new(None, Some(object)).into()), - JsonInput::Array(array) => Ok(JsonArgs::new(Some(array), None).into()), + JsonValue::Object(object) => Ok(JsonArgs::new(None, Some(object)).into()), + JsonValue::Array(array) => Ok(JsonArgs::new(Some(array), None).into()), _ => Err(ValError::new(ErrorTypeDefaults::ArgumentsType, self)), } } fn validate_dataclass_args(&'a self, class_name: &str) -> ValResult<'a, GenericArguments<'a>> { match self { - JsonInput::Object(object) => Ok(JsonArgs::new(None, Some(object)).into()), + JsonValue::Object(object) => Ok(JsonArgs::new(None, Some(object)).into()), _ => { let class_name = class_name.to_string(); Err(ValError::new( @@ -76,29 +76,29 @@ impl<'a> Input<'a> for JsonInput { } } - fn parse_json(&'a self) -> ValResult<'a, JsonInput> { + fn parse_json(&'a self) -> ValResult<'a, JsonValue> { match self { - JsonInput::String(s) => serde_json::from_str(s.as_str()).map_err(|e| map_json_err(self, e)), + JsonValue::String(s) => JsonValue::parse(s.as_bytes()).map_err(|e| map_json_err(self, e)), _ => Err(ValError::new(ErrorTypeDefaults::JsonType, self)), } } fn strict_str(&'a self) -> ValResult> { match self { - JsonInput::String(s) => Ok(s.as_str().into()), + JsonValue::String(s) => Ok(s.as_str().into()), _ => Err(ValError::new(ErrorTypeDefaults::StringType, self)), } } fn lax_str(&'a self) -> ValResult> { match self { - JsonInput::String(s) => Ok(s.as_str().into()), + JsonValue::String(s) => Ok(s.as_str().into()), _ => Err(ValError::new(ErrorTypeDefaults::StringType, self)), } } fn validate_bytes(&'a self, _strict: bool) -> ValResult> { match self { - JsonInput::String(s) => Ok(s.as_bytes().into()), + JsonValue::String(s) => Ok(s.as_bytes().into()), _ => Err(ValError::new(ErrorTypeDefaults::BytesType, self)), } } @@ -109,16 +109,16 @@ impl<'a> Input<'a> for JsonInput { fn strict_bool(&self) -> ValResult { match self { - JsonInput::Bool(b) => Ok(*b), + JsonValue::Bool(b) => Ok(*b), _ => Err(ValError::new(ErrorTypeDefaults::BoolType, self)), } } fn lax_bool(&self) -> ValResult { match self { - JsonInput::Bool(b) => Ok(*b), - JsonInput::String(s) => str_as_bool(self, s), - JsonInput::Int(int) => int_as_bool(self, *int), - JsonInput::Float(float) => match float_as_int(self, *float) { + JsonValue::Bool(b) => Ok(*b), + JsonValue::String(s) => str_as_bool(self, s), + JsonValue::Int(int) => int_as_bool(self, *int), + JsonValue::Float(float) => match float_as_int(self, *float) { Ok(int) => int .as_bool() .ok_or_else(|| ValError::new(ErrorTypeDefaults::BoolParsing, self)), @@ -130,60 +130,56 @@ impl<'a> Input<'a> for JsonInput { fn strict_int(&'a self) -> ValResult> { match self { - JsonInput::Int(i) => Ok(EitherInt::I64(*i)), - JsonInput::Uint(u) => Ok(EitherInt::U64(*u)), - JsonInput::BigInt(b) => Ok(EitherInt::BigInt(b.clone())), + JsonValue::Int(i) => Ok(EitherInt::I64(*i)), + JsonValue::BigInt(b) => Ok(EitherInt::BigInt(b.clone())), _ => Err(ValError::new(ErrorTypeDefaults::IntType, self)), } } fn lax_int(&'a self) -> ValResult> { match self { - JsonInput::Bool(b) => match *b { + JsonValue::Bool(b) => match *b { true => Ok(EitherInt::I64(1)), false => Ok(EitherInt::I64(0)), }, - JsonInput::Int(i) => Ok(EitherInt::I64(*i)), - JsonInput::Uint(u) => Ok(EitherInt::U64(*u)), - JsonInput::BigInt(b) => Ok(EitherInt::BigInt(b.clone())), - JsonInput::Float(f) => float_as_int(self, *f), - JsonInput::String(str) => str_as_int(self, str), + JsonValue::Int(i) => Ok(EitherInt::I64(*i)), + JsonValue::BigInt(b) => Ok(EitherInt::BigInt(b.clone())), + JsonValue::Float(f) => float_as_int(self, *f), + JsonValue::String(str) => str_as_int(self, str), _ => Err(ValError::new(ErrorTypeDefaults::IntType, self)), } } fn ultra_strict_float(&'a self) -> ValResult> { match self { - JsonInput::Float(f) => Ok(EitherFloat::F64(*f)), + JsonValue::Float(f) => Ok(EitherFloat::F64(*f)), _ => Err(ValError::new(ErrorTypeDefaults::FloatType, self)), } } fn strict_float(&'a self) -> ValResult> { match self { - JsonInput::Float(f) => Ok(EitherFloat::F64(*f)), - JsonInput::Int(i) => Ok(EitherFloat::F64(*i as f64)), - JsonInput::Uint(u) => Ok(EitherFloat::F64(*u as f64)), + JsonValue::Float(f) => Ok(EitherFloat::F64(*f)), + JsonValue::Int(i) => Ok(EitherFloat::F64(*i as f64)), _ => Err(ValError::new(ErrorTypeDefaults::FloatType, self)), } } fn lax_float(&'a self) -> ValResult> { match self { - JsonInput::Bool(b) => match *b { + JsonValue::Bool(b) => match *b { true => Ok(EitherFloat::F64(1.0)), false => Ok(EitherFloat::F64(0.0)), }, - JsonInput::Float(f) => Ok(EitherFloat::F64(*f)), - JsonInput::Int(i) => Ok(EitherFloat::F64(*i as f64)), - JsonInput::Uint(u) => Ok(EitherFloat::F64(*u as f64)), - JsonInput::String(str) => str_as_float(self, str), + JsonValue::Float(f) => Ok(EitherFloat::F64(*f)), + JsonValue::Int(i) => Ok(EitherFloat::F64(*i as f64)), + JsonValue::String(str) => str_as_float(self, str), _ => Err(ValError::new(ErrorTypeDefaults::FloatType, self)), } } fn strict_decimal(&'a self, py: Python<'a>) -> ValResult<&'a PyAny> { match self { - JsonInput::Float(f) => create_decimal(PyString::new(py, &f.to_string()), self, py), + JsonValue::Float(f) => create_decimal(PyString::new(py, &f.to_string()), self, py), - JsonInput::String(..) | JsonInput::Int(..) | JsonInput::Uint(..) | JsonInput::BigInt(..) => { + JsonValue::String(..) | JsonValue::Int(..) | JsonValue::BigInt(..) => { create_decimal(self.to_object(py).into_ref(py), self, py) } _ => Err(ValError::new(ErrorTypeDefaults::DecimalType, self)), @@ -192,7 +188,7 @@ impl<'a> Input<'a> for JsonInput { fn validate_dict(&'a self, _strict: bool) -> ValResult> { match self { - JsonInput::Object(dict) => Ok(dict.into()), + JsonValue::Object(dict) => Ok(dict.into()), _ => Err(ValError::new(ErrorTypeDefaults::DictType, self)), } } @@ -203,7 +199,7 @@ impl<'a> Input<'a> for JsonInput { fn validate_list(&'a self, _strict: bool) -> ValResult> { match self { - JsonInput::Array(a) => Ok(GenericIterable::JsonArray(a)), + JsonValue::Array(a) => Ok(GenericIterable::JsonArray(a)), _ => Err(ValError::new(ErrorTypeDefaults::ListType, self)), } } @@ -215,7 +211,7 @@ impl<'a> Input<'a> for JsonInput { fn validate_tuple(&'a self, _strict: bool) -> ValResult> { // just as in set's case, List has to be allowed match self { - JsonInput::Array(a) => Ok(GenericIterable::JsonArray(a)), + JsonValue::Array(a) => Ok(GenericIterable::JsonArray(a)), _ => Err(ValError::new(ErrorTypeDefaults::TupleType, self)), } } @@ -227,7 +223,7 @@ impl<'a> Input<'a> for JsonInput { fn validate_set(&'a self, _strict: bool) -> ValResult> { // we allow a list here since otherwise it would be impossible to create a set from JSON match self { - JsonInput::Array(a) => Ok(GenericIterable::JsonArray(a)), + JsonValue::Array(a) => Ok(GenericIterable::JsonArray(a)), _ => Err(ValError::new(ErrorTypeDefaults::SetType, self)), } } @@ -239,7 +235,7 @@ impl<'a> Input<'a> for JsonInput { fn validate_frozenset(&'a self, _strict: bool) -> ValResult> { // we allow a list here since otherwise it would be impossible to create a frozenset from JSON match self { - JsonInput::Array(a) => Ok(GenericIterable::JsonArray(a)), + JsonValue::Array(a) => Ok(GenericIterable::JsonArray(a)), _ => Err(ValError::new(ErrorTypeDefaults::FrozenSetType, self)), } } @@ -250,20 +246,20 @@ impl<'a> Input<'a> for JsonInput { fn extract_generic_iterable(&self) -> ValResult { match self { - JsonInput::Array(a) => Ok(GenericIterable::JsonArray(a)), - JsonInput::String(s) => Ok(GenericIterable::JsonString(s)), - JsonInput::Object(object) => Ok(GenericIterable::JsonObject(object)), + JsonValue::Array(a) => Ok(GenericIterable::JsonArray(a)), + JsonValue::String(s) => Ok(GenericIterable::JsonString(s)), + JsonValue::Object(object) => Ok(GenericIterable::JsonObject(object)), _ => Err(ValError::new(ErrorTypeDefaults::IterableType, self)), } } fn validate_iter(&self) -> ValResult { match self { - JsonInput::Array(a) => Ok(a.clone().into()), - JsonInput::String(s) => Ok(string_to_vec(s).into()), - JsonInput::Object(object) => { + JsonValue::Array(a) => Ok(a.clone().into()), + JsonValue::String(s) => Ok(string_to_vec(s).into()), + JsonValue::Object(object) => { // return keys iterator to match python's behavior - let keys: JsonArray = JsonArray::new(object.keys().map(|k| JsonInput::String(k.clone())).collect()); + let keys: JsonArray = JsonArray::new(object.keys().map(|k| JsonValue::String(k.clone())).collect()); Ok(keys.into()) } _ => Err(ValError::new(ErrorTypeDefaults::IterableType, self)), @@ -272,7 +268,7 @@ impl<'a> Input<'a> for JsonInput { fn validate_date(&self, _strict: bool) -> ValResult { match self { - JsonInput::String(v) => bytes_as_date(self, v.as_bytes()), + JsonValue::String(v) => bytes_as_date(self, v.as_bytes()), _ => Err(ValError::new(ErrorTypeDefaults::DateType, self)), } } @@ -288,16 +284,16 @@ impl<'a> Input<'a> for JsonInput { microseconds_overflow_behavior: MicrosecondsPrecisionOverflowBehavior, ) -> ValResult { match self { - JsonInput::String(v) => bytes_as_time(self, v.as_bytes(), microseconds_overflow_behavior), + JsonValue::String(v) => bytes_as_time(self, v.as_bytes(), microseconds_overflow_behavior), _ => Err(ValError::new(ErrorTypeDefaults::TimeType, self)), } } fn lax_time(&self, microseconds_overflow_behavior: MicrosecondsPrecisionOverflowBehavior) -> ValResult { match self { - JsonInput::String(v) => bytes_as_time(self, v.as_bytes(), microseconds_overflow_behavior), - JsonInput::Int(v) => int_as_time(self, *v, 0), - JsonInput::Float(v) => float_as_time(self, *v), - JsonInput::BigInt(_) => Err(ValError::new( + JsonValue::String(v) => bytes_as_time(self, v.as_bytes(), microseconds_overflow_behavior), + JsonValue::Int(v) => int_as_time(self, *v, 0), + JsonValue::Float(v) => float_as_time(self, *v), + JsonValue::BigInt(_) => Err(ValError::new( ErrorType::TimeParsing { error: Cow::Borrowed( speedate::ParseError::TimeTooLarge @@ -317,7 +313,7 @@ impl<'a> Input<'a> for JsonInput { microseconds_overflow_behavior: MicrosecondsPrecisionOverflowBehavior, ) -> ValResult { match self { - JsonInput::String(v) => bytes_as_datetime(self, v.as_bytes(), microseconds_overflow_behavior), + JsonValue::String(v) => bytes_as_datetime(self, v.as_bytes(), microseconds_overflow_behavior), _ => Err(ValError::new(ErrorTypeDefaults::DatetimeType, self)), } } @@ -326,9 +322,9 @@ impl<'a> Input<'a> for JsonInput { microseconds_overflow_behavior: MicrosecondsPrecisionOverflowBehavior, ) -> ValResult { match self { - JsonInput::String(v) => bytes_as_datetime(self, v.as_bytes(), microseconds_overflow_behavior), - JsonInput::Int(v) => int_as_datetime(self, *v, 0), - JsonInput::Float(v) => float_as_datetime(self, *v), + JsonValue::String(v) => bytes_as_datetime(self, v.as_bytes(), microseconds_overflow_behavior), + JsonValue::Int(v) => int_as_datetime(self, *v, 0), + JsonValue::Float(v) => float_as_datetime(self, *v), _ => Err(ValError::new(ErrorTypeDefaults::DatetimeType, self)), } } @@ -338,7 +334,7 @@ impl<'a> Input<'a> for JsonInput { microseconds_overflow_behavior: MicrosecondsPrecisionOverflowBehavior, ) -> ValResult { match self { - JsonInput::String(v) => bytes_as_timedelta(self, v.as_bytes(), microseconds_overflow_behavior), + JsonValue::String(v) => bytes_as_timedelta(self, v.as_bytes(), microseconds_overflow_behavior), _ => Err(ValError::new(ErrorTypeDefaults::TimeDeltaType, self)), } } @@ -347,9 +343,9 @@ impl<'a> Input<'a> for JsonInput { microseconds_overflow_behavior: MicrosecondsPrecisionOverflowBehavior, ) -> ValResult { match self { - JsonInput::String(v) => bytes_as_timedelta(self, v.as_bytes(), microseconds_overflow_behavior), - JsonInput::Int(v) => Ok(int_as_duration(self, *v)?.into()), - JsonInput::Float(v) => Ok(float_as_duration(self, *v)?.into()), + JsonValue::String(v) => bytes_as_timedelta(self, v.as_bytes(), microseconds_overflow_behavior), + JsonValue::Int(v) => Ok(int_as_duration(self, *v)?.into()), + JsonValue::Float(v) => Ok(float_as_duration(self, *v)?.into()), _ => Err(ValError::new(ErrorTypeDefaults::TimeDeltaType, self)), } } @@ -391,8 +387,8 @@ impl<'a> Input<'a> for String { )) } - fn parse_json(&'a self) -> ValResult<'a, JsonInput> { - serde_json::from_str(self.as_str()).map_err(|e| map_json_err(self, e)) + fn parse_json(&'a self) -> ValResult<'a, JsonValue> { + JsonValue::parse(self.as_bytes()).map_err(|e| map_json_err(self, e)) } fn validate_str(&'a self, _strict: bool) -> ValResult> { @@ -551,5 +547,5 @@ impl<'a> Input<'a> for String { } fn string_to_vec(s: &str) -> JsonArray { - JsonArray::new(s.chars().map(|c| JsonInput::String(c.to_string())).collect()) + JsonArray::new(s.chars().map(|c| JsonValue::String(c.to_string())).collect()) } diff --git a/src/input/input_python.rs b/src/input/input_python.rs index 3fbf20240..c6d97e3bd 100644 --- a/src/input/input_python.rs +++ b/src/input/input_python.rs @@ -9,6 +9,8 @@ use pyo3::types::{ #[cfg(not(PyPy))] use pyo3::types::{PyDictItems, PyDictKeys, PyDictValues}; use pyo3::{intern, AsPyPointer, PyTypeInfo}; + +use jiter::JsonValue; use speedate::MicrosecondsPrecisionOverflowBehavior; use crate::errors::{ErrorType, ErrorTypeDefaults, InputValue, LocItem, ValError, ValResult}; @@ -24,7 +26,7 @@ use super::datetime::{ use super::shared::{decimal_as_int, float_as_int, int_as_bool, map_json_err, str_as_bool, str_as_float, str_as_int}; use super::{ py_string_str, EitherBytes, EitherFloat, EitherInt, EitherString, EitherTimedelta, GenericArguments, - GenericIterable, GenericIterator, GenericMapping, Input, JsonInput, PyArgs, + GenericIterable, GenericIterator, GenericMapping, Input, PyArgs, }; #[cfg(not(PyPy))] @@ -180,19 +182,20 @@ impl<'a> Input<'a> for PyAny { } } - fn parse_json(&'a self) -> ValResult<'a, JsonInput> { - if let Ok(py_bytes) = self.downcast::() { - serde_json::from_slice(py_bytes.as_bytes()).map_err(|e| map_json_err(self, e)) + fn parse_json(&'a self) -> ValResult<'a, JsonValue> { + let bytes = if let Ok(py_bytes) = self.downcast::() { + py_bytes.as_bytes() } else if let Ok(py_str) = self.downcast::() { let str = py_str.to_str()?; - serde_json::from_str(str).map_err(|e| map_json_err(self, e)) + str.as_bytes() } else if let Ok(py_byte_array) = self.downcast::() { // Safety: from_slice does not run arbitrary Python code and the GIL is held so the // bytes array will not be mutated while from_slice is reading it - serde_json::from_slice(unsafe { py_byte_array.as_bytes() }).map_err(|e| map_json_err(self, e)) + unsafe { py_byte_array.as_bytes() } } else { - Err(ValError::new(ErrorTypeDefaults::JsonType, self)) - } + return Err(ValError::new(ErrorTypeDefaults::JsonType, self)); + }; + JsonValue::parse(bytes).map_err(|e| map_json_err(self, e)) } fn strict_str(&'a self) -> ValResult> { diff --git a/src/input/mod.rs b/src/input/mod.rs index c15f54b2a..bbe644869 100644 --- a/src/input/mod.rs +++ b/src/input/mod.rs @@ -6,7 +6,6 @@ mod datetime; mod input_abstract; mod input_json; mod input_python; -mod parse_json; mod return_enums; mod shared; @@ -16,7 +15,6 @@ pub(crate) use datetime::{ EitherTime, EitherTimedelta, }; pub(crate) use input_abstract::{Input, InputType}; -pub(crate) use parse_json::{JsonInput, JsonObject}; pub(crate) use return_enums::{ py_string_str, AttributesGenericIterator, DictGenericIterator, EitherBytes, EitherFloat, EitherInt, EitherString, GenericArguments, GenericIterable, GenericIterator, GenericMapping, Int, JsonArgs, JsonObjectGenericIterator, diff --git a/src/input/return_enums.rs b/src/input/return_enums.rs index e97fe8b81..80c6c5c82 100644 --- a/src/input/return_enums.rs +++ b/src/input/return_enums.rs @@ -4,6 +4,7 @@ use std::ops::Rem; use std::slice::Iter as SliceIter; use std::str::FromStr; +use jiter::{JsonArray, JsonObject, JsonValue}; use num_bigint::BigInt; use pyo3::exceptions::PyTypeError; @@ -25,7 +26,6 @@ use crate::errors::{py_err_string, ErrorType, ErrorTypeDefaults, InputValue, Val use crate::tools::py_err; use crate::validators::{CombinedValidator, ValidationState, Validator}; -use super::parse_json::{JsonArray, JsonInput, JsonObject}; use super::{py_error_on_minusone, Input}; /// Container for all the collections (sized iterable containers) types, which @@ -49,7 +49,7 @@ pub enum GenericIterable<'a> { PyByteArray(&'a PyByteArray), Sequence(&'a PySequence), Iterator(&'a PyIterator), - JsonArray(&'a [JsonInput]), + JsonArray(&'a [JsonValue]), JsonObject(&'a JsonObject), JsonString(&'a String), } @@ -561,7 +561,7 @@ impl<'py> Iterator for AttributesGenericIterator<'py> { } pub struct JsonObjectGenericIterator<'py> { - object_iter: SliceIter<'py, (String, JsonInput)>, + object_iter: SliceIter<'py, (String, JsonValue)>, } impl<'py> JsonObjectGenericIterator<'py> { @@ -573,7 +573,7 @@ impl<'py> JsonObjectGenericIterator<'py> { } impl<'py> Iterator for JsonObjectGenericIterator<'py> { - type Item = ValResult<'py, (&'py String, &'py JsonInput)>; + type Item = ValResult<'py, (&'py String, &'py JsonValue)>; fn next(&mut self) -> Option { self.object_iter.next().map(|(key, value)| Ok((key, value))) @@ -641,7 +641,7 @@ pub struct GenericJsonIterator { } impl GenericJsonIterator { - pub fn next(&mut self, _py: Python) -> PyResult> { + pub fn next(&mut self, _py: Python) -> PyResult> { if self.index < self.array.len() { // panic here is impossible due to bounds check above; compiler should be // able to optimize it away even @@ -655,7 +655,7 @@ impl GenericJsonIterator { } pub fn input_as_error_value<'py>(&self, _py: Python<'py>) -> InputValue<'py> { - InputValue::JsonInput(JsonInput::Array(self.array.clone())) + InputValue::JsonInput(JsonValue::Array(self.array.clone())) } pub fn index(&self) -> usize { @@ -677,12 +677,12 @@ impl<'a> PyArgs<'a> { #[cfg_attr(debug_assertions, derive(Debug))] pub struct JsonArgs<'a> { - pub args: Option<&'a [JsonInput]>, + pub args: Option<&'a [JsonValue]>, pub kwargs: Option<&'a JsonObject>, } impl<'a> JsonArgs<'a> { - pub fn new(args: Option<&'a [JsonInput]>, kwargs: Option<&'a JsonObject>) -> Self { + pub fn new(args: Option<&'a [JsonValue]>, kwargs: Option<&'a JsonObject>) -> Self { Self { args, kwargs } } } diff --git a/src/input/shared.rs b/src/input/shared.rs index d8733bd31..7e93ca72f 100644 --- a/src/input/shared.rs +++ b/src/input/shared.rs @@ -1,15 +1,17 @@ -use num_bigint::BigInt; use pyo3::{intern, PyAny, Python}; +use jiter::JsonValueError; +use num_bigint::BigInt; + use crate::errors::{ErrorType, ErrorTypeDefaults, ValError, ValResult}; use crate::input::EitherInt; use super::{EitherFloat, Input}; -pub fn map_json_err<'a>(input: &'a impl Input<'a>, error: serde_json::Error) -> ValError<'a> { +pub fn map_json_err<'a>(input: &'a impl Input<'a>, error: JsonValueError) -> ValError<'a> { ValError::new( ErrorType::JsonInvalid { - error: error.to_string(), + error: error.error_type.to_string(), context: None, }, input, diff --git a/src/lazy_index_map.rs b/src/lazy_index_map.rs deleted file mode 100644 index c5621f877..000000000 --- a/src/lazy_index_map.rs +++ /dev/null @@ -1,63 +0,0 @@ -use std::borrow::Borrow; -use std::cmp::{Eq, PartialEq}; -use std::fmt::Debug; -use std::hash::Hash; -use std::slice::Iter as SliceIter; -use std::sync::OnceLock; - -use ahash::AHashMap; -use smallvec::SmallVec; - -#[derive(Debug, Clone, Default)] -pub struct LazyIndexMap { - vec: SmallVec<[(K, V); 8]>, - map: OnceLock>, -} - -/// Like [IndexMap](https://docs.rs/indexmap/latest/indexmap/) but only builds the lookup map when it's needed. -impl LazyIndexMap -where - K: Clone + Debug + Eq + Hash, - V: Debug, -{ - pub fn new() -> Self { - Self { - vec: SmallVec::new(), - map: OnceLock::new(), - } - } - - pub fn insert(&mut self, key: K, value: V) { - if let Some(map) = self.map.get_mut() { - map.insert(key.clone(), self.vec.len()); - } - self.vec.push((key, value)); - } - - pub fn len(&self) -> usize { - self.vec.len() - } - - pub fn get(&self, key: &Q) -> Option<&V> - where - K: Borrow + PartialEq, - Q: Hash + Eq, - { - let map = self.map.get_or_init(|| { - self.vec - .iter() - .enumerate() - .map(|(index, (key, _))| (key.clone(), index)) - .collect() - }); - map.get(key).map(|&i| &self.vec[i].1) - } - - pub fn keys(&self) -> impl Iterator { - self.vec.iter().map(|(k, _)| k) - } - - pub fn iter(&self) -> SliceIter<'_, (K, V)> { - self.vec.iter() - } -} diff --git a/src/lib.rs b/src/lib.rs index cbf668b9d..c30034846 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,6 @@ mod build_tools; mod definitions; mod errors; mod input; -mod lazy_index_map; mod lookup_key; mod recursion_guard; mod serializers; diff --git a/src/lookup_key.rs b/src/lookup_key.rs index 49915e3b4..df43d192f 100644 --- a/src/lookup_key.rs +++ b/src/lookup_key.rs @@ -5,9 +5,11 @@ use pyo3::exceptions::{PyAttributeError, PyTypeError}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList, PyMapping, PyString}; +use jiter::{JsonObject, JsonValue}; + use crate::build_tools::py_schema_err; use crate::errors::{ErrorType, ValLineError}; -use crate::input::{Input, JsonInput, JsonObject}; +use crate::input::Input; use crate::tools::{extract_i64, py_err}; /// Used for getting items from python dicts, python objects, or JSON objects, in different ways @@ -235,7 +237,7 @@ impl LookupKey { pub fn json_get<'data, 's>( &'s self, dict: &'data JsonObject, - ) -> PyResult> { + ) -> PyResult> { match self { Self::Simple { key, path, .. } => match dict.get(key) { Some(value) => Ok(Some((path, value))), @@ -260,13 +262,13 @@ impl LookupKey { // first step is different from the rest as we already know dict is JsonObject // because of above checks, we know that path should have at least one element, hence unwrap - let v: &JsonInput = match path_iter.next().unwrap().json_obj_get(dict) { + let v: &JsonValue = match path_iter.next().unwrap().json_obj_get(dict) { Some(v) => v, None => continue, }; // similar to above - // iterate over the path and plug each value into the JsonInput from the last step, starting with v + // iterate over the path and plug each value into the JsonValue from the last step, starting with v // from the first step, this could just be a loop but should be somewhat faster with a functional design if let Some(v) = path_iter.try_fold(v, |d, loc| loc.json_get(d)) { // Successfully found an item, return it @@ -452,10 +454,10 @@ impl PathItem { } } - pub fn json_get<'a>(&self, any_json: &'a JsonInput) -> Option<&'a JsonInput> { + pub fn json_get<'a>(&self, any_json: &'a JsonValue) -> Option<&'a JsonValue> { match any_json { - JsonInput::Object(v_obj) => self.json_obj_get(v_obj), - JsonInput::Array(v_array) => match self { + JsonValue::Object(v_obj) => self.json_obj_get(v_obj), + JsonValue::Array(v_array) => match self { Self::Pos(index) => v_array.get(*index), Self::Neg(index) => { if let Some(index) = v_array.len().checked_sub(*index) { @@ -470,7 +472,7 @@ impl PathItem { } } - pub fn json_obj_get<'a>(&self, json_obj: &'a JsonObject) -> Option<&'a JsonInput> { + pub fn json_obj_get<'a>(&self, json_obj: &'a JsonObject) -> Option<&'a JsonValue> { match self { Self::S(key, _) => json_obj.get(key), _ => None,