Skip to content

Commit

Permalink
Move json_type_def to separate module
Browse files Browse the repository at this point in the history
  • Loading branch information
Semyon Uchvatov committed Dec 13, 2024
1 parent 2f37d18 commit 5d40368
Showing 7 changed files with 90 additions and 103 deletions.
6 changes: 6 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 24 additions & 0 deletions src/stdlib/json_utils/json_type_def.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
use crate::prelude::{Collection, TypeDef};
use crate::value::Kind;

pub(crate) fn json_inner_kind() -> Kind {
Kind::null()
| Kind::bytes()
| Kind::integer()
| Kind::float()
| Kind::boolean()
| Kind::array(Collection::any())
| Kind::object(Collection::any())
}

pub(crate) fn json_type_def() -> TypeDef {
TypeDef::bytes()
.fallible()
.or_boolean()
.or_integer()
.or_float()
.add_null()
.or_null()
.or_array(Collection::from_unknown(json_inner_kind()))
.or_object(Collection::from_unknown(json_inner_kind()))
}
1 change: 1 addition & 0 deletions src/stdlib/json_utils/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub(crate) mod json_type_def;
1 change: 1 addition & 0 deletions src/stdlib/mod.rs
Original file line number Diff line number Diff line change
@@ -31,6 +31,7 @@ pub use wasm_unsupported_function::WasmUnsupportedFunction;

use crate::compiler::Function;

mod json_utils;
mod string_utils;
mod util;
mod wasm_unsupported_function;
65 changes: 36 additions & 29 deletions src/stdlib/parse_cbor.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::compiler::prelude::*;
use crate::stdlib::json_utils::json_type_def::json_type_def;
use ciborium::de::from_reader;
use zstd::zstd_safe::WriteBuf;

@@ -27,11 +28,38 @@ impl Function for ParseCbor {
}

fn examples(&self) -> &'static [Example] {
&[Example {
title: "object",
source: r#"parse_cbor!(decode_base64!("oWVmaWVsZGV2YWx1ZQ=="))"#,
result: Ok(r#"{ "field": "value" }"#),
}]
&[
Example {
title: "object",
source: r#"parse_cbor!(decode_base64!("oWVmaWVsZGV2YWx1ZQ=="))"#,
result: Ok(r#"{ "field": "value" }"#),
},
Example {
title: "array",
source: r#"parse_cbor!(decode_base64!("gvUA"))"#,
result: Ok("[true, 0]"),
},
Example {
title: "string",
source: r#"parse_cbor!(decode_base64!("ZWhlbGxv"))"#,
result: Ok("hello"),
},
Example {
title: "integer",
source: r#"parse_cbor!(decode_base64!("GCo="))"#,
result: Ok("42"),
},
Example {
title: "float",
source: r#"parse_cbor!(decode_base64!("+0BFEKPXCj1x"))"#,
result: Ok("42.13"),
},
Example {
title: "boolean",
source: r#"parse_cbor!(decode_base64!("9A=="))"#,
result: Ok("false"),
},
]
}

fn compile(
@@ -65,31 +93,10 @@ impl FunctionExpression for ParseCborFn {
}

fn type_def(&self, _: &state::TypeState) -> TypeDef {
type_def()
json_type_def()
}
}

fn inner_kind() -> Kind {
Kind::null()
| Kind::bytes()
| Kind::integer()
| Kind::float()
| Kind::boolean()
| Kind::array(Collection::any())
| Kind::object(Collection::any())
}

fn type_def() -> TypeDef {
TypeDef::bytes()
.fallible()
.or_boolean()
.or_integer()
.or_float()
.add_null()
.or_array(Collection::from_unknown(inner_kind()))
.or_object(Collection::from_unknown(inner_kind()))
}

#[cfg(test)]
mod tests {
use super::*;
@@ -113,13 +120,13 @@ mod tests {
parses {
args: func_args![ value: value!(read_cbor_file("simple.cbor").as_bytes()) ],
want: Ok(value!({ field: "value" })),
tdef: type_def(),
tdef: json_type_def(),
}

complex_cbor {
args: func_args![ value: value!(read_cbor_file("complex.cbor").as_bytes()) ],
want: Ok(value!({ object: {string: "value", number: 42, array: ["hello", "world"], boolean: false} })),
tdef: type_def(),
tdef: json_type_def(),
}
];
}
68 changes: 18 additions & 50 deletions src/stdlib/parse_json.rs
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ use serde_json::{
};

use crate::compiler::prelude::*;
use crate::stdlib::json_utils::json_type_def::json_type_def;

fn parse_json(value: Value, lossy: Option<Value>) -> Resolved {
let lossy = lossy.map(Value::try_boolean).transpose()?.unwrap_or(true);
@@ -226,7 +227,7 @@ impl FunctionExpression for ParseJsonFn {
}

fn type_def(&self, _: &state::TypeState) -> TypeDef {
type_def()
json_type_def()
}
}

@@ -250,31 +251,10 @@ impl FunctionExpression for ParseJsonMaxDepthFn {
}

fn type_def(&self, _: &state::TypeState) -> TypeDef {
type_def()
json_type_def()
}
}

fn inner_kind() -> Kind {
Kind::null()
| Kind::bytes()
| Kind::integer()
| Kind::float()
| Kind::boolean()
| Kind::array(Collection::any())
| Kind::object(Collection::any())
}

fn type_def() -> TypeDef {
TypeDef::bytes()
.fallible()
.or_boolean()
.or_integer()
.or_float()
.add_null()
.or_array(Collection::from_unknown(inner_kind()))
.or_object(Collection::from_unknown(inner_kind()))
}

#[cfg(test)]
mod tests {
use super::*;
@@ -286,74 +266,62 @@ mod tests {
parses {
args: func_args![ value: r#"{"field": "value"}"# ],
want: Ok(value!({ field: "value" })),
tdef: type_def(),
tdef: json_type_def(),
}

complex_json {
args: func_args![ value: r#"{"object": {"string":"value","number":42,"array":["hello","world"],"boolean":false}}"# ],
want: Ok(value!({ object: {string: "value", number: 42, array: ["hello", "world"], boolean: false} })),
tdef: type_def(),
tdef: json_type_def(),
}

invalid_json_errors {
args: func_args![ value: r#"{"field": "value"# ],
want: Err("unable to parse json: EOF while parsing a string at line 1 column 16"),
tdef: TypeDef::bytes().fallible()
.or_boolean()
.or_integer()
.or_float()
.or_null()
.or_array(Collection::from_unknown(inner_kind()))
.or_object(Collection::from_unknown(inner_kind())),
tdef: json_type_def(),
}

max_depth {
args: func_args![ value: r#"{"top_layer": {"layer_one": "finish", "layer_two": 2}}"#, max_depth: 1],
want: Ok(value!({ top_layer: r#"{"layer_one": "finish", "layer_two": 2}"# })),
tdef: type_def(),
tdef: json_type_def(),
}

max_depth_array {
args: func_args![ value: r#"[{"top_layer": {"next_layer": ["finish"]}}]"#, max_depth: 2],
want: Ok(value!([{ top_layer: r#"{"next_layer": ["finish"]}"# }])),
tdef: type_def(),
tdef: json_type_def(),
}

max_depth_exceeds_layers {
args: func_args![ value: r#"{"top_layer": {"layer_one": "finish", "layer_two": 2}}"#, max_depth: 10],
want: Ok(value!({ top_layer: {layer_one: "finish", layer_two: 2} })),
tdef: type_def(),
tdef: json_type_def(),
}

invalid_json_with_max_depth {
args: func_args![ value: r#"{"field": "value"#, max_depth: 3 ],
want: Err("unable to read json: EOF while parsing a string at line 1 column 16"),
tdef: TypeDef::bytes().fallible()
.or_boolean()
.or_integer()
.or_float()
.or_null()
.or_array(Collection::from_unknown(inner_kind()))
.or_object(Collection::from_unknown(inner_kind())),
tdef: json_type_def(),
}

invalid_input_max_depth {
args: func_args![ value: r#"{"top_layer": "finish"}"#, max_depth: 129],
want: Err("max_depth value should be greater than 0 and less than 128, got 129"),
tdef: type_def(),
tdef: json_type_def(),
}

// // TODO: provide a function version of the `test_function!` macro.
max_int {
args: func_args![ value: format!("{{\"num\": {}}}", i64::MAX - 1)],
want: Ok(value!({"num": 9_223_372_036_854_775_806_i64})),
tdef: type_def(),
tdef: json_type_def(),
}

lossy_float_conversion {
args: func_args![ value: r#"{"num": 9223372036854775808}"#],
want: Ok(value!({"num": 9.223_372_036_854_776e18})),
tdef: type_def(),
tdef: json_type_def(),
}

// Checks that the parsing uses the default lossy argument value
@@ -362,21 +330,21 @@ mod tests {
// 0xf5 is out of the range of valid UTF-8 bytes
args: func_args![ value: Bytes::from_static(&[0x22,0xf5,0x22])],
want: Ok(value!(std::char::REPLACEMENT_CHARACTER.to_string())),
tdef: type_def(),
tdef: json_type_def(),
}

parse_invalid_utf8_lossy_arg_true {
// 0xf5 is out of the range of valid UTF-8 bytes
args: func_args![ value: Bytes::from_static(&[0x22,0xf5,0x22]), lossy: true],
// U+FFFD is the replacement character for invalid UTF-8
want: Ok(value!(std::char::REPLACEMENT_CHARACTER.to_string())),
tdef: type_def(),
tdef: json_type_def(),
}

invalid_utf8_json_lossy_arg_false {
args: func_args![ value: Bytes::from_static(&[0x22,0xf5,0x22]), lossy: false],
want: Err("unable to parse json: invalid unicode code point at line 1 column 3"),
tdef: type_def(),
tdef: json_type_def(),
}
];

@@ -387,7 +355,7 @@ mod tests {
no_roundtrip_float_conversion {
args: func_args![ value: r#"{"num": 1626175065.5934923}"#],
want: Ok(value!({"num": 1_626_175_065.593_492_5})),
tdef: type_def(),
tdef: json_type_def(),
}
];

@@ -398,7 +366,7 @@ mod tests {
roundtrip_float_conversion {
args: func_args![ value: r#"{"num": 1626175065.5934923}"#],
want: Ok(value!({"num": 1_626_175_065.593_492_3})),
tdef: type_def(),
tdef: json_type_def(),
}
];
}
28 changes: 4 additions & 24 deletions src/stdlib/parse_proto.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use crate::compiler::prelude::*;
use crate::protobuf::get_message_descriptor;
use crate::protobuf::parse_proto;
use crate::stdlib::json_utils::json_type_def::json_type_def;
use once_cell::sync::Lazy;
use prost_reflect::MessageDescriptor;
use std::env;
@@ -112,31 +113,10 @@ impl FunctionExpression for ParseProtoFn {
}

fn type_def(&self, _: &state::TypeState) -> TypeDef {
type_def()
json_type_def()
}
}

fn inner_kind() -> Kind {
Kind::null()
| Kind::bytes()
| Kind::integer()
| Kind::float()
| Kind::boolean()
| Kind::array(Collection::any())
| Kind::object(Collection::any())
}

fn type_def() -> TypeDef {
TypeDef::bytes()
.fallible()
.or_boolean()
.or_integer()
.or_float()
.add_null()
.or_array(Collection::from_unknown(inner_kind()))
.or_object(Collection::from_unknown(inner_kind()))
}

#[cfg(test)]
mod tests {
use super::*;
@@ -159,15 +139,15 @@ mod tests {
desc_file: test_data_dir().join("test_protobuf.desc").to_str().unwrap().to_owned(),
message_type: "test_protobuf.Person"],
want: Ok(value!({ name: "someone", phones: [{number: "123456"}] })),
tdef: type_def(),
tdef: json_type_def(),
}

parses_proto3 {
args: func_args![ value: read_pb_file("person_someone3.pb"),
desc_file: test_data_dir().join("test_protobuf3.desc").to_str().unwrap().to_owned(),
message_type: "test_protobuf3.Person"],
want: Ok(value!({ data: {data_phone: "HOME"}, name: "someone", phones: [{number: "1234", type: "MOBILE"}] })),
tdef: type_def(),
tdef: json_type_def(),
}
];
}

0 comments on commit 5d40368

Please sign in to comment.