diff --git a/Cargo.lock b/Cargo.lock index b9a195538..a7ff9ee2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1539,6 +1539,12 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" diff --git a/src/stdlib/json_utils/json_type_def.rs b/src/stdlib/json_utils/json_type_def.rs new file mode 100644 index 000000000..77ffa5cd4 --- /dev/null +++ b/src/stdlib/json_utils/json_type_def.rs @@ -0,0 +1,24 @@ +use crate::prelude::{Collection, TypeDef}; +use crate::value::Kind; + +pub(crate) fn json_inner_kind() -> Kind { + Kind::null() + | Kind::bytes() + | Kind::integer() + | Kind::float() + | Kind::boolean() + | Kind::array(Collection::any()) + | Kind::object(Collection::any()) +} + +pub(crate) fn json_type_def() -> TypeDef { + TypeDef::bytes() + .fallible() + .or_boolean() + .or_integer() + .or_float() + .add_null() + .or_null() + .or_array(Collection::from_unknown(json_inner_kind())) + .or_object(Collection::from_unknown(json_inner_kind())) +} diff --git a/src/stdlib/json_utils/mod.rs b/src/stdlib/json_utils/mod.rs new file mode 100644 index 000000000..472564627 --- /dev/null +++ b/src/stdlib/json_utils/mod.rs @@ -0,0 +1 @@ +pub(crate) mod json_type_def; diff --git a/src/stdlib/mod.rs b/src/stdlib/mod.rs index d0d94f1e8..b87aba469 100644 --- a/src/stdlib/mod.rs +++ b/src/stdlib/mod.rs @@ -31,6 +31,7 @@ pub use wasm_unsupported_function::WasmUnsupportedFunction; use crate::compiler::Function; +mod json_utils; mod string_utils; mod util; mod wasm_unsupported_function; diff --git a/src/stdlib/parse_cbor.rs b/src/stdlib/parse_cbor.rs index 4f958c9ee..c384d776c 100644 --- a/src/stdlib/parse_cbor.rs +++ b/src/stdlib/parse_cbor.rs @@ -1,4 +1,5 @@ use crate::compiler::prelude::*; +use crate::stdlib::json_utils::json_type_def::json_type_def; use ciborium::de::from_reader; use zstd::zstd_safe::WriteBuf; @@ -27,11 +28,38 @@ impl Function for ParseCbor { } fn examples(&self) -> &'static [Example] { - &[Example { - title: "object", - source: r#"parse_cbor!(decode_base64!("oWVmaWVsZGV2YWx1ZQ=="))"#, - result: Ok(r#"{ "field": "value" }"#), - }] + &[ + Example { + title: "object", + source: r#"parse_cbor!(decode_base64!("oWVmaWVsZGV2YWx1ZQ=="))"#, + result: Ok(r#"{ "field": "value" }"#), + }, + Example { + title: "array", + source: r#"parse_cbor!(decode_base64!("gvUA"))"#, + result: Ok("[true, 0]"), + }, + Example { + title: "string", + source: r#"parse_cbor!(decode_base64!("ZWhlbGxv"))"#, + result: Ok("hello"), + }, + Example { + title: "integer", + source: r#"parse_cbor!(decode_base64!("GCo="))"#, + result: Ok("42"), + }, + Example { + title: "float", + source: r#"parse_cbor!(decode_base64!("+0BFEKPXCj1x"))"#, + result: Ok("42.13"), + }, + Example { + title: "boolean", + source: r#"parse_cbor!(decode_base64!("9A=="))"#, + result: Ok("false"), + }, + ] } fn compile( @@ -65,31 +93,10 @@ impl FunctionExpression for ParseCborFn { } fn type_def(&self, _: &state::TypeState) -> TypeDef { - type_def() + json_type_def() } } -fn inner_kind() -> Kind { - Kind::null() - | Kind::bytes() - | Kind::integer() - | Kind::float() - | Kind::boolean() - | Kind::array(Collection::any()) - | Kind::object(Collection::any()) -} - -fn type_def() -> TypeDef { - TypeDef::bytes() - .fallible() - .or_boolean() - .or_integer() - .or_float() - .add_null() - .or_array(Collection::from_unknown(inner_kind())) - .or_object(Collection::from_unknown(inner_kind())) -} - #[cfg(test)] mod tests { use super::*; @@ -113,13 +120,13 @@ mod tests { parses { args: func_args![ value: value!(read_cbor_file("simple.cbor").as_bytes()) ], want: Ok(value!({ field: "value" })), - tdef: type_def(), + tdef: json_type_def(), } complex_cbor { args: func_args![ value: value!(read_cbor_file("complex.cbor").as_bytes()) ], want: Ok(value!({ object: {string: "value", number: 42, array: ["hello", "world"], boolean: false} })), - tdef: type_def(), + tdef: json_type_def(), } ]; } diff --git a/src/stdlib/parse_json.rs b/src/stdlib/parse_json.rs index a26316c48..06ef79b59 100644 --- a/src/stdlib/parse_json.rs +++ b/src/stdlib/parse_json.rs @@ -6,6 +6,7 @@ use serde_json::{ }; use crate::compiler::prelude::*; +use crate::stdlib::json_utils::json_type_def::json_type_def; fn parse_json(value: Value, lossy: Option) -> Resolved { let lossy = lossy.map(Value::try_boolean).transpose()?.unwrap_or(true); @@ -226,7 +227,7 @@ impl FunctionExpression for ParseJsonFn { } fn type_def(&self, _: &state::TypeState) -> TypeDef { - type_def() + json_type_def() } } @@ -250,31 +251,10 @@ impl FunctionExpression for ParseJsonMaxDepthFn { } fn type_def(&self, _: &state::TypeState) -> TypeDef { - type_def() + json_type_def() } } -fn inner_kind() -> Kind { - Kind::null() - | Kind::bytes() - | Kind::integer() - | Kind::float() - | Kind::boolean() - | Kind::array(Collection::any()) - | Kind::object(Collection::any()) -} - -fn type_def() -> TypeDef { - TypeDef::bytes() - .fallible() - .or_boolean() - .or_integer() - .or_float() - .add_null() - .or_array(Collection::from_unknown(inner_kind())) - .or_object(Collection::from_unknown(inner_kind())) -} - #[cfg(test)] mod tests { use super::*; @@ -286,74 +266,62 @@ mod tests { parses { args: func_args![ value: r#"{"field": "value"}"# ], want: Ok(value!({ field: "value" })), - tdef: type_def(), + tdef: json_type_def(), } complex_json { args: func_args![ value: r#"{"object": {"string":"value","number":42,"array":["hello","world"],"boolean":false}}"# ], want: Ok(value!({ object: {string: "value", number: 42, array: ["hello", "world"], boolean: false} })), - tdef: type_def(), + tdef: json_type_def(), } invalid_json_errors { args: func_args![ value: r#"{"field": "value"# ], want: Err("unable to parse json: EOF while parsing a string at line 1 column 16"), - tdef: TypeDef::bytes().fallible() - .or_boolean() - .or_integer() - .or_float() - .or_null() - .or_array(Collection::from_unknown(inner_kind())) - .or_object(Collection::from_unknown(inner_kind())), + tdef: json_type_def(), } max_depth { args: func_args![ value: r#"{"top_layer": {"layer_one": "finish", "layer_two": 2}}"#, max_depth: 1], want: Ok(value!({ top_layer: r#"{"layer_one": "finish", "layer_two": 2}"# })), - tdef: type_def(), + tdef: json_type_def(), } max_depth_array { args: func_args![ value: r#"[{"top_layer": {"next_layer": ["finish"]}}]"#, max_depth: 2], want: Ok(value!([{ top_layer: r#"{"next_layer": ["finish"]}"# }])), - tdef: type_def(), + tdef: json_type_def(), } max_depth_exceeds_layers { args: func_args![ value: r#"{"top_layer": {"layer_one": "finish", "layer_two": 2}}"#, max_depth: 10], want: Ok(value!({ top_layer: {layer_one: "finish", layer_two: 2} })), - tdef: type_def(), + tdef: json_type_def(), } invalid_json_with_max_depth { args: func_args![ value: r#"{"field": "value"#, max_depth: 3 ], want: Err("unable to read json: EOF while parsing a string at line 1 column 16"), - tdef: TypeDef::bytes().fallible() - .or_boolean() - .or_integer() - .or_float() - .or_null() - .or_array(Collection::from_unknown(inner_kind())) - .or_object(Collection::from_unknown(inner_kind())), + tdef: json_type_def(), } invalid_input_max_depth { args: func_args![ value: r#"{"top_layer": "finish"}"#, max_depth: 129], want: Err("max_depth value should be greater than 0 and less than 128, got 129"), - tdef: type_def(), + tdef: json_type_def(), } // // TODO: provide a function version of the `test_function!` macro. max_int { args: func_args![ value: format!("{{\"num\": {}}}", i64::MAX - 1)], want: Ok(value!({"num": 9_223_372_036_854_775_806_i64})), - tdef: type_def(), + tdef: json_type_def(), } lossy_float_conversion { args: func_args![ value: r#"{"num": 9223372036854775808}"#], want: Ok(value!({"num": 9.223_372_036_854_776e18})), - tdef: type_def(), + tdef: json_type_def(), } // Checks that the parsing uses the default lossy argument value @@ -362,7 +330,7 @@ mod tests { // 0xf5 is out of the range of valid UTF-8 bytes args: func_args![ value: Bytes::from_static(&[0x22,0xf5,0x22])], want: Ok(value!(std::char::REPLACEMENT_CHARACTER.to_string())), - tdef: type_def(), + tdef: json_type_def(), } parse_invalid_utf8_lossy_arg_true { @@ -370,13 +338,13 @@ mod tests { args: func_args![ value: Bytes::from_static(&[0x22,0xf5,0x22]), lossy: true], // U+FFFD is the replacement character for invalid UTF-8 want: Ok(value!(std::char::REPLACEMENT_CHARACTER.to_string())), - tdef: type_def(), + tdef: json_type_def(), } invalid_utf8_json_lossy_arg_false { args: func_args![ value: Bytes::from_static(&[0x22,0xf5,0x22]), lossy: false], want: Err("unable to parse json: invalid unicode code point at line 1 column 3"), - tdef: type_def(), + tdef: json_type_def(), } ]; @@ -387,7 +355,7 @@ mod tests { no_roundtrip_float_conversion { args: func_args![ value: r#"{"num": 1626175065.5934923}"#], want: Ok(value!({"num": 1_626_175_065.593_492_5})), - tdef: type_def(), + tdef: json_type_def(), } ]; @@ -398,7 +366,7 @@ mod tests { roundtrip_float_conversion { args: func_args![ value: r#"{"num": 1626175065.5934923}"#], want: Ok(value!({"num": 1_626_175_065.593_492_3})), - tdef: type_def(), + tdef: json_type_def(), } ]; } diff --git a/src/stdlib/parse_proto.rs b/src/stdlib/parse_proto.rs index 0c55f2930..aacbe199e 100644 --- a/src/stdlib/parse_proto.rs +++ b/src/stdlib/parse_proto.rs @@ -1,6 +1,7 @@ use crate::compiler::prelude::*; use crate::protobuf::get_message_descriptor; use crate::protobuf::parse_proto; +use crate::stdlib::json_utils::json_type_def::json_type_def; use once_cell::sync::Lazy; use prost_reflect::MessageDescriptor; use std::env; @@ -112,31 +113,10 @@ impl FunctionExpression for ParseProtoFn { } fn type_def(&self, _: &state::TypeState) -> TypeDef { - type_def() + json_type_def() } } -fn inner_kind() -> Kind { - Kind::null() - | Kind::bytes() - | Kind::integer() - | Kind::float() - | Kind::boolean() - | Kind::array(Collection::any()) - | Kind::object(Collection::any()) -} - -fn type_def() -> TypeDef { - TypeDef::bytes() - .fallible() - .or_boolean() - .or_integer() - .or_float() - .add_null() - .or_array(Collection::from_unknown(inner_kind())) - .or_object(Collection::from_unknown(inner_kind())) -} - #[cfg(test)] mod tests { use super::*; @@ -159,7 +139,7 @@ mod tests { desc_file: test_data_dir().join("test_protobuf.desc").to_str().unwrap().to_owned(), message_type: "test_protobuf.Person"], want: Ok(value!({ name: "someone", phones: [{number: "123456"}] })), - tdef: type_def(), + tdef: json_type_def(), } parses_proto3 { @@ -167,7 +147,7 @@ mod tests { desc_file: test_data_dir().join("test_protobuf3.desc").to_str().unwrap().to_owned(), message_type: "test_protobuf3.Person"], want: Ok(value!({ data: {data_phone: "HOME"}, name: "someone", phones: [{number: "1234", type: "MOBILE"}] })), - tdef: type_def(), + tdef: json_type_def(), } ]; }