From af792c7b3412b85fffe1b69b9e5cf2c752d39c36 Mon Sep 17 00:00:00 2001 From: Matthew Paras <34500476+mattwparas@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:31:12 -0800 Subject: [PATCH] Add rustls library and fix free identifier error message (#291) * add rustls and fix free identifier error message * remove prints * remove warnings * more fixes * fix warnings * fix warning --- Cargo.lock | 258 +++++++++++++++--- crates/steel-core/src/compiler/compiler.rs | 45 ++- crates/steel-core/src/primitives/http.rs | 20 +- .../src/scheme/modules/parameters.scm | 5 +- crates/steel-core/src/steel_vm/ffi.rs | 84 +++++- crates/steel-core/src/values/json_vals.rs | 3 +- crates/steel-core/src/values/port.rs | 1 + libs/steel-rustls/Cargo.toml | 16 ++ libs/steel-rustls/rustls.scm | 20 ++ libs/steel-rustls/src/lib.rs | 143 ++++++++++ libs/steel-rustls/src/main.rs | 13 + libs/steel-sqlite/Cargo.toml | 2 +- libs/steel-sqlite/src/lib.rs | 3 +- 13 files changed, 565 insertions(+), 48 deletions(-) create mode 100644 libs/steel-rustls/Cargo.toml create mode 100644 libs/steel-rustls/rustls.scm create mode 100644 libs/steel-rustls/src/lib.rs create mode 100644 libs/steel-rustls/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index b29ae0cf2..456f275fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ dependencies = [ "core_extensions", "crossbeam-channel", "generational-arena", - "libloading", + "libloading 0.7.4", "lock_api", "parking_lot", "paste", @@ -65,6 +65,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "ahash" version = "0.8.11" @@ -177,7 +183,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -188,7 +194,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -197,6 +203,33 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "aws-lc-rs" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7c2840b66236045acd2607d5866e274380afd87ef99d6226e961e2cb47df45" +dependencies = [ + "aws-lc-sys", + "mirai-annotations", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3a619a9de81e1d7de1f1186dcba4506ed661a0e483d84410fdef0ee87b2f96" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", +] + [[package]] name = "axum" version = "0.6.20" @@ -256,7 +289,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.4", "object", "rustc-demangle", ] @@ -301,6 +334,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.87", + "which", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -446,9 +502,20 @@ version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68064e60dbf1f17005c2fde4d07c16d8baa506fd7ffed8ccab702d93617975c7" dependencies = [ + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -506,6 +573,17 @@ dependencies = [ "half", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading 0.8.5", +] + [[package]] name = "clap" version = "4.1.4" @@ -552,6 +630,15 @@ dependencies = [ "error-code", ] +[[package]] +name = "cmake" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +dependencies = [ + "cc", +] + [[package]] name = "codegen" version = "0.2.0" @@ -971,7 +1058,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -982,7 +1069,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -1039,7 +1126,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -1049,7 +1136,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4abae7035bf79b9877b779505d8cf3749285b80c43941eda66604841889451dc" dependencies = [ "derive_builder_core", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -1089,6 +1176,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.13.0" @@ -1212,12 +1305,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.31" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.0", ] [[package]] @@ -1235,6 +1328,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.30" @@ -1305,7 +1404,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -1406,6 +1505,12 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "half" version = "2.4.1" @@ -1703,6 +1808,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.70" @@ -1730,6 +1844,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.156" @@ -1746,6 +1866,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "libloading" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libm" version = "0.2.8" @@ -1872,6 +2002,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.4" @@ -1881,6 +2017,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "0.8.11" @@ -1905,6 +2050,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mirai-annotations" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" + [[package]] name = "nibble_vec" version = "0.1.0" @@ -1938,6 +2089,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num" version = "0.4.3" @@ -2081,9 +2242,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.103" +version = "0.9.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" dependencies = [ "cc", "libc", @@ -2161,7 +2322,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -2290,6 +2451,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +dependencies = [ + "proc-macro2", + "syn 2.0.87", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2631,6 +2802,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.0" @@ -2667,10 +2844,11 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "7f1a745511c54ba6d4465e8d5dfbd81b45791756de28d4981af70d6dca128f1e" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring 0.17.8", @@ -2703,16 +2881,17 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ + "aws-lc-rs", "ring 0.17.8", "rustls-pki-types", "untrusted 0.9.0", @@ -2858,7 +3037,7 @@ checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -2891,7 +3070,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -3119,7 +3298,7 @@ version = "0.5.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -3219,6 +3398,17 @@ dependencies = [ "steel-parser", ] +[[package]] +name = "steel-rustls" +version = "0.6.0" +dependencies = [ + "abi_stable", + "flate2", + "rustls 0.23.17", + "steel-core", + "webpki-roots", +] + [[package]] name = "steel-sqlite" version = "0.6.0" @@ -3318,9 +3508,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.74" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -3424,7 +3614,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -3511,7 +3701,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -3614,7 +3804,7 @@ checksum = "84fd902d4e0b9a4b27f2f440108dc034e1758628a9b702f8ec61ad66355422fa" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -3643,7 +3833,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] @@ -3790,7 +3980,7 @@ dependencies = [ "flate2", "log", "once_cell", - "rustls 0.23.12", + "rustls 0.23.17", "rustls-pki-types", "serde", "serde_json", @@ -3931,7 +4121,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -3953,7 +4143,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4245,7 +4435,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.87", ] [[package]] diff --git a/crates/steel-core/src/compiler/compiler.rs b/crates/steel-core/src/compiler/compiler.rs index 7f9f6ef50..f33a47003 100644 --- a/crates/steel-core/src/compiler/compiler.rs +++ b/crates/steel-core/src/compiler/compiler.rs @@ -71,6 +71,8 @@ impl DebruijnIndicesInterner { instructions: &mut [Instruction], symbol_map: &mut SymbolMap, ) -> Result<()> { + let mut flat_defines_non_closure: HashSet = HashSet::default(); + for i in 2..instructions.len() { match (&instructions[i], &instructions[i - 1], &instructions[i - 2]) { ( @@ -93,11 +95,11 @@ impl DebruijnIndicesInterner { .. }, ) => { + // Exiting a definition, clear it + flat_defines_non_closure.clear(); + let idx = symbol_map.add(s); self.flat_defines.insert(s.to_owned()); - // if !self.flat_defines.insert(s.to_owned()) { - // stop!(BadSyntax => format!("Cannot redefine define within the same scope: {}", s); *span); - // } if let Some(x) = instructions.get_mut(i) { x.payload_size = u24::from_usize(idx); @@ -109,23 +111,48 @@ impl DebruijnIndicesInterner { contents: Some(Expr::Atom(SyntaxObject { ty: TokenType::Identifier(s), - // span, + span, .. })), .. }, + Instruction { + op_code: OpCode::EDEF, + .. + }, .., ) => { let idx = symbol_map.add(s); self.flat_defines.insert(s.to_owned()); - // if !self.flat_defines.insert(s.to_owned()) { - // stop!(BadSyntax => format!("Cannot redefine define within the same scope: {}", s); *span); - // } + + if flat_defines_non_closure.contains(s) { + stop!(BadSyntax => format!("Cannot reference identifier before its definition: {}", s.resolve()); *span); + } + + flat_defines_non_closure.clear(); + + // self.flat_defines_idx.insert(idx); + // flat_defines_non_closure.insert(s.to_owned()); if let Some(x) = instructions.get_mut(i) { x.payload_size = u24::from_usize(idx); } } + ( + Instruction { + op_code: OpCode::CALLGLOBAL, + contents: + Some(Expr::Atom(SyntaxObject { + ty: TokenType::Identifier(s), + .. + })), + .. + }, + .., + ) => { + // We're referencing things within the scope + flat_defines_non_closure.insert(*s); + } _ => {} } } @@ -181,6 +208,10 @@ impl DebruijnIndicesInterner { } => { // Keep track of where the defines actually are in the process self.second_pass_defines.insert(s.to_owned()); + + // let idx = symbol_map.get(s).unwrap(); + + // self.second_pass_defines_idx.insert(idx); } Instruction { op_code: OpCode::PUSH, diff --git a/crates/steel-core/src/primitives/http.rs b/crates/steel-core/src/primitives/http.rs index 8985173c7..88809d874 100644 --- a/crates/steel-core/src/primitives/http.rs +++ b/crates/steel-core/src/primitives/http.rs @@ -86,6 +86,23 @@ pub fn headers(value: &SteelVal) -> Result { )))) } +#[steel_derive::function(name = "http-response-headers")] +pub fn resp_headers(value: &SteelVal) -> Result { + let resp = SteelResponse::as_ref(value)?; + + Ok(SteelVal::HashMapV(SteelHashMap(Gc::new( + resp.headers + .iter() + .map(|x| { + ( + SteelVal::StringV(x.name.clone().into()), + SteelVal::ByteVector(SteelByteVector::new(x.value.clone())), + ) + }) + .collect::>(), + )))) +} + // If not complete, try again? fn parse_request(buf: &[u8]) -> Result { // Pull more bytes from the stream? @@ -121,7 +138,7 @@ fn parse_request(buf: &[u8]) -> Result { fn parse_response(buf: &[u8]) -> Result { // Pull more bytes from the stream? - let mut headers = [httparse::EMPTY_HEADER; 16]; + let mut headers = [httparse::EMPTY_HEADER; 64]; let mut req = httparse::Response::new(&mut headers); let res = req.parse(&buf).unwrap(); if res.is_complete() { @@ -171,6 +188,7 @@ pub fn http_module() -> BuiltInModule { .register_native_fn_definition(PATH_DEFINITION) .register_native_fn_definition(BODY_OFFSET_DEFINITION) .register_native_fn_definition(HEADERS_DEFINITION) + .register_native_fn_definition(RESP_HEADERS_DEFINITION) .register_native_fn_definition(PARSE_HTTP_RESPONSE_DEFINITION); // module diff --git a/crates/steel-core/src/scheme/modules/parameters.scm b/crates/steel-core/src/scheme/modules/parameters.scm index 098ab0d8b..85d859a18 100644 --- a/crates/steel-core/src/scheme/modules/parameters.scm +++ b/crates/steel-core/src/scheme/modules/parameters.scm @@ -73,7 +73,10 @@ (define write-char #%raw-write-char) -(define write #%raw-write) +(define write + (case-lambda + [(arg) (#%raw-write arg (current-output-port))] + [(arg port) (#%raw-write arg port)])) ;;;;;;;;;;;;;;;;;;;;; Port functions ;;;;;;;;;;;;;;;;;;;;; diff --git a/crates/steel-core/src/steel_vm/ffi.rs b/crates/steel-core/src/steel_vm/ffi.rs index 4a01ddc3b..083369737 100644 --- a/crates/steel-core/src/steel_vm/ffi.rs +++ b/crates/steel-core/src/steel_vm/ffi.rs @@ -1,5 +1,6 @@ use std::{ borrow::Cow, + io::BufReader, marker::PhantomData, sync::{Arc, Mutex}, }; @@ -14,7 +15,11 @@ use crate::{ as_underlying_type_mut, Custom, CustomType, FutureResult, IntoSteelVal, MaybeSendSyncStatic, Result, SteelByteVector, SteelHashMap, SteelVal, }, - values::functions::{BoxedDynFunction, StaticOrRcStr}, + values::{ + functions::{BoxedDynFunction, StaticOrRcStr}, + port::SteelPort, + SteelPortRepr, + }, SteelErr, }; @@ -23,7 +28,7 @@ use abi_stable::{ std_types::{ RArc, RBoxError, RCowStr, RHashMap, RResult, RSlice, RSliceMut, RStr, RString, RVec, Tuple2, }, - RMut, StableAbi, + DynTrait, RMut, StableAbi, }; use futures_util::FutureExt; @@ -331,6 +336,16 @@ impl<'a> FromFFIArg<'a> for RSliceMut<'a, FFIValue> { } } +impl<'a> FromFFIArg<'a> for RVec { + fn from_ffi_arg(val: FFIArg<'a>) -> RResult { + if let FFIArg::ByteVector(b) = val { + RResult::ROk(b) + } else { + conversion_error!(bytevector, val) + } + } +} + impl<'a, T: Custom + Clone + 'static> FromFFIArg<'a> for T { fn from_ffi_arg(val: FFIArg<'a>) -> RResult { let lifted = unsafe { std::mem::transmute::, FFIArg<'static>>(val) }; @@ -929,6 +944,52 @@ macro_rules! declare_module { }; } +#[repr(C)] +#[derive(StableAbi)] +#[sabi(impl_InterfaceType(Sync, Send, IoWrite))] +pub struct WriterInterface; + +#[repr(C)] +#[derive(StableAbi)] +#[sabi(impl_InterfaceType(Sync, Send, IoRead))] +pub struct ReaderInterface; + +#[repr(C)] +#[derive(StableAbi)] +pub struct DynWriter { + pub writer: DynTrait<'static, RBox<()>, WriterInterface>, +} + +impl DynWriter { + fn into_port(self) -> SteelPortRepr { + SteelPortRepr::DynWriter(Arc::new(Mutex::new(self.writer))) + } +} + +#[repr(C)] +#[derive(StableAbi)] +pub struct DynReader { + pub reader: DynTrait<'static, RBox<()>, ReaderInterface>, +} + +impl DynReader { + fn into_port(self) -> SteelPortRepr { + SteelPortRepr::DynReader(BufReader::new(Box::new(self.reader))) + } +} + +impl IntoFFIVal for DynWriter { + fn into_ffi_val(self) -> RResult { + RResult::ROk(FFIValue::DynWriter(self)) + } +} + +impl IntoFFIVal for DynReader { + fn into_ffi_val(self) -> RResult { + RResult::ROk(FFIValue::DynReader(self)) + } +} + #[repr(C)] #[derive(StableAbi)] pub struct MutableString { @@ -937,6 +998,8 @@ pub struct MutableString { impl Custom for MutableString {} +#[repr(C)] +#[derive(StableAbi)] struct FFIVector { vec: RVec, } @@ -1012,6 +1075,7 @@ pub enum FFIArg<'a> { fut: FfiFuture, RBoxError>>, }, HostFunction(HostRuntimeFunction), + ByteVector(RVec), } impl<'a> std::default::Default for FFIArg<'a> { @@ -1101,6 +1165,8 @@ pub enum FFIValue { fut: SyncFfiFuture, }, ByteVector(RVec), + DynWriter(DynWriter), + DynReader(DynReader), } #[repr(C)] @@ -1167,6 +1233,8 @@ impl std::fmt::Debug for FFIValue { FFIValue::HashMap(h) => write!(f, "{:?}", h), FFIValue::Future { .. } => write!(f, "#"), FFIValue::ByteVector(b) => write!(f, "{:?}", b), + FFIValue::DynWriter(_) => write!(f, "#"), + FFIValue::DynReader(_) => write!(f, "#"), } } } @@ -1294,6 +1362,14 @@ impl IntoSteelVal for FFIValue { )))), Self::ByteVector(b) => Ok(SteelVal::ByteVector(SteelByteVector::new(b.into()))), + + Self::DynWriter(d) => Ok(SteelVal::PortV(SteelPort { + port: Gc::new_mut(d.into_port()), + })), + + Self::DynReader(d) => Ok(SteelVal::PortV(SteelPort { + port: Gc::new_mut(d.into_port()), + })), } } } @@ -1501,6 +1577,10 @@ fn as_ffi_argument(value: &SteelVal) -> Result> { SteelVal::NumV(n) => Ok(FFIArg::NumV(*n)), SteelVal::CharV(c) => Ok(FFIArg::CharV { c: *c }), SteelVal::Void => Ok(FFIArg::Void), + + // TODO: Find a way to not have to copy the whole byte vector + SteelVal::ByteVector(b) => Ok(FFIArg::ByteVector(b.vec.read().iter().copied().collect())), + // We can really only look at values that were made from the FFI boundary. SteelVal::Custom(c) => { // let mut guard = if let Ok(guard) = RefCell::try_borrow_mut(c) { diff --git a/crates/steel-core/src/values/json_vals.rs b/crates/steel-core/src/values/json_vals.rs index 00ce97ab7..29f18e261 100644 --- a/crates/steel-core/src/values/json_vals.rs +++ b/crates/steel-core/src/values/json_vals.rs @@ -22,7 +22,8 @@ use steel_derive::function; /// ``` #[function(name = "string->jsexpr")] pub fn string_to_jsexpr(value: &SteelString) -> Result { - let unescaped = unescape(&value); + // let unescaped = unescape(&value); + let unescaped = value; let res: std::result::Result = serde_json::from_str(unescaped.as_str()); match res { diff --git a/crates/steel-core/src/values/port.rs b/crates/steel-core/src/values/port.rs index b672f18e4..9a9ab282e 100644 --- a/crates/steel-core/src/values/port.rs +++ b/crates/steel-core/src/values/port.rs @@ -191,6 +191,7 @@ impl SteelPortRepr { SteelPortRepr::StdInput(br) => port_read_str_fn!(br, read_to_string), SteelPortRepr::ChildStdOutput(br) => port_read_str_fn!(br, read_to_string), SteelPortRepr::ChildStdError(br) => port_read_str_fn!(br, read_to_string), + SteelPortRepr::DynReader(br) => port_read_str_fn!(br, read_to_string), _x => stop!(Generic => "read-all-str"), } } diff --git a/libs/steel-rustls/Cargo.toml b/libs/steel-rustls/Cargo.toml new file mode 100644 index 000000000..539f47bc6 --- /dev/null +++ b/libs/steel-rustls/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "steel-rustls" +edition = "2021" +version.workspace = true + +[lib] +name = "steel_rustls" +crate-type = ["rlib", "cdylib"] + +[dependencies] +steel-core = { path = "../../crates/steel-core", version = "0.6.0", features = ["dylibs", "sync"] } +abi_stable = "0.11.1" +rustls = "0.23.17" +webpki-roots = "0.26" +flate2 = "1.0.35" + diff --git a/libs/steel-rustls/rustls.scm b/libs/steel-rustls/rustls.scm new file mode 100644 index 000000000..861b777fe --- /dev/null +++ b/libs/steel-rustls/rustls.scm @@ -0,0 +1,20 @@ +(#%require-dylib "libsteel_rustls" (only-in + client-connection + gz-decode + tcp-connect + tcp-reader + tcp-writer + tls-reader + tls-stream + tls-writer +)) +(provide + client-connection + gz-decode + tcp-connect + tcp-reader + tcp-writer + tls-reader + tls-stream + tls-writer +) diff --git a/libs/steel-rustls/src/lib.rs b/libs/steel-rustls/src/lib.rs new file mode 100644 index 000000000..163242c66 --- /dev/null +++ b/libs/steel-rustls/src/lib.rs @@ -0,0 +1,143 @@ +use abi_stable::{std_types::RVec, DynTrait}; +use steel::{ + rvals::Custom, + steel_vm::ffi::{DynReader, DynWriter, FFIModule, RegisterFFIFn}, +}; + +use std::sync::Mutex; +use std::{io::Cursor, sync::Arc}; + +use rustls::RootCertStore; + +struct RustlsClientConnection(Option); +impl Custom for RustlsClientConnection {} + +#[derive(Clone)] +struct RustlsStream(Arc>>); +impl Custom for RustlsStream {} + +impl std::io::Write for RustlsStream { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.0.lock().unwrap().flush() + } +} + +impl std::io::Read for RustlsStream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.0.lock().unwrap().read(buf) + } +} + +impl RustlsClientConnection { + pub fn new(server_name: String) -> Self { + let root_store = RootCertStore { + roots: webpki_roots::TLS_SERVER_ROOTS.into(), + }; + let mut config = rustls::ClientConfig::builder() + .with_root_certificates(root_store) + .with_no_client_auth(); + + // Allow using SSLKEYLOGFILE. + config.key_log = Arc::new(rustls::KeyLogFile::new()); + + RustlsClientConnection(Some( + rustls::ClientConnection::new(Arc::new(config), server_name.try_into().unwrap()) + .unwrap(), + )) + } +} + +impl RustlsStream { + // Pass the new stream in... + pub fn new(client_connection: &mut RustlsClientConnection, stream: RustlsTcpStream) -> Self { + Self(Arc::new(Mutex::new(rustls::StreamOwned::new( + client_connection.0.take().unwrap(), + stream.0, + )))) + } +} + +steel::declare_module!(build_module); + +use std::net::TcpStream; + +struct RustlsTcpStream(TcpStream); +impl Custom for RustlsTcpStream {} + +impl Clone for RustlsTcpStream { + fn clone(&self) -> Self { + RustlsTcpStream(self.0.try_clone().unwrap()) + } +} + +#[allow(unused)] +#[derive(Debug)] +enum RustlsError { + Io(std::io::Error), +} + +impl std::fmt::Display for RustlsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl std::error::Error for RustlsError {} + +impl Custom for RustlsError {} + +fn connect(addr: &str) -> Result { + TcpStream::connect(addr) + .map(RustlsTcpStream) + .map_err(RustlsError::Io) +} + +fn tcp_reader(stream: &RustlsTcpStream) -> DynReader { + DynReader { + reader: DynTrait::from_value(stream.0.try_clone().unwrap()), + } +} + +fn tcp_writer(stream: &RustlsTcpStream) -> DynWriter { + DynWriter { + writer: DynTrait::from_value(stream.0.try_clone().unwrap()), + } +} + +fn tls_tcp_reader(stream: &RustlsStream) -> DynReader { + DynReader { + reader: DynTrait::from_value(stream.clone()), + } +} + +fn tls_tcp_writer(stream: &RustlsStream) -> DynWriter { + DynWriter { + writer: DynTrait::from_value(stream.clone()), + } +} + +fn gz_decoder(buf: RVec) -> DynReader { + DynReader { + reader: DynTrait::from_value(flate2::read::GzDecoder::new(Cursor::new(buf))), + } +} + +pub fn build_module() -> FFIModule { + let mut module = FFIModule::new("steel/rustls"); + + module + .register_fn("tcp-connect", connect) + .register_fn("tcp-reader", tcp_reader) + .register_fn("tcp-writer", tcp_writer) + .register_fn("client-connection", RustlsClientConnection::new) + .register_fn("tls-stream", RustlsStream::new) + .register_fn("tls-reader", tls_tcp_reader) + .register_fn("tls-writer", tls_tcp_writer) + .register_fn("gz-decode", gz_decoder); + + module +} diff --git a/libs/steel-rustls/src/main.rs b/libs/steel-rustls/src/main.rs new file mode 100644 index 000000000..5b34c4715 --- /dev/null +++ b/libs/steel-rustls/src/main.rs @@ -0,0 +1,13 @@ +use steel_rustls::build_module; + +fn main() { + std::process::Command::new("cargo-steel-lib") + .spawn() + .unwrap() + .wait() + .unwrap(); + + build_module() + .emit_package_to_file("libsteel_rustls", "rustls.scm") + .unwrap() +} diff --git a/libs/steel-sqlite/Cargo.toml b/libs/steel-sqlite/Cargo.toml index 573a21725..57f224165 100644 --- a/libs/steel-sqlite/Cargo.toml +++ b/libs/steel-sqlite/Cargo.toml @@ -14,4 +14,4 @@ crate-type = ["cdylib"] abi_stable = "0.11.1" # steel-core = { path = "../../crates/steel-core", version = "0.6.0", features = ["dylibs"] } steel-core = { workspace = true } -rusqlite = { version = "0.28.0", features = ["bundled"] } +rusqlite = { version = "0.28.0", features = ["bundled", "functions"] } diff --git a/libs/steel-sqlite/src/lib.rs b/libs/steel-sqlite/src/lib.rs index 11a725d94..03749ff40 100644 --- a/libs/steel-sqlite/src/lib.rs +++ b/libs/steel-sqlite/src/lib.rs @@ -142,6 +142,7 @@ impl Drop for SqliteTransaction { } } +#[allow(unused)] #[derive(Debug)] enum SqliteError { TransactionAlreadyCompleted, @@ -262,7 +263,7 @@ impl<'a> ToSql for FFIWrapper<'a> { fn to_sql(&self) -> rusqlite::Result> { match &self.0 { // FFIValue::BoxedFunction(_) => todo!(), - // FFIValue::BoolV(b) => Ok(rusqlite::types::ToSqlOutput::Owned(Value::Bo)), + FFIArg::BoolV(b) => Ok(ToSqlOutput::Owned(Value::Integer(if *b { 1 } else { 0 }))), FFIArg::NumV(f) => Ok(ToSqlOutput::Owned(Value::Real(*f))), FFIArg::IntV(i) => Ok(ToSqlOutput::Owned(Value::Integer(*i as i64))), FFIArg::Void => Ok(ToSqlOutput::Owned(Value::Null)),