From 4a11ec3b9c491309fb45a333f833d384c137bdef Mon Sep 17 00:00:00 2001 From: afinch7 Date: Mon, 18 Nov 2019 12:29:42 -0500 Subject: [PATCH 01/24] wip: native plugin ops --- Cargo.lock | 59 ++++++++++++++++++++++ cli/Cargo.toml | 1 + cli/deno_error.rs | 15 ++++++ cli/flags.rs | 14 ++++++ cli/ops/mod.rs | 1 + cli/ops/native_plugins.rs | 103 ++++++++++++++++++++++++++++++++++++++ cli/permissions.rs | 9 ++++ cli/state.rs | 5 ++ core/lib.rs | 2 + core/plugins.rs | 22 ++++++++ 10 files changed, 231 insertions(+) create mode 100644 cli/ops/native_plugins.rs create mode 100644 core/plugins.rs diff --git a/Cargo.lock b/Cargo.lock index 16f6caad68d781..fbafd4a57ad576 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,6 +293,7 @@ dependencies = [ "deno 0.24.0", "deno_typescript 0.24.0", "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "dlopen 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fwdansi 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "http 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)", @@ -357,6 +358,27 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "dlopen" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dlopen_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "dlopen_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "downcast-rs" version = "1.1.1" @@ -969,6 +991,14 @@ name = "proc-macro-nested" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "proc-macro2" version = "1.0.6" @@ -989,6 +1019,14 @@ dependencies = [ "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "quote" version = "1.0.2" @@ -1436,6 +1474,16 @@ name = "strsim" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "syn" version = "1.0.7" @@ -1786,6 +1834,11 @@ name = "unicode-width" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicode-xid" version = "0.2.0" @@ -2075,6 +2128,8 @@ dependencies = [ "checksum ct-logs 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4d3686f5fa27dbc1d76c751300376e167c5a43387f44bb451fd1c24776e49113" "checksum dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" "checksum dirs-sys 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "afa0b23de8fd801745c471deffa6e12d248f962c9fd4b4c33787b055599bde7b" +"checksum dlopen 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "71e80ad39f814a9abe68583cd50a2d45c8a67561c3361ab8da240587dda80937" +"checksum dlopen_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f236d9e1b1fbd81cea0f9cbdc8dcc7e8ebcd80e6659cd7cb2ad5f6c05946c581" "checksum downcast-rs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "52ba6eb47c2131e784a38b726eb54c1e1484904f013e576a25354d0124161af6" "checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e" "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" @@ -2144,8 +2199,10 @@ dependencies = [ "checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" "checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" "checksum proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" +"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" "checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27" "checksum publicsuffix 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9bf259a81de2b2eb9850ec990ec78e6a25319715584fd7652b9b26f96fcb1510" +"checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" "checksum rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" @@ -2194,6 +2251,7 @@ dependencies = [ "checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +"checksum syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)" = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" "checksum syn 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0e7bedb3320d0f3035594b0b723c8a28d7d336a3eda3881db79e61d676fb644c" "checksum synstructure 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "575be94ccb86e8da37efb894a87e2b660be299b41d8ef347f9d6d79fbe61b1ba" "checksum sys-info 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)" = "0079fe39cec2c8215e21b0bc4ccec9031004c160b88358f531b601e96b77f0df" @@ -2226,6 +2284,7 @@ dependencies = [ "checksum unicode-normalization 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "09c8070a9942f5e7cfccd93f490fdebd230ee3c3c9f107cb25bad5351ef671cf" "checksum unicode-segmentation 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49f5526225fd8b77342d5986ab5f6055552e9c0776193b5b63fd53b46debfad7" "checksum unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7007dbd421b92cc6e28410fe7362e2e0a2503394908f417b68ec8d1c364c4e20" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" "checksum untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60369ef7a31de49bcb3f6ca728d4ba7300d9a1658f94c727d4cab8c8d9f4aece" "checksum url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index c3fdf66b953523..bdb2df0b481a4f 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -31,6 +31,7 @@ base64 = "0.11.0" byteorder = "1.3.2" clap = "2.33.0" dirs = "2.0.2" +dlopen = "0.1.8" futures = { version = "0.3", features = [ "compat", "io-compat" ] } http = "0.1.19" hyper = "0.12.35" diff --git a/cli/deno_error.rs b/cli/deno_error.rs index 346149cd16da26..7c4c24e3c90141 100644 --- a/cli/deno_error.rs +++ b/cli/deno_error.rs @@ -6,6 +6,7 @@ pub use crate::msg::ErrorKind; use deno::AnyError; use deno::ErrBox; use deno::ModuleResolutionError; +use dlopen::Error as DlopenError; use http::uri; use hyper; use reqwest; @@ -287,6 +288,19 @@ mod unix { } } +impl GetErrorKind for DlopenError { + fn kind(&self) -> ErrorKind { + use dlopen::Error::*; + match self { + NullCharacter(_) => ErrorKind::Other, + OpeningLibraryError(e) => GetErrorKind::kind(e), + SymbolGettingError(e) => GetErrorKind::kind(e), + NullSymbol => ErrorKind::Other, + AddrNotMatchingDll(e) => GetErrorKind::kind(e), + } + } +} + impl GetErrorKind for dyn AnyError { fn kind(&self) -> ErrorKind { use self::GetErrorKind as Get; @@ -320,6 +334,7 @@ impl GetErrorKind for dyn AnyError { .downcast_ref::() .map(Get::kind) }) + .or_else(|| self.downcast_ref::().map(Get::kind)) .or_else(|| unix_error_kind(self)) .unwrap_or_else(|| { panic!("Can't get ErrorKind for {:?}", self); diff --git a/cli/flags.rs b/cli/flags.rs index 5a95d39e8c535a..0cc8c68e42f21f 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -48,6 +48,7 @@ pub struct DenoFlags { pub net_whitelist: Vec, pub allow_env: bool, pub allow_run: bool, + pub allow_native: bool, pub allow_hrtime: bool, pub no_prompts: bool, pub no_fetch: bool, @@ -105,6 +106,11 @@ fn add_run_args<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> { .long("allow-run") .help("Allow running subprocesses"), ) + .arg( + Arg::with_name("allow-native") + .long("allow-native") + .help("Allow opening native plugins"), + ) .arg( Arg::with_name("allow-hrtime") .long("allow-hrtime") @@ -843,6 +849,9 @@ fn parse_run_args(mut flags: DenoFlags, matches: &ArgMatches) -> DenoFlags { if matches.is_present("allow-run") { flags.allow_run = true; } + if matches.is_present("allow-native") { + flags.allow_native = true; + } if matches.is_present("allow-hrtime") { flags.allow_hrtime = true; } @@ -853,6 +862,7 @@ fn parse_run_args(mut flags: DenoFlags, matches: &ArgMatches) -> DenoFlags { flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; + flags.allow_native = true; flags.allow_hrtime = true; } if matches.is_present("no-fetch") { @@ -970,6 +980,7 @@ pub fn flags_from_vec( flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; + flags.allow_native = true; flags.allow_hrtime = true; let code: &str = eval_match.value_of("code").unwrap(); argv.extend(vec![code.to_string()]); @@ -1048,6 +1059,7 @@ pub fn flags_from_vec( flags.allow_net = true; flags.allow_env = true; flags.allow_run = true; + flags.allow_native = true; argv.push(INSTALLER_URL.to_string()); if install_match.is_present("dir") { @@ -1138,6 +1150,7 @@ pub fn flags_from_vec( flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; + flags.allow_native = true; flags.allow_hrtime = true; argv.push(XEVAL_URL.to_string()); @@ -1181,6 +1194,7 @@ pub fn flags_from_vec( flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; + flags.allow_native = true; flags.allow_hrtime = true; DenoSubcommand::Repl } diff --git a/cli/ops/mod.rs b/cli/ops/mod.rs index 9b33d59189ec72..d039d5e132680c 100644 --- a/cli/ops/mod.rs +++ b/cli/ops/mod.rs @@ -13,6 +13,7 @@ pub mod fetch; pub mod files; pub mod fs; pub mod io; +pub mod native_plugins; pub mod net; pub mod os; pub mod permissions; diff --git a/cli/ops/native_plugins.rs b/cli/ops/native_plugins.rs new file mode 100644 index 00000000000000..e62a442e71b8c1 --- /dev/null +++ b/cli/ops/native_plugins.rs @@ -0,0 +1,103 @@ +use super::dispatch_json::{Deserialize, JsonOp, Value}; +use crate::fs as deno_fs; +use crate::ops::json_op; +use crate::state::ThreadSafeState; +use deno::*; +use dlopen::symbor::Library; +use std::collections::HashMap; +use std::ffi::OsStr; +use std::sync::Arc; +use std::sync::Mutex; + +pub fn init( + i: &mut Isolate, + s: &ThreadSafeState, + isolate_arc: Arc>, +) { + let isolate_arc_ = isolate_arc.clone(); + i.register_op( + "open_native_plugin", + s.core_op(json_op(s.stateful_op(move |state, args, zero_copy| { + op_open_native_plugin(&isolate_arc_, state, args, zero_copy) + }))), + ); +} + +fn open_plugin>(lib_path: P) -> Result { + debug!("Loading Native Plugin: {:#?}", lib_path.as_ref()); + + Library::open(lib_path).map_err(ErrBox::from) +} + +struct NativePluginResource { + lib: Library, + ops: Vec<(String, OpId)>, +} + +impl Resource for NativePluginResource {} + +struct InitContext { + isolate: Arc>, + plugin_rid: ResourceId, + ops: Vec<(String, OpId)>, +} + +impl PluginInitContext for InitContext { + fn register_op( + &mut self, + name: &str, + op: Box) -> CoreOp + Send + Sync + 'static>, + ) -> OpId { + let mut i = self.isolate.lock().unwrap(); + let opid = i.register_op(&format!("{}_{}", self.plugin_rid, name), op); + self.ops.push((name.to_string(), opid)); + opid + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct OpenNativePluginArgs { + filename: String, +} + +pub fn op_open_native_plugin( + isolate: &Arc>, + state: &ThreadSafeState, + args: Value, + _zero_copy: Option, +) -> Result { + let args: OpenNativePluginArgs = serde_json::from_value(args)?; + let (filename, filename_) = deno_fs::resolve_from_cwd(&args.filename)?; + + state.check_native(&filename_)?; + + let lib = open_plugin(filename)?; + let plugin_resource = NativePluginResource { + lib, + ops: Vec::new(), + }; + let mut table = state.lock_resource_table(); + let rid = table.add("native_plugin", Box::new(plugin_resource)); + let plugin_resource = table.get_mut::(rid).unwrap(); + + let init_fn = *unsafe { + plugin_resource + .lib + .symbol::("native_plugin_init") + }?; + let mut init_context = InitContext { + isolate: isolate.clone(), + plugin_rid: rid, + ops: Vec::new(), + }; + init_fn(&mut init_context); + plugin_resource.ops.append(&mut init_context.ops); + let ops: HashMap = plugin_resource + .ops + .iter() + .map(|record| (record.0.clone(), record.1)) + .collect(); + + Ok(JsonOp::Sync(json!({ "rid": rid, "ops": ops }))) +} diff --git a/cli/permissions.rs b/cli/permissions.rs index fe0a7d473c89ac..f1e61e26ee0c14 100644 --- a/cli/permissions.rs +++ b/cli/permissions.rs @@ -141,6 +141,7 @@ pub struct DenoPermissions { pub net_whitelist: Arc>, pub allow_env: PermissionAccessor, pub allow_run: PermissionAccessor, + pub allow_native: PermissionAccessor, pub allow_hrtime: PermissionAccessor, } @@ -157,6 +158,7 @@ impl DenoPermissions { net_whitelist: Arc::new(flags.net_whitelist.iter().cloned().collect()), allow_env: PermissionAccessor::from(flags.allow_env), allow_run: PermissionAccessor::from(flags.allow_run), + allow_native: PermissionAccessor::from(flags.allow_native), allow_hrtime: PermissionAccessor::from(flags.allow_hrtime), } } @@ -300,6 +302,13 @@ impl DenoPermissions { .request("Deno requests to access to high precision time.") } + pub fn check_native(&self, filename: &str) -> Result<(), ErrBox> { + self.allow_native.get_state().check( + &format!("access to open a native plugin: {}", filename), + "run again with the --allow-run flag", + ) + } + pub fn get_permission_state( &self, name: &str, diff --git a/cli/state.rs b/cli/state.rs index 245919e7f000c7..c421dcc990e798 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -288,6 +288,11 @@ impl ThreadSafeState { self.permissions.check_run() } + #[inline] + pub fn check_native(&self, filename: &str) -> Result<(), ErrBox> { + self.permissions.check_native(filename) + } + pub fn check_dyn_import( self: &Self, module_specifier: &ModuleSpecifier, diff --git a/core/lib.rs b/core/lib.rs index 31f717769e929b..f1becb5d7e8f9b 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -14,6 +14,7 @@ mod libdeno; mod module_specifier; mod modules; mod ops; +mod plugins; mod resources; mod shared_queue; @@ -27,6 +28,7 @@ pub use crate::libdeno::PinnedBuf; pub use crate::module_specifier::*; pub use crate::modules::*; pub use crate::ops::*; +pub use crate::plugins::*; pub use crate::resources::*; pub fn v8_version() -> &'static str { diff --git a/core/plugins.rs b/core/plugins.rs new file mode 100644 index 00000000000000..0c20bc0e45fad0 --- /dev/null +++ b/core/plugins.rs @@ -0,0 +1,22 @@ +use crate::libdeno::{OpId, PinnedBuf}; +use crate::ops::CoreOp; + +pub type PluginInitFn = fn(context: &mut dyn PluginInitContext); + +pub trait PluginInitContext { + fn register_op( + &mut self, + name: &str, + op: Box) -> CoreOp + Send + Sync + 'static>, + ) -> OpId; +} + +#[macro_export] +macro_rules! init_fn { + ($fn:path) => { + #[no_mangle] + pub fn native_plugin_init(context: &mut dyn PluginInitContext) { + $fn(context) + } + }; +} From 6758ea9d87a784e67a25411d42c5f032a9973185 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Mon, 18 Nov 2019 14:16:42 -0500 Subject: [PATCH 02/24] implement native plugins in typescript --- cli/js/deno.ts | 1 + cli/js/dispatch.ts | 18 +++++++++- cli/js/lib.deno_runtime.d.ts | 22 ++++++++++++ cli/js/native_plugins.ts | 66 ++++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 cli/js/native_plugins.ts diff --git a/cli/js/deno.ts b/cli/js/deno.ts index 2a72747271c678..ea98181de6f1ba 100644 --- a/cli/js/deno.ts +++ b/cli/js/deno.ts @@ -75,6 +75,7 @@ export { } from "./permissions.ts"; export { truncateSync, truncate } from "./truncate.ts"; export { FileInfo } from "./file_info.ts"; +export { openPlugin } from "./native_plugins.ts"; export { connect, dial, listen, Listener, Conn } from "./net.ts"; export { dialTLS, listenTLS } from "./tls.ts"; export { metrics, Metrics } from "./metrics.ts"; diff --git a/cli/js/dispatch.ts b/cli/js/dispatch.ts index c2690ad32596e2..9d9b91fc6a3a15 100644 --- a/cli/js/dispatch.ts +++ b/cli/js/dispatch.ts @@ -1,6 +1,7 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. import * as minimal from "./dispatch_minimal.ts"; import * as json from "./dispatch_json.ts"; +import { AsyncHandler } from "./native_plugins.ts"; // These consts are shared with Rust. Update with care. export let OP_READ: number; @@ -66,6 +67,16 @@ export let OP_CWD: number; export let OP_FETCH_ASSET: number; export let OP_DIAL_TLS: number; export let OP_HOSTNAME: number; +export let OP_OPEN_NATIVE_PLUGIN: number; + +const NATIVE_PLUGIN_ASYNC_HANDLER_MAP: Map = new Map(); + +export function setPluginAsyncHandler( + opId: number, + handler: AsyncHandler +): void { + NATIVE_PLUGIN_ASYNC_HANDLER_MAP.set(opId, handler); +} export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void { switch (opId) { @@ -109,6 +120,11 @@ export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void { json.asyncMsgFromRust(opId, ui8); break; default: - throw Error("bad async opId"); + const handler = NATIVE_PLUGIN_ASYNC_HANDLER_MAP.get(opId); + if (handler) { + handler(opId, ui8); + } else { + throw Error("bad async opId"); + } } } diff --git a/cli/js/lib.deno_runtime.d.ts b/cli/js/lib.deno_runtime.d.ts index c7c40398796b86..b519381a6c901b 100644 --- a/cli/js/lib.deno_runtime.d.ts +++ b/cli/js/lib.deno_runtime.d.ts @@ -967,6 +967,28 @@ declare namespace Deno { */ export function truncate(name: string, len?: number): Promise; + // @url js/native_plugins.d.ts + + export interface AsyncHandler { + (opId: number, msg: Uint8Array): void; + } + + export interface NativePluginOp { + dispatch( + control: Uint8Array, + zeroCopy?: ArrayBufferView | null + ): Uint8Array | null; + setAsyncHandler(handler: MessageCallback): void; + } + + export interface NativePlugin { + ops: { + [name: string]: NativePluginOp; + }; + } + + export function openPlugin(filename: string): NativePlugin; + // @url js/net.d.ts type Transport = "tcp"; diff --git a/cli/js/native_plugins.ts b/cli/js/native_plugins.ts new file mode 100644 index 00000000000000..abf4b4aba1055d --- /dev/null +++ b/cli/js/native_plugins.ts @@ -0,0 +1,66 @@ +import { sendSync } from "./dispatch_json.ts"; +import { OP_OPEN_NATIVE_PLUGIN, setPluginAsyncHandler } from "./dispatch.ts"; +import { core } from "./core.ts"; + +export interface AsyncHandler { + (opId: number, msg: Uint8Array): void; +} + +interface NativePluginOp { + dispatch( + control: Uint8Array, + zeroCopy?: ArrayBufferView | null + ): Uint8Array | null; + setAsyncHandler(handler: AsyncHandler): void; +} + +class NativePluginOpImpl implements NativePluginOp { + constructor(private readonly opId: number) {} + + dispatch( + control: Uint8Array, + zeroCopy?: ArrayBufferView | null + ): Uint8Array | null { + return core.dispatch(this.opId, control, zeroCopy); + } + + setAsyncHandler(handler: AsyncHandler): void { + setPluginAsyncHandler(this.opId, handler); + } +} + +// TODO(afinch7): add close method. + +interface NativePlugin { + ops: { + [name: string]: NativePluginOp; + }; +} + +class NativePluginImpl implements NativePlugin { + private _ops: { [name: string]: NativePluginOp } = {}; + + constructor(private readonly rid: number, ops: { [name: string]: number }) { + for (const op in ops) { + this._ops[op] = new NativePluginOpImpl(ops[op]); + } + } + + get ops(): { [name: string]: NativePluginOp } { + return Object.assign({}, this._ops); + } +} + +interface OpenPluginResponse { + rid: number; + ops: { + [name: string]: number; + }; +} + +export function openPlugin(filename: string): NativePlugin { + const response: OpenPluginResponse = sendSync(OP_OPEN_NATIVE_PLUGIN, { + filename + }); + return new NativePluginImpl(response.rid, response.ops); +} From 7dc9fed3b8f989b732aec5aef6fd3a216b097a16 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Mon, 18 Nov 2019 14:21:32 -0500 Subject: [PATCH 03/24] fix flags tests and adjust permissions for install --- cli/flags.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/flags.rs b/cli/flags.rs index 0cc8c68e42f21f..075bf3de730db1 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -1059,7 +1059,6 @@ pub fn flags_from_vec( flags.allow_net = true; flags.allow_env = true; flags.allow_run = true; - flags.allow_native = true; argv.push(INSTALLER_URL.to_string()); if install_match.is_present("dir") { @@ -1354,6 +1353,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, + allow_native: true, allow_hrtime: true, ..DenoFlags::default() } @@ -1504,6 +1504,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, + allow_native: true, allow_hrtime: true, ..DenoFlags::default() } @@ -1523,6 +1524,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, + allow_native: true, allow_hrtime: true, ..DenoFlags::default() } @@ -1550,6 +1552,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, + allow_native: true, allow_hrtime: true, ..DenoFlags::default() } From 631b25728efce1ffe258bb10021b9e8e0756af26 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Mon, 18 Nov 2019 15:38:09 -0500 Subject: [PATCH 04/24] wip: add test plugin --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + cli/tests/test_native_plugin/Cargo.toml | 13 +++++++++++++ cli/tests/test_native_plugin/mod.ts | 6 ++++++ cli/tests/test_native_plugin/src/lib.rs | 26 +++++++++++++++++++++++++ cli/worker.rs | 1 + 6 files changed, 54 insertions(+) create mode 100644 cli/tests/test_native_plugin/Cargo.toml create mode 100644 cli/tests/test_native_plugin/mod.ts create mode 100644 cli/tests/test_native_plugin/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index fbafd4a57ad576..b723944c924a12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1535,6 +1535,13 @@ dependencies = [ "wincolor 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "test_native_plugin" +version = "0.1.0" +dependencies = [ + "deno 0.24.0", +] + [[package]] name = "textwrap" version = "0.11.0" diff --git a/Cargo.toml b/Cargo.toml index a54477a120b0ea..df68c7121370de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,4 +4,5 @@ members = [ "core", "tools/hyper_hello", "deno_typescript", + "cli/tests/test_native_plugin" ] diff --git a/cli/tests/test_native_plugin/Cargo.toml b/cli/tests/test_native_plugin/Cargo.toml new file mode 100644 index 00000000000000..ad61564e4fddf9 --- /dev/null +++ b/cli/tests/test_native_plugin/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "test_native_plugin" +version = "0.1.0" +authors = ["afinch7 "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +crate-type = ["cdylib"] + +[dependencies] +deno = { path = "../../../core" } \ No newline at end of file diff --git a/cli/tests/test_native_plugin/mod.ts b/cli/tests/test_native_plugin/mod.ts new file mode 100644 index 00000000000000..fb181227541f1e --- /dev/null +++ b/cli/tests/test_native_plugin/mod.ts @@ -0,0 +1,6 @@ +const plugin = Deno.openPlugin( + "./../../../target/debug/libtest_native_plugin.so" +); +const op_test_io_async = plugin.ops.op_test_io_async; + +console.log(op_test_io_async); diff --git a/cli/tests/test_native_plugin/src/lib.rs b/cli/tests/test_native_plugin/src/lib.rs new file mode 100644 index 00000000000000..1c5b26e5b1f8c7 --- /dev/null +++ b/cli/tests/test_native_plugin/src/lib.rs @@ -0,0 +1,26 @@ +use deno::CoreOp; +use deno::Op; +use deno::PluginInitContext; +use deno::{Buf, PinnedBuf}; + +#[macro_use] +extern crate deno; + +fn init(context: &mut dyn PluginInitContext) { + context.register_op("test_io_sync", Box::new(op_test_io_sync)); +} +init_fn!(init); + +pub fn op_test_io_sync(data: &[u8], zero_copy: Option) -> CoreOp { + if let Some(buf) = zero_copy { + let data_str = std::str::from_utf8(&data[..]).unwrap(); + let buf_str = std::str::from_utf8(&buf[..]).unwrap(); + println!( + "Hello from native bindings. data: {} | zero_copy: {}", + data_str, buf_str + ); + } + let result = b"test"; + let result_box: Buf = Box::new(*result); + Op::Sync(result_box) +} diff --git a/cli/worker.rs b/cli/worker.rs index d5cc801d869a52..8218907b2ff658 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -57,6 +57,7 @@ impl Worker { ops::files::init(&mut i, &state); ops::fs::init(&mut i, &state); ops::io::init(&mut i, &state); + ops::native_plugins::init(&mut i, &state, isolate.clone()); ops::net::init(&mut i, &state); ops::tls::init(&mut i, &state); ops::os::init(&mut i, &state); From fe8ed8dbee505bde865012d21fffefbbdc673210 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Mon, 18 Nov 2019 17:03:43 -0500 Subject: [PATCH 05/24] rebase fixes --- cli/ops/native_plugins.rs | 22 +++++++++------------- cli/worker.rs | 3 ++- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/cli/ops/native_plugins.rs b/cli/ops/native_plugins.rs index e62a442e71b8c1..3315b4d4e656ac 100644 --- a/cli/ops/native_plugins.rs +++ b/cli/ops/native_plugins.rs @@ -7,18 +7,13 @@ use dlopen::symbor::Library; use std::collections::HashMap; use std::ffi::OsStr; use std::sync::Arc; -use std::sync::Mutex; -pub fn init( - i: &mut Isolate, - s: &ThreadSafeState, - isolate_arc: Arc>, -) { - let isolate_arc_ = isolate_arc.clone(); +pub fn init(i: &mut Isolate, s: &ThreadSafeState, r: Arc) { + let r_ = r.clone(); i.register_op( "open_native_plugin", s.core_op(json_op(s.stateful_op(move |state, args, zero_copy| { - op_open_native_plugin(&isolate_arc_, state, args, zero_copy) + op_open_native_plugin(&r_, state, args, zero_copy) }))), ); } @@ -37,7 +32,7 @@ struct NativePluginResource { impl Resource for NativePluginResource {} struct InitContext { - isolate: Arc>, + registry: Arc, plugin_rid: ResourceId, ops: Vec<(String, OpId)>, } @@ -48,8 +43,9 @@ impl PluginInitContext for InitContext { name: &str, op: Box) -> CoreOp + Send + Sync + 'static>, ) -> OpId { - let mut i = self.isolate.lock().unwrap(); - let opid = i.register_op(&format!("{}_{}", self.plugin_rid, name), op); + let opid = self + .registry + .register(&format!("{}_{}", self.plugin_rid, name), op); self.ops.push((name.to_string(), opid)); opid } @@ -62,7 +58,7 @@ struct OpenNativePluginArgs { } pub fn op_open_native_plugin( - isolate: &Arc>, + registry: &Arc, state: &ThreadSafeState, args: Value, _zero_copy: Option, @@ -87,7 +83,7 @@ pub fn op_open_native_plugin( .symbol::("native_plugin_init") }?; let mut init_context = InitContext { - isolate: isolate.clone(), + registry: registry.clone(), plugin_rid: rid, ops: Vec::new(), }; diff --git a/cli/worker.rs b/cli/worker.rs index 8218907b2ff658..1594e740ce8ec3 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -50,6 +50,7 @@ impl Worker { let isolate = Arc::new(Mutex::new(deno::Isolate::new(startup_data, false))); { let mut i = isolate.lock().unwrap(); + let op_registry = i.op_registry.clone(); ops::compiler::init(&mut i, &state); ops::errors::init(&mut i, &state); @@ -57,7 +58,7 @@ impl Worker { ops::files::init(&mut i, &state); ops::fs::init(&mut i, &state); ops::io::init(&mut i, &state); - ops::native_plugins::init(&mut i, &state, isolate.clone()); + ops::native_plugins::init(&mut i, &state, op_registry); ops::net::init(&mut i, &state); ops::tls::init(&mut i, &state); ops::os::init(&mut i, &state); From 112622d214baa723da18ade942e0ae301b86c9d8 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Tue, 19 Nov 2019 16:36:56 -0500 Subject: [PATCH 06/24] added tests and fixed a issue with types --- cli/js/lib.deno_runtime.d.ts | 2 +- cli/tests/052_native_plugins.ts | 29 +++++++++++++++++++++++++++++ cli/tests/052_native_plugins.ts.out | 2 ++ cli/tests/integration_tests.rs | 14 ++++++++++++++ cli/tests/test_native_plugin/mod.ts | 6 ------ 5 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 cli/tests/052_native_plugins.ts create mode 100644 cli/tests/052_native_plugins.ts.out diff --git a/cli/js/lib.deno_runtime.d.ts b/cli/js/lib.deno_runtime.d.ts index b519381a6c901b..6205dc28d28332 100644 --- a/cli/js/lib.deno_runtime.d.ts +++ b/cli/js/lib.deno_runtime.d.ts @@ -978,7 +978,7 @@ declare namespace Deno { control: Uint8Array, zeroCopy?: ArrayBufferView | null ): Uint8Array | null; - setAsyncHandler(handler: MessageCallback): void; + setAsyncHandler(handler: AsyncHandler): void; } export interface NativePlugin { diff --git a/cli/tests/052_native_plugins.ts b/cli/tests/052_native_plugins.ts new file mode 100644 index 00000000000000..2df078ab88d8e4 --- /dev/null +++ b/cli/tests/052_native_plugins.ts @@ -0,0 +1,29 @@ +const filenameBase = "test_native_plugin"; + +let filenameSuffix = ".so"; +let filenamePrefix = "lib"; + +if (Deno.build.os === "win") { + filenameSuffix = ".dll"; + filenamePrefix = ""; +} +if (Deno.build.os === "mac") { + filenameSuffix = ".dylib"; +} + +const filename = `${filenamePrefix}${filenameBase}${filenameSuffix}`; + +const plugin = Deno.openPlugin(`./../../target/${Deno.args[1]}/${filename}`); + +// eslint-disable-next-line @typescript-eslint/camelcase +const test_io_sync = plugin.ops.test_io_sync; + +const textDecoder = new TextDecoder(); + +// eslint-disable-next-line @typescript-eslint/camelcase +const response = test_io_sync.dispatch( + new Uint8Array([116, 101, 115, 116]), + new Uint8Array([116, 101, 115, 116]) +); + +console.log(`Native Binding Response: ${textDecoder.decode(response)}`); diff --git a/cli/tests/052_native_plugins.ts.out b/cli/tests/052_native_plugins.ts.out new file mode 100644 index 00000000000000..b8cfcce4567f5f --- /dev/null +++ b/cli/tests/052_native_plugins.ts.out @@ -0,0 +1,2 @@ +Hello from native bindings. data: test | zero_copy: test +Native Binding Response: test diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 195202d49c0b87..cc45a5b3b98dd4 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -356,6 +356,20 @@ itest!(_051_wasm_import { http_server: true, }); +#[cfg(not(debug_assertions))] +itest!(_052_native_plugins_release { + args: "run --reload --allow-native 052_native_plugins.ts release", + output: "052_native_plugins.ts.out", + http_server: false, +}); + +#[cfg(debug_assertions)] +itest!(_052_native_plugins_debug { + args: "run --reload --allow-native 052_native_plugins.ts debug", + output: "052_native_plugins.ts.out", + http_server: false, +}); + itest!(lock_check_ok { args: "run --lock=lock_check_ok.json http://127.0.0.1:4545/cli/tests/003_relative_import.ts", output: "003_relative_import.ts.out", diff --git a/cli/tests/test_native_plugin/mod.ts b/cli/tests/test_native_plugin/mod.ts index fb181227541f1e..e69de29bb2d1d6 100644 --- a/cli/tests/test_native_plugin/mod.ts +++ b/cli/tests/test_native_plugin/mod.ts @@ -1,6 +0,0 @@ -const plugin = Deno.openPlugin( - "./../../../target/debug/libtest_native_plugin.so" -); -const op_test_io_async = plugin.ops.op_test_io_async; - -console.log(op_test_io_async); From cb94110c2170e5a766d82759415caf1de7ad63e2 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Tue, 19 Nov 2019 19:40:13 -0500 Subject: [PATCH 07/24] refactor `InitContext` --- cli/js/dispatch.ts | 5 ++--- cli/js/native_plugins.ts | 8 ++------ cli/ops/native_plugins.rs | 42 ++++++++++++++++----------------------- core/ops.rs | 2 +- core/plugins.rs | 4 ++-- 5 files changed, 24 insertions(+), 37 deletions(-) diff --git a/cli/js/dispatch.ts b/cli/js/dispatch.ts index 9d9b91fc6a3a15..f0bb7dd5da041c 100644 --- a/cli/js/dispatch.ts +++ b/cli/js/dispatch.ts @@ -1,7 +1,6 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. import * as minimal from "./dispatch_minimal.ts"; import * as json from "./dispatch_json.ts"; -import { AsyncHandler } from "./native_plugins.ts"; // These consts are shared with Rust. Update with care. export let OP_READ: number; @@ -69,11 +68,11 @@ export let OP_DIAL_TLS: number; export let OP_HOSTNAME: number; export let OP_OPEN_NATIVE_PLUGIN: number; -const NATIVE_PLUGIN_ASYNC_HANDLER_MAP: Map = new Map(); +const NATIVE_PLUGIN_ASYNC_HANDLER_MAP: Map = new Map(); export function setPluginAsyncHandler( opId: number, - handler: AsyncHandler + handler: MessageCallback ): void { NATIVE_PLUGIN_ASYNC_HANDLER_MAP.set(opId, handler); } diff --git a/cli/js/native_plugins.ts b/cli/js/native_plugins.ts index abf4b4aba1055d..af41fa8889d760 100644 --- a/cli/js/native_plugins.ts +++ b/cli/js/native_plugins.ts @@ -2,16 +2,12 @@ import { sendSync } from "./dispatch_json.ts"; import { OP_OPEN_NATIVE_PLUGIN, setPluginAsyncHandler } from "./dispatch.ts"; import { core } from "./core.ts"; -export interface AsyncHandler { - (opId: number, msg: Uint8Array): void; -} - interface NativePluginOp { dispatch( control: Uint8Array, zeroCopy?: ArrayBufferView | null ): Uint8Array | null; - setAsyncHandler(handler: AsyncHandler): void; + setAsyncHandler(handler: MessageCallback): void; } class NativePluginOpImpl implements NativePluginOp { @@ -24,7 +20,7 @@ class NativePluginOpImpl implements NativePluginOp { return core.dispatch(this.opId, control, zeroCopy); } - setAsyncHandler(handler: AsyncHandler): void { + setAsyncHandler(handler: MessageCallback): void { setPluginAsyncHandler(this.opId, handler); } } diff --git a/cli/ops/native_plugins.rs b/cli/ops/native_plugins.rs index 3315b4d4e656ac..4c48841107c36e 100644 --- a/cli/ops/native_plugins.rs +++ b/cli/ops/native_plugins.rs @@ -26,28 +26,22 @@ fn open_plugin>(lib_path: P) -> Result { struct NativePluginResource { lib: Library, - ops: Vec<(String, OpId)>, + ops: HashMap, } impl Resource for NativePluginResource {} struct InitContext { - registry: Arc, - plugin_rid: ResourceId, - ops: Vec<(String, OpId)>, + ops: HashMap>, } impl PluginInitContext for InitContext { - fn register_op( - &mut self, - name: &str, - op: Box) -> CoreOp + Send + Sync + 'static>, - ) -> OpId { - let opid = self - .registry - .register(&format!("{}_{}", self.plugin_rid, name), op); - self.ops.push((name.to_string(), opid)); - opid + fn register_op(&mut self, name: &str, op: Box) { + let existing = self.ops.insert(name.to_string(), op); + assert!( + existing.is_none(), + format!("Op already registered: {}", name) + ); } } @@ -71,7 +65,7 @@ pub fn op_open_native_plugin( let lib = open_plugin(filename)?; let plugin_resource = NativePluginResource { lib, - ops: Vec::new(), + ops: HashMap::new(), }; let mut table = state.lock_resource_table(); let rid = table.add("native_plugin", Box::new(plugin_resource)); @@ -83,17 +77,15 @@ pub fn op_open_native_plugin( .symbol::("native_plugin_init") }?; let mut init_context = InitContext { - registry: registry.clone(), - plugin_rid: rid, - ops: Vec::new(), + ops: HashMap::new(), }; init_fn(&mut init_context); - plugin_resource.ops.append(&mut init_context.ops); - let ops: HashMap = plugin_resource - .ops - .iter() - .map(|record| (record.0.clone(), record.1)) - .collect(); + for op in init_context.ops { + let op_id = registry.register(&op.0, op.1); + plugin_resource.ops.insert(op.0, op_id); + } - Ok(JsonOp::Sync(json!({ "rid": rid, "ops": ops }))) + Ok(JsonOp::Sync( + json!({ "rid": rid, "ops": plugin_resource.ops }), + )) } diff --git a/core/ops.rs b/core/ops.rs index 6dc0a732392eb9..c840ed979e2d0b 100644 --- a/core/ops.rs +++ b/core/ops.rs @@ -27,7 +27,7 @@ pub type CoreError = (); pub type CoreOp = Op; /// Main type describing op -type OpDispatcher = +pub type OpDispatcher = dyn Fn(&[u8], Option) -> CoreOp + Send + Sync + 'static; #[derive(Default)] diff --git a/core/plugins.rs b/core/plugins.rs index 0c20bc0e45fad0..92b1036f9f37dc 100644 --- a/core/plugins.rs +++ b/core/plugins.rs @@ -1,4 +1,4 @@ -use crate::libdeno::{OpId, PinnedBuf}; +use crate::libdeno::PinnedBuf; use crate::ops::CoreOp; pub type PluginInitFn = fn(context: &mut dyn PluginInitContext); @@ -8,7 +8,7 @@ pub trait PluginInitContext { &mut self, name: &str, op: Box) -> CoreOp + Send + Sync + 'static>, - ) -> OpId; + ); } #[macro_export] From 2913dc97404d91b664b64209b379d9e7a2e0a859 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Fri, 22 Nov 2019 12:45:23 -0500 Subject: [PATCH 08/24] patch by @bartlomieju --- cli/ops/workers.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/ops/workers.rs b/cli/ops/workers.rs index 42f93ec5736f3a..cbc0c560870f2c 100644 --- a/cli/ops/workers.rs +++ b/cli/ops/workers.rs @@ -216,7 +216,12 @@ fn op_host_get_worker_closed( }; let op = future.then(move |_result| { let mut workers_table = state_.workers.lock().unwrap(); - workers_table.remove(&id); + let maybe_worker = workers_table.remove(&id); + if let Some(worker) = maybe_worker { + let mut channels = worker.state.worker_channels.lock().unwrap(); + channels.sender.close_channel(); + channels.receiver.close(); + }; futures::future::ok(json!({})) }); From 496329a0afb3c35b379e64e1671d78e051b3dd98 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Fri, 22 Nov 2019 13:13:45 -0500 Subject: [PATCH 09/24] fix issue with `setAsyncHandler` and added simple async test --- Cargo.lock | 1 + cli/js/dispatch.ts | 7 ++++--- cli/js/lib.deno_runtime.d.ts | 2 +- cli/js/native_plugins.ts | 8 +++++-- cli/tests/052_native_plugins.ts | 21 ++++++++++++++++--- cli/tests/052_native_plugins.ts.out | 4 +++- cli/tests/test_native_plugin/Cargo.toml | 3 ++- cli/tests/test_native_plugin/src/lib.rs | 28 ++++++++++++++++++++++--- 8 files changed, 60 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b723944c924a12..ed962bb032f8c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1540,6 +1540,7 @@ name = "test_native_plugin" version = "0.1.0" dependencies = [ "deno 0.24.0", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/cli/js/dispatch.ts b/cli/js/dispatch.ts index f0bb7dd5da041c..9c3de75685ba60 100644 --- a/cli/js/dispatch.ts +++ b/cli/js/dispatch.ts @@ -1,6 +1,7 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. import * as minimal from "./dispatch_minimal.ts"; import * as json from "./dispatch_json.ts"; +import { AsyncHandler } from "./native_plugins.ts"; // These consts are shared with Rust. Update with care. export let OP_READ: number; @@ -68,11 +69,11 @@ export let OP_DIAL_TLS: number; export let OP_HOSTNAME: number; export let OP_OPEN_NATIVE_PLUGIN: number; -const NATIVE_PLUGIN_ASYNC_HANDLER_MAP: Map = new Map(); +const NATIVE_PLUGIN_ASYNC_HANDLER_MAP: Map = new Map(); export function setPluginAsyncHandler( opId: number, - handler: MessageCallback + handler: AsyncHandler ): void { NATIVE_PLUGIN_ASYNC_HANDLER_MAP.set(opId, handler); } @@ -121,7 +122,7 @@ export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void { default: const handler = NATIVE_PLUGIN_ASYNC_HANDLER_MAP.get(opId); if (handler) { - handler(opId, ui8); + handler(ui8); } else { throw Error("bad async opId"); } diff --git a/cli/js/lib.deno_runtime.d.ts b/cli/js/lib.deno_runtime.d.ts index 6205dc28d28332..b2879d7a836925 100644 --- a/cli/js/lib.deno_runtime.d.ts +++ b/cli/js/lib.deno_runtime.d.ts @@ -970,7 +970,7 @@ declare namespace Deno { // @url js/native_plugins.d.ts export interface AsyncHandler { - (opId: number, msg: Uint8Array): void; + (msg: Uint8Array): void; } export interface NativePluginOp { diff --git a/cli/js/native_plugins.ts b/cli/js/native_plugins.ts index af41fa8889d760..6e333c65948d4b 100644 --- a/cli/js/native_plugins.ts +++ b/cli/js/native_plugins.ts @@ -2,12 +2,16 @@ import { sendSync } from "./dispatch_json.ts"; import { OP_OPEN_NATIVE_PLUGIN, setPluginAsyncHandler } from "./dispatch.ts"; import { core } from "./core.ts"; +export interface AsyncHandler { + (msg: Uint8Array): void; +} + interface NativePluginOp { dispatch( control: Uint8Array, zeroCopy?: ArrayBufferView | null ): Uint8Array | null; - setAsyncHandler(handler: MessageCallback): void; + setAsyncHandler(handler: AsyncHandler): void; } class NativePluginOpImpl implements NativePluginOp { @@ -20,7 +24,7 @@ class NativePluginOpImpl implements NativePluginOp { return core.dispatch(this.opId, control, zeroCopy); } - setAsyncHandler(handler: MessageCallback): void { + setAsyncHandler(handler: AsyncHandler): void { setPluginAsyncHandler(this.opId, handler); } } diff --git a/cli/tests/052_native_plugins.ts b/cli/tests/052_native_plugins.ts index 2df078ab88d8e4..f35264d20a5537 100644 --- a/cli/tests/052_native_plugins.ts +++ b/cli/tests/052_native_plugins.ts @@ -17,13 +17,28 @@ const plugin = Deno.openPlugin(`./../../target/${Deno.args[1]}/${filename}`); // eslint-disable-next-line @typescript-eslint/camelcase const test_io_sync = plugin.ops.test_io_sync; +// eslint-disable-next-line @typescript-eslint/camelcase +const test_io_async = plugin.ops.test_io_async; const textDecoder = new TextDecoder(); -// eslint-disable-next-line @typescript-eslint/camelcase -const response = test_io_sync.dispatch( +{ + // eslint-disable-next-line @typescript-eslint/camelcase + const response = test_io_sync.dispatch( + new Uint8Array([116, 101, 115, 116]), + new Uint8Array([116, 101, 115, 116]) + ); + + console.log(`Native Binding Sync Response: ${textDecoder.decode(response)}`); +} + +test_io_async.setAsyncHandler(response => { + console.log(`Native Binding Async Response: ${textDecoder.decode(response)}`); +}); + +const response = test_io_async.dispatch( new Uint8Array([116, 101, 115, 116]), new Uint8Array([116, 101, 115, 116]) ); -console.log(`Native Binding Response: ${textDecoder.decode(response)}`); +console.log(`Native Binding Async Response: ${textDecoder.decode(response)}`); diff --git a/cli/tests/052_native_plugins.ts.out b/cli/tests/052_native_plugins.ts.out index b8cfcce4567f5f..252b1be26350c7 100644 --- a/cli/tests/052_native_plugins.ts.out +++ b/cli/tests/052_native_plugins.ts.out @@ -1,2 +1,4 @@ Hello from native bindings. data: test | zero_copy: test -Native Binding Response: test +Native Binding Sync Response: test +Hello from native bindings. data: test | zero_copy: test +Native Binding Async Response: test diff --git a/cli/tests/test_native_plugin/Cargo.toml b/cli/tests/test_native_plugin/Cargo.toml index ad61564e4fddf9..bf3e757dc87c08 100644 --- a/cli/tests/test_native_plugin/Cargo.toml +++ b/cli/tests/test_native_plugin/Cargo.toml @@ -10,4 +10,5 @@ edition = "2018" crate-type = ["cdylib"] [dependencies] -deno = { path = "../../../core" } \ No newline at end of file +deno = { path = "../../../core" } +futures = "0.3" \ No newline at end of file diff --git a/cli/tests/test_native_plugin/src/lib.rs b/cli/tests/test_native_plugin/src/lib.rs index 1c5b26e5b1f8c7..e893fb0c70852d 100644 --- a/cli/tests/test_native_plugin/src/lib.rs +++ b/cli/tests/test_native_plugin/src/lib.rs @@ -1,13 +1,16 @@ +#[macro_use] +extern crate deno; +extern crate futures; + use deno::CoreOp; use deno::Op; use deno::PluginInitContext; use deno::{Buf, PinnedBuf}; - -#[macro_use] -extern crate deno; +use futures::future::FutureExt; fn init(context: &mut dyn PluginInitContext) { context.register_op("test_io_sync", Box::new(op_test_io_sync)); + context.register_op("test_io_async", Box::new(op_test_io_async)); } init_fn!(init); @@ -24,3 +27,22 @@ pub fn op_test_io_sync(data: &[u8], zero_copy: Option) -> CoreOp { let result_box: Buf = Box::new(*result); Op::Sync(result_box) } + +pub fn op_test_io_async(data: &[u8], zero_copy: Option) -> CoreOp { + let data_str = std::str::from_utf8(&data[..]).unwrap().to_string(); + let fut = async move { + if let Some(buf) = zero_copy { + let buf_str = std::str::from_utf8(&buf[..]).unwrap(); + println!( + "Hello from native bindings. data: {} | zero_copy: {}", + data_str, buf_str + ); + } + // TODO(afinch7): add a delayed async of some type in here. + let result = b"test"; + let result_box: Buf = Box::new(*result); + Ok(result_box) + }; + + Op::Async(fut.boxed()) +} From 62c126c78b6bd6e34401d202a32309e188da474c Mon Sep 17 00:00:00 2001 From: afinch7 Date: Fri, 22 Nov 2019 13:17:36 -0500 Subject: [PATCH 10/24] make sure test native plugin gets built for tests --- cli/tests/052_native_plugins.ts | 2 ++ cli/tests/test_native_plugin/src/lib.rs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/cli/tests/052_native_plugins.ts b/cli/tests/052_native_plugins.ts index f35264d20a5537..299ad1ea93c5ae 100644 --- a/cli/tests/052_native_plugins.ts +++ b/cli/tests/052_native_plugins.ts @@ -32,10 +32,12 @@ const textDecoder = new TextDecoder(); console.log(`Native Binding Sync Response: ${textDecoder.decode(response)}`); } +// eslint-disable-next-line @typescript-eslint/camelcase test_io_async.setAsyncHandler(response => { console.log(`Native Binding Async Response: ${textDecoder.decode(response)}`); }); +// eslint-disable-next-line @typescript-eslint/camelcase const response = test_io_async.dispatch( new Uint8Array([116, 101, 115, 116]), new Uint8Array([116, 101, 115, 116]) diff --git a/cli/tests/test_native_plugin/src/lib.rs b/cli/tests/test_native_plugin/src/lib.rs index e893fb0c70852d..d0897b347fe43f 100644 --- a/cli/tests/test_native_plugin/src/lib.rs +++ b/cli/tests/test_native_plugin/src/lib.rs @@ -46,3 +46,8 @@ pub fn op_test_io_async(data: &[u8], zero_copy: Option) -> CoreOp { Op::Async(fut.boxed()) } + +#[test] +fn placeholder() { + // This test is just here to make sure the plugin gets built +} From 102831e1e68e73a1871b917f6e65572aca962d46 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Fri, 22 Nov 2019 13:41:22 -0500 Subject: [PATCH 11/24] fix debug test --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbdf02d57dde7c..b903ff996ce99d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,6 +129,7 @@ jobs: - name: Test debug if: matrix.kind == 'test_debug' run: | + cargo build --locked -p test_native_plugin echo ::set-env name=DENO_BUILD_MODE::debug cargo test --locked --all-targets From 96d4afc80a80304d77be19922fec2ae2bc173377 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Sat, 23 Nov 2019 11:32:53 -0500 Subject: [PATCH 12/24] review fixes and other misc fixes * Rename `allow_native`/`allow-native` to `allow_plugin`/`allow-plugin`. This makes more sense with the primary public api function being `openPlugin`. * Fully intagrate the newly added permission type `plugin`. * Add docs to `Deno.openPlugin`. * Generate op names for native plugin ops that avoid potential collision. Lastly this commit adds a delayed future to native plugins async test op. This should properly test the function of contexts/tasks/wakers in async native plugin ops. This would not function correctly with futures 0.1, and thus was a major blocker. This test proves this issue no longer exists in this implementation. The most analogous problem I can think of here is loading two copies of the same TS lib: the share the same type thus are interoperable, but don't share the same module local data(closest comparison to TLS used to store current task in futures). This isn't a problem anymore, since futures now pass `Context` values directly during calls to `poll` with the new API. --- cli/flags.rs | 26 +++++++++--------- cli/js/lib.deno_runtime.d.ts | 13 +++++++++ cli/js/permissions.ts | 5 ++++ cli/js/permissions_test.ts | 14 +++++++--- cli/js/test_util.ts | 14 +++++++++- cli/ops/native_plugins.rs | 10 +++++-- cli/ops/permissions.rs | 2 ++ cli/permissions.rs | 36 ++++++++++++++++++++----- cli/state.rs | 4 +-- cli/tests/052_native_plugins.ts | 4 ++- cli/tests/integration_tests.rs | 4 +-- cli/tests/test_native_plugin/src/lib.rs | 7 ++++- 12 files changed, 107 insertions(+), 32 deletions(-) diff --git a/cli/flags.rs b/cli/flags.rs index 075bf3de730db1..0861697432fdf2 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -48,7 +48,7 @@ pub struct DenoFlags { pub net_whitelist: Vec, pub allow_env: bool, pub allow_run: bool, - pub allow_native: bool, + pub allow_plugin: bool, pub allow_hrtime: bool, pub no_prompts: bool, pub no_fetch: bool, @@ -107,8 +107,8 @@ fn add_run_args<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> { .help("Allow running subprocesses"), ) .arg( - Arg::with_name("allow-native") - .long("allow-native") + Arg::with_name("allow-plugin") + .long("allow-plugin") .help("Allow opening native plugins"), ) .arg( @@ -849,8 +849,8 @@ fn parse_run_args(mut flags: DenoFlags, matches: &ArgMatches) -> DenoFlags { if matches.is_present("allow-run") { flags.allow_run = true; } - if matches.is_present("allow-native") { - flags.allow_native = true; + if matches.is_present("allow-plugin") { + flags.allow_plugin = true; } if matches.is_present("allow-hrtime") { flags.allow_hrtime = true; @@ -862,7 +862,7 @@ fn parse_run_args(mut flags: DenoFlags, matches: &ArgMatches) -> DenoFlags { flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; - flags.allow_native = true; + flags.allow_plugin = true; flags.allow_hrtime = true; } if matches.is_present("no-fetch") { @@ -980,7 +980,7 @@ pub fn flags_from_vec( flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; - flags.allow_native = true; + flags.allow_plugin = true; flags.allow_hrtime = true; let code: &str = eval_match.value_of("code").unwrap(); argv.extend(vec![code.to_string()]); @@ -1149,7 +1149,7 @@ pub fn flags_from_vec( flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; - flags.allow_native = true; + flags.allow_plugin = true; flags.allow_hrtime = true; argv.push(XEVAL_URL.to_string()); @@ -1193,7 +1193,7 @@ pub fn flags_from_vec( flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; - flags.allow_native = true; + flags.allow_plugin = true; flags.allow_hrtime = true; DenoSubcommand::Repl } @@ -1353,7 +1353,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, - allow_native: true, + allow_plugin: true, allow_hrtime: true, ..DenoFlags::default() } @@ -1504,7 +1504,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, - allow_native: true, + allow_plugin: true, allow_hrtime: true, ..DenoFlags::default() } @@ -1524,7 +1524,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, - allow_native: true, + allow_plugin: true, allow_hrtime: true, ..DenoFlags::default() } @@ -1552,7 +1552,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, - allow_native: true, + allow_plugin: true, allow_hrtime: true, ..DenoFlags::default() } diff --git a/cli/js/lib.deno_runtime.d.ts b/cli/js/lib.deno_runtime.d.ts index b2879d7a836925..5b3e05040dfd5b 100644 --- a/cli/js/lib.deno_runtime.d.ts +++ b/cli/js/lib.deno_runtime.d.ts @@ -892,6 +892,7 @@ declare namespace Deno { | "write" | "net" | "env" + | "plugin" | "hrtime"; /** https://w3c.github.io/permissions/#status-of-a-permission */ export type PermissionState = "granted" | "denied" | "prompt"; @@ -909,6 +910,9 @@ declare namespace Deno { interface EnvPermissionDescriptor { name: "env"; } + interface PluginPermissionDescriptor { + name: "plugin"; + } interface HrtimePermissionDescriptor { name: "hrtime"; } @@ -918,6 +922,7 @@ declare namespace Deno { | ReadWritePermissionDescriptor | NetPermissionDescriptor | EnvPermissionDescriptor + | PluginPermissionDescriptor | HrtimePermissionDescriptor; export class Permissions { @@ -987,6 +992,14 @@ declare namespace Deno { }; } + /** Open and initalize a native plugin. + * Requires the `--allow-plugin` flag. + * + * const plugin = Deno.openPlugin("./path/to/some/plugin.so"); + * const some_op = plugin.ops.some_op; + * const response = some_op.dispatch(new Uint8Array([1,2,3,4])); + * console.log(`Response from native plugin ${response}`); + */ export function openPlugin(filename: string): NativePlugin; // @url js/net.d.ts diff --git a/cli/js/permissions.ts b/cli/js/permissions.ts index c3530e9700ae04..e0fb8a84c18e56 100644 --- a/cli/js/permissions.ts +++ b/cli/js/permissions.ts @@ -11,6 +11,7 @@ export type PermissionName = | "net" | "env" | "run" + | "plugin" | "hrtime"; // NOTE: Keep in sync with cli/permissions.rs @@ -31,6 +32,9 @@ interface NetPermissionDescriptor { interface EnvPermissionDescriptor { name: "env"; } +interface PluginPermissionDescriptor { + name: "plugin"; +} interface HrtimePermissionDescriptor { name: "hrtime"; } @@ -40,6 +44,7 @@ type PermissionDescriptor = | ReadWritePermissionDescriptor | NetPermissionDescriptor | EnvPermissionDescriptor + | PluginPermissionDescriptor | HrtimePermissionDescriptor; /** https://w3c.github.io/permissions/#permissionstatus */ diff --git a/cli/js/permissions_test.ts b/cli/js/permissions_test.ts index d9ba538f0ed7f9..a50718652fb291 100644 --- a/cli/js/permissions_test.ts +++ b/cli/js/permissions_test.ts @@ -7,11 +7,12 @@ const knownPermissions: Deno.PermissionName[] = [ "write", "net", "env", + "plugin", "hrtime" ]; -for (const grant of knownPermissions) { - testPerm({ [grant]: true }, async function envGranted(): Promise { +function genFunc(grant: Deno.PermissionName): () => Promise { + const gen: () => Promise = async function Granted(): Promise { const status0 = await Deno.permissions.query({ name: grant }); assert(status0 != null); assertEquals(status0.state, "granted"); @@ -19,7 +20,14 @@ for (const grant of knownPermissions) { const status1 = await Deno.permissions.revoke({ name: grant }); assert(status1 != null); assertEquals(status1.state, "prompt"); - }); + }; + // Properly name these generated functions. + Object.defineProperty(gen, "name", { value: grant + "Granted" }); + return gen; +} + +for (const grant of knownPermissions) { + testPerm({ [grant]: true }, genFunc(grant)); } test(async function permissionInvalidName(): Promise { diff --git a/cli/js/test_util.ts b/cli/js/test_util.ts index 85fffabe645d67..aac70d2edeabe8 100644 --- a/cli/js/test_util.ts +++ b/cli/js/test_util.ts @@ -26,6 +26,7 @@ interface TestPermissions { net?: boolean; env?: boolean; run?: boolean; + plugin?: boolean; hrtime?: boolean; } @@ -35,6 +36,7 @@ export interface Permissions { net: boolean; env: boolean; run: boolean; + plugin: boolean; hrtime: boolean; } @@ -48,6 +50,7 @@ async function getProcessPermissions(): Promise { write: await isGranted("write"), net: await isGranted("net"), env: await isGranted("env"), + plugin: await isGranted("plugin"), hrtime: await isGranted("hrtime") }; } @@ -75,8 +78,9 @@ function permToString(perms: Permissions): string { const n = perms.net ? 1 : 0; const e = perms.env ? 1 : 0; const u = perms.run ? 1 : 0; + const p = perms.plugin ? 1 : 0; const h = perms.hrtime ? 1 : 0; - return `permR${r}W${w}N${n}E${e}U${u}H${h}`; + return `permR${r}W${w}N${n}E${e}U${u}P${p}H${h}`; } function registerPermCombination(perms: Permissions): void { @@ -93,6 +97,7 @@ function normalizeTestPermissions(perms: TestPermissions): Permissions { net: !!perms.net, run: !!perms.run, env: !!perms.env, + plugin: !!perms.plugin, hrtime: !!perms.hrtime }; } @@ -120,6 +125,7 @@ export function test(fn: testing.TestFunction): void { net: false, env: false, run: false, + plugin: false, hrtime: false }, fn @@ -176,6 +182,7 @@ test(function permissionsMatches(): void { net: false, env: false, run: false, + plugin: false, hrtime: false }, normalizeTestPermissions({ read: true }) @@ -190,6 +197,7 @@ test(function permissionsMatches(): void { net: false, env: false, run: false, + plugin: false, hrtime: false }, normalizeTestPermissions({}) @@ -204,6 +212,7 @@ test(function permissionsMatches(): void { net: true, env: true, run: true, + plugin: true, hrtime: true }, normalizeTestPermissions({ read: true }) @@ -219,6 +228,7 @@ test(function permissionsMatches(): void { net: true, env: false, run: false, + plugin: false, hrtime: false }, normalizeTestPermissions({ read: true }) @@ -234,6 +244,7 @@ test(function permissionsMatches(): void { net: true, env: true, run: true, + plugin: true, hrtime: true }, { @@ -242,6 +253,7 @@ test(function permissionsMatches(): void { net: true, env: true, run: true, + plugin: true, hrtime: true } ) diff --git a/cli/ops/native_plugins.rs b/cli/ops/native_plugins.rs index 4c48841107c36e..68ae08d511bf2a 100644 --- a/cli/ops/native_plugins.rs +++ b/cli/ops/native_plugins.rs @@ -60,7 +60,7 @@ pub fn op_open_native_plugin( let args: OpenNativePluginArgs = serde_json::from_value(args)?; let (filename, filename_) = deno_fs::resolve_from_cwd(&args.filename)?; - state.check_native(&filename_)?; + state.check_plugin(&filename_)?; let lib = open_plugin(filename)?; let plugin_resource = NativePluginResource { @@ -81,7 +81,13 @@ pub fn op_open_native_plugin( }; init_fn(&mut init_context); for op in init_context.ops { - let op_id = registry.register(&op.0, op.1); + // Register each plugin op in the `OpRegistry` with the name + // formated like this `native_plugin_{plugin_rid}_{name}`. + // The inclusion of prefix and rid is designed to avoid any + // op name collision beyond the bound of a single loaded + // native plugin instance. + let op_id = + registry.register(&format!("native_plugin_{}_{}", rid, op.0), op.1); plugin_resource.ops.insert(op.0, op_id); } diff --git a/cli/ops/permissions.rs b/cli/ops/permissions.rs index 0f40b642c266dd..c7ce15d73c1a3a 100644 --- a/cli/ops/permissions.rs +++ b/cli/ops/permissions.rs @@ -53,6 +53,7 @@ pub fn op_revoke_permission( "write" => state.permissions.allow_write.revoke(), "net" => state.permissions.allow_net.revoke(), "env" => state.permissions.allow_env.revoke(), + "plugin" => state.permissions.allow_plugin.revoke(), "hrtime" => state.permissions.allow_hrtime.revoke(), _ => {} }; @@ -86,6 +87,7 @@ pub fn op_request_permission( .permissions .request_net(&args.url.as_ref().map(String::as_str)), "env" => Ok(state.permissions.request_env()), + "plugin" => Ok(state.permissions.request_plugin()), "hrtime" => Ok(state.permissions.request_hrtime()), n => Err(type_error(format!("No such permission name: {}", n))), }?; diff --git a/cli/permissions.rs b/cli/permissions.rs index f1e61e26ee0c14..62bc6efe492a0b 100644 --- a/cli/permissions.rs +++ b/cli/permissions.rs @@ -141,7 +141,7 @@ pub struct DenoPermissions { pub net_whitelist: Arc>, pub allow_env: PermissionAccessor, pub allow_run: PermissionAccessor, - pub allow_native: PermissionAccessor, + pub allow_plugin: PermissionAccessor, pub allow_hrtime: PermissionAccessor, } @@ -158,7 +158,7 @@ impl DenoPermissions { net_whitelist: Arc::new(flags.net_whitelist.iter().cloned().collect()), allow_env: PermissionAccessor::from(flags.allow_env), allow_run: PermissionAccessor::from(flags.allow_run), - allow_native: PermissionAccessor::from(flags.allow_native), + allow_plugin: PermissionAccessor::from(flags.allow_plugin), allow_hrtime: PermissionAccessor::from(flags.allow_hrtime), } } @@ -251,6 +251,13 @@ impl DenoPermissions { ) } + pub fn check_plugin(&self, filename: &str) -> Result<(), ErrBox> { + self.allow_plugin.get_state().check( + &format!("access to open a native plugin: {}", filename), + "run again with the --allow-plugin flag", + ) + } + pub fn request_run(&self) -> PermissionAccessorState { self .allow_run @@ -302,11 +309,10 @@ impl DenoPermissions { .request("Deno requests to access to high precision time.") } - pub fn check_native(&self, filename: &str) -> Result<(), ErrBox> { - self.allow_native.get_state().check( - &format!("access to open a native plugin: {}", filename), - "run again with the --allow-run flag", - ) + pub fn request_plugin(&self) -> PermissionAccessorState { + self + .allow_plugin + .request("Deno requests to open native plugins.") } pub fn get_permission_state( @@ -321,6 +327,7 @@ impl DenoPermissions { "write" => Ok(self.get_state_write(path)), "net" => self.get_state_net_url(url), "env" => Ok(self.allow_env.get_state()), + "plugin" => Ok(self.allow_plugin.get_state()), "hrtime" => Ok(self.allow_hrtime.get_state()), n => Err(type_error(format!("No such permission name: {}", n))), } @@ -703,6 +710,21 @@ mod tests { assert_eq!(perms1.request_env(), PermissionAccessorState::Deny); } + #[test] + fn test_permissions_request_plugin() { + let perms0 = DenoPermissions::from_flags(&DenoFlags { + ..Default::default() + }); + set_prompt_result(true); + assert_eq!(perms0.request_plugin(), PermissionAccessorState::Allow); + + let perms1 = DenoPermissions::from_flags(&DenoFlags { + ..Default::default() + }); + set_prompt_result(false); + assert_eq!(perms1.request_plugin(), PermissionAccessorState::Deny); + } + #[test] fn test_permissions_request_hrtime() { let perms0 = DenoPermissions::from_flags(&DenoFlags { diff --git a/cli/state.rs b/cli/state.rs index c421dcc990e798..a9304fa7930d9a 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -289,8 +289,8 @@ impl ThreadSafeState { } #[inline] - pub fn check_native(&self, filename: &str) -> Result<(), ErrBox> { - self.permissions.check_native(filename) + pub fn check_plugin(&self, filename: &str) -> Result<(), ErrBox> { + self.permissions.check_plugin(filename) } pub fn check_dyn_import( diff --git a/cli/tests/052_native_plugins.ts b/cli/tests/052_native_plugins.ts index 299ad1ea93c5ae..5add1a35bf2691 100644 --- a/cli/tests/052_native_plugins.ts +++ b/cli/tests/052_native_plugins.ts @@ -43,4 +43,6 @@ const response = test_io_async.dispatch( new Uint8Array([116, 101, 115, 116]) ); -console.log(`Native Binding Async Response: ${textDecoder.decode(response)}`); +if (response != null || response != undefined) { + throw new Error("Expected null response!"); +} diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index cc45a5b3b98dd4..4c6d610f9d52fa 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -358,14 +358,14 @@ itest!(_051_wasm_import { #[cfg(not(debug_assertions))] itest!(_052_native_plugins_release { - args: "run --reload --allow-native 052_native_plugins.ts release", + args: "run --reload --allow-plugin 052_native_plugins.ts release", output: "052_native_plugins.ts.out", http_server: false, }); #[cfg(debug_assertions)] itest!(_052_native_plugins_debug { - args: "run --reload --allow-native 052_native_plugins.ts debug", + args: "run --reload --allow-plugin 052_native_plugins.ts debug", output: "052_native_plugins.ts.out", http_server: false, }); diff --git a/cli/tests/test_native_plugin/src/lib.rs b/cli/tests/test_native_plugin/src/lib.rs index d0897b347fe43f..6e49673cdfe0f1 100644 --- a/cli/tests/test_native_plugin/src/lib.rs +++ b/cli/tests/test_native_plugin/src/lib.rs @@ -38,7 +38,12 @@ pub fn op_test_io_async(data: &[u8], zero_copy: Option) -> CoreOp { data_str, buf_str ); } - // TODO(afinch7): add a delayed async of some type in here. + let (tx, rx) = futures::channel::oneshot::channel::>(); + std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_secs(1)); + tx.send(Ok(())).unwrap(); + }); + assert!(rx.await.is_ok()); let result = b"test"; let result_box: Buf = Box::new(*result); Ok(result_box) From 4b660e8c33c205b3cf42bc3aa379a86e2e6b3d47 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Sun, 24 Nov 2019 11:50:00 -0500 Subject: [PATCH 13/24] fix --- cli/permissions.rs | 10 +++++----- cli/state.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cli/permissions.rs b/cli/permissions.rs index 391a9d597c6902..3ef205ed85160a 100644 --- a/cli/permissions.rs +++ b/cli/permissions.rs @@ -267,7 +267,7 @@ impl DenoPermissions { .request("Deno requests to access to high precision time.") } - pub fn request_plugin(&self) -> PermissionAccessorState { + pub fn request_plugin(&mut self) -> PermissionState { self .allow_plugin .request("Deno requests to open native plugins.") @@ -670,17 +670,17 @@ mod tests { #[test] fn test_permissions_request_plugin() { - let perms0 = DenoPermissions::from_flags(&DenoFlags { + let mut perms0 = DenoPermissions::from_flags(&DenoFlags { ..Default::default() }); set_prompt_result(true); - assert_eq!(perms0.request_plugin(), PermissionAccessorState::Allow); + assert_eq!(perms0.request_plugin(), PermissionState::Allow); - let perms1 = DenoPermissions::from_flags(&DenoFlags { + let mut perms1 = DenoPermissions::from_flags(&DenoFlags { ..Default::default() }); set_prompt_result(false); - assert_eq!(perms1.request_plugin(), PermissionAccessorState::Deny); + assert_eq!(perms1.request_plugin(), PermissionState::Deny); } #[test] diff --git a/cli/state.rs b/cli/state.rs index e4baac60dd9584..21c389a2584e04 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -296,7 +296,7 @@ impl ThreadSafeState { #[inline] pub fn check_plugin(&self, filename: &str) -> Result<(), ErrBox> { - self.permissions.check_plugin(filename) + self.permissions.lock().unwrap().check_plugin(filename) } pub fn check_dyn_import( From 2a3851524637edb5a2ded285faa39091f58f1d0f Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Sun, 24 Nov 2019 11:43:38 -0500 Subject: [PATCH 14/24] Move test_native_plugin to root - add failing test --- .github/workflows/ci.yml | 1 - Cargo.toml | 2 +- {cli/tests/test_native_plugin => test_plugin}/Cargo.toml | 2 +- {cli/tests/test_native_plugin => test_plugin}/mod.ts | 0 {cli/tests/test_native_plugin => test_plugin}/src/lib.rs | 5 ----- test_plugin/tests/integration_tests.rs | 5 +++++ 6 files changed, 7 insertions(+), 8 deletions(-) rename {cli/tests/test_native_plugin => test_plugin}/Cargo.toml (89%) rename {cli/tests/test_native_plugin => test_plugin}/mod.ts (100%) rename {cli/tests/test_native_plugin => test_plugin}/src/lib.rs (94%) create mode 100644 test_plugin/tests/integration_tests.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b903ff996ce99d..bbdf02d57dde7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,7 +129,6 @@ jobs: - name: Test debug if: matrix.kind == 'test_debug' run: | - cargo build --locked -p test_native_plugin echo ::set-env name=DENO_BUILD_MODE::debug cargo test --locked --all-targets diff --git a/Cargo.toml b/Cargo.toml index df68c7121370de..b273e4222f2448 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,5 +4,5 @@ members = [ "core", "tools/hyper_hello", "deno_typescript", - "cli/tests/test_native_plugin" + "test_plugin" ] diff --git a/cli/tests/test_native_plugin/Cargo.toml b/test_plugin/Cargo.toml similarity index 89% rename from cli/tests/test_native_plugin/Cargo.toml rename to test_plugin/Cargo.toml index bf3e757dc87c08..3ea9ef15425b66 100644 --- a/cli/tests/test_native_plugin/Cargo.toml +++ b/test_plugin/Cargo.toml @@ -10,5 +10,5 @@ edition = "2018" crate-type = ["cdylib"] [dependencies] -deno = { path = "../../../core" } +deno = { path = "../core" } futures = "0.3" \ No newline at end of file diff --git a/cli/tests/test_native_plugin/mod.ts b/test_plugin/mod.ts similarity index 100% rename from cli/tests/test_native_plugin/mod.ts rename to test_plugin/mod.ts diff --git a/cli/tests/test_native_plugin/src/lib.rs b/test_plugin/src/lib.rs similarity index 94% rename from cli/tests/test_native_plugin/src/lib.rs rename to test_plugin/src/lib.rs index 6e49673cdfe0f1..c78c600431caa3 100644 --- a/cli/tests/test_native_plugin/src/lib.rs +++ b/test_plugin/src/lib.rs @@ -51,8 +51,3 @@ pub fn op_test_io_async(data: &[u8], zero_copy: Option) -> CoreOp { Op::Async(fut.boxed()) } - -#[test] -fn placeholder() { - // This test is just here to make sure the plugin gets built -} diff --git a/test_plugin/tests/integration_tests.rs b/test_plugin/tests/integration_tests.rs new file mode 100644 index 00000000000000..c66459e5046dc7 --- /dev/null +++ b/test_plugin/tests/integration_tests.rs @@ -0,0 +1,5 @@ +#[test] +fn plugin1() { + // Here we should be able to load .so + assert!(false); // Placeholder to check that it gets run. +} From 8901da5eae4eaf11aa755df69ad0deaee471ffc1 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 27 Nov 2019 08:26:35 -0800 Subject: [PATCH 15/24] fix --- cli/flags.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/flags.rs b/cli/flags.rs index 3e8655a51c31bb..36b1b880f4191d 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -472,6 +472,7 @@ fn run_test_args_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) { flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; + flags.allow_plugin = true; flags.allow_hrtime = true; } if matches.is_present("no-fetch") { From e0004c3f9ebbc479f53f40ec9d12ed0076a906c7 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 27 Nov 2019 08:55:45 -0800 Subject: [PATCH 16/24] test_plugin WIP --- Cargo.lock | 4 ++-- cli/tests/052_native_plugins.ts.out | 4 ---- cli/tests/integration_tests.rs | 14 -------------- test_plugin/Cargo.toml | 9 ++++----- test_plugin/tests/integration_tests.rs | 10 +++++++--- .../tests/test.js | 0 6 files changed, 13 insertions(+), 28 deletions(-) delete mode 100644 cli/tests/052_native_plugins.ts.out rename cli/tests/052_native_plugins.ts => test_plugin/tests/test.js (100%) diff --git a/Cargo.lock b/Cargo.lock index 490c48ef7ebc55..9c7424e98397f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1536,8 +1536,8 @@ dependencies = [ ] [[package]] -name = "test_native_plugin" -version = "0.1.0" +name = "test_plugin" +version = "0.0.1" dependencies = [ "deno 0.25.0", "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/cli/tests/052_native_plugins.ts.out b/cli/tests/052_native_plugins.ts.out deleted file mode 100644 index 252b1be26350c7..00000000000000 --- a/cli/tests/052_native_plugins.ts.out +++ /dev/null @@ -1,4 +0,0 @@ -Hello from native bindings. data: test | zero_copy: test -Native Binding Sync Response: test -Hello from native bindings. data: test | zero_copy: test -Native Binding Async Response: test diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 708a11ad4cf763..3cc8039db5db24 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -397,20 +397,6 @@ itest!(_051_wasm_import { http_server: true, }); -#[cfg(not(debug_assertions))] -itest!(_052_native_plugins_release { - args: "run --reload --allow-plugin 052_native_plugins.ts release", - output: "052_native_plugins.ts.out", - http_server: false, -}); - -#[cfg(debug_assertions)] -itest!(_052_native_plugins_debug { - args: "run --reload --allow-plugin 052_native_plugins.ts debug", - output: "052_native_plugins.ts.out", - http_server: false, -}); - itest!(lock_check_ok { args: "run --lock=lock_check_ok.json http://127.0.0.1:4545/cli/tests/003_relative_import.ts", output: "003_relative_import.ts.out", diff --git a/test_plugin/Cargo.toml b/test_plugin/Cargo.toml index 3ea9ef15425b66..3c2a621a5050c2 100644 --- a/test_plugin/Cargo.toml +++ b/test_plugin/Cargo.toml @@ -1,10 +1,9 @@ [package] -name = "test_native_plugin" -version = "0.1.0" -authors = ["afinch7 "] +name = "test_plugin" +version = "0.0.1" +authors = ["the deno authors"] edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +publish = false [lib] crate-type = ["cdylib"] diff --git a/test_plugin/tests/integration_tests.rs b/test_plugin/tests/integration_tests.rs index c66459e5046dc7..261e8bc2aa5e32 100644 --- a/test_plugin/tests/integration_tests.rs +++ b/test_plugin/tests/integration_tests.rs @@ -1,5 +1,9 @@ +use deno::*; + #[test] -fn plugin1() { - // Here we should be able to load .so - assert!(false); // Placeholder to check that it gets run. +fn basic() { + let src = include_str!("test.js"); + + let mut isolate = Isolate::new(StartupData::None, false); + js_check(isolate.execute("test.js", src)); } diff --git a/cli/tests/052_native_plugins.ts b/test_plugin/tests/test.js similarity index 100% rename from cli/tests/052_native_plugins.ts rename to test_plugin/tests/test.js From ae129f7f1a2ea7b99601bc2926d8e8478a686b6f Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 28 Nov 2019 08:09:49 -0800 Subject: [PATCH 17/24] Fix test_plugin --- Cargo.lock | 1 + cli/js/dispatch.ts | 8 +++--- cli/js/lib.deno_runtime.d.ts | 8 +++--- cli/js/native_plugins.ts | 24 +++++++++--------- test_plugin/Cargo.toml | 5 +++- test_plugin/tests/integration_tests.rs | 34 ++++++++++++++++++++++---- test_plugin/tests/test.js | 6 ++--- 7 files changed, 57 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c7424e98397f8..fddd5664230b67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1540,6 +1540,7 @@ name = "test_plugin" version = "0.0.1" dependencies = [ "deno 0.25.0", + "deno_cli 0.25.0", "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/cli/js/dispatch.ts b/cli/js/dispatch.ts index d6effe3f94b6cc..93b0948b884ef5 100644 --- a/cli/js/dispatch.ts +++ b/cli/js/dispatch.ts @@ -68,15 +68,15 @@ export let OP_CWD: number; export let OP_FETCH_ASSET: number; export let OP_DIAL_TLS: number; export let OP_HOSTNAME: number; -export let OP_OPEN_NATIVE_PLUGIN: number; +export let OP_OPEN_PLUGIN: number; -const NATIVE_PLUGIN_ASYNC_HANDLER_MAP: Map = new Map(); +const PLUGIN_ASYNC_HANDLER_MAP: Map = new Map(); export function setPluginAsyncHandler( opId: number, handler: AsyncHandler ): void { - NATIVE_PLUGIN_ASYNC_HANDLER_MAP.set(opId, handler); + PLUGIN_ASYNC_HANDLER_MAP.set(opId, handler); } export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void { @@ -122,7 +122,7 @@ export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void { json.asyncMsgFromRust(opId, ui8); break; default: - const handler = NATIVE_PLUGIN_ASYNC_HANDLER_MAP.get(opId); + const handler = PLUGIN_ASYNC_HANDLER_MAP.get(opId); if (handler) { handler(ui8); } else { diff --git a/cli/js/lib.deno_runtime.d.ts b/cli/js/lib.deno_runtime.d.ts index 65971256ae8839..623be438b96763 100644 --- a/cli/js/lib.deno_runtime.d.ts +++ b/cli/js/lib.deno_runtime.d.ts @@ -993,7 +993,7 @@ declare namespace Deno { (msg: Uint8Array): void; } - export interface NativePluginOp { + export interface PluginOp { dispatch( control: Uint8Array, zeroCopy?: ArrayBufferView | null @@ -1001,9 +1001,9 @@ declare namespace Deno { setAsyncHandler(handler: AsyncHandler): void; } - export interface NativePlugin { + export interface Plugin { ops: { - [name: string]: NativePluginOp; + [name: string]: PluginOp; }; } @@ -1015,7 +1015,7 @@ declare namespace Deno { * const response = some_op.dispatch(new Uint8Array([1,2,3,4])); * console.log(`Response from native plugin ${response}`); */ - export function openPlugin(filename: string): NativePlugin; + export function openPlugin(filename: string): Plugin; // @url js/net.d.ts diff --git a/cli/js/native_plugins.ts b/cli/js/native_plugins.ts index 6e333c65948d4b..324ae3408224d5 100644 --- a/cli/js/native_plugins.ts +++ b/cli/js/native_plugins.ts @@ -1,12 +1,12 @@ import { sendSync } from "./dispatch_json.ts"; -import { OP_OPEN_NATIVE_PLUGIN, setPluginAsyncHandler } from "./dispatch.ts"; +import { OP_OPEN_PLUGIN, setPluginAsyncHandler } from "./dispatch.ts"; import { core } from "./core.ts"; export interface AsyncHandler { (msg: Uint8Array): void; } -interface NativePluginOp { +interface PluginOp { dispatch( control: Uint8Array, zeroCopy?: ArrayBufferView | null @@ -14,7 +14,7 @@ interface NativePluginOp { setAsyncHandler(handler: AsyncHandler): void; } -class NativePluginOpImpl implements NativePluginOp { +class PluginOpImpl implements PluginOp { constructor(private readonly opId: number) {} dispatch( @@ -31,22 +31,22 @@ class NativePluginOpImpl implements NativePluginOp { // TODO(afinch7): add close method. -interface NativePlugin { +interface Plugin { ops: { - [name: string]: NativePluginOp; + [name: string]: PluginOp; }; } -class NativePluginImpl implements NativePlugin { - private _ops: { [name: string]: NativePluginOp } = {}; +class PluginImpl implements Plugin { + private _ops: { [name: string]: PluginOp } = {}; constructor(private readonly rid: number, ops: { [name: string]: number }) { for (const op in ops) { - this._ops[op] = new NativePluginOpImpl(ops[op]); + this._ops[op] = new PluginOpImpl(ops[op]); } } - get ops(): { [name: string]: NativePluginOp } { + get ops(): { [name: string]: PluginOp } { return Object.assign({}, this._ops); } } @@ -58,9 +58,9 @@ interface OpenPluginResponse { }; } -export function openPlugin(filename: string): NativePlugin { - const response: OpenPluginResponse = sendSync(OP_OPEN_NATIVE_PLUGIN, { +export function openPlugin(filename: string): Plugin { + const response: OpenPluginResponse = sendSync(OP_OPEN_PLUGIN, { filename }); - return new NativePluginImpl(response.rid, response.ops); + return new PluginImpl(response.rid, response.ops); } diff --git a/test_plugin/Cargo.toml b/test_plugin/Cargo.toml index 3c2a621a5050c2..00e7eb1c9c3820 100644 --- a/test_plugin/Cargo.toml +++ b/test_plugin/Cargo.toml @@ -9,5 +9,8 @@ publish = false crate-type = ["cdylib"] [dependencies] +futures = "0.3" deno = { path = "../core" } -futures = "0.3" \ No newline at end of file + +[dev-dependencies] +deno_cli = { path = "../cli" } \ No newline at end of file diff --git a/test_plugin/tests/integration_tests.rs b/test_plugin/tests/integration_tests.rs index 261e8bc2aa5e32..07f20830e9c6a9 100644 --- a/test_plugin/tests/integration_tests.rs +++ b/test_plugin/tests/integration_tests.rs @@ -1,9 +1,33 @@ -use deno::*; +use deno_cli::test_util::*; +use std::process::Command; + +fn deno_cmd() -> Command { + Command::new(deno_exe_path()) +} + +#[cfg(debug_assertions)] +const BUILD_VARIANT: &str = "debug"; + +#[cfg(not(debug_assertions))] +const BUILD_VARIANT: &str = "release"; #[test] fn basic() { - let src = include_str!("test.js"); - - let mut isolate = Isolate::new(StartupData::None, false); - js_check(isolate.execute("test.js", src)); + let output = deno_cmd() + .arg("--allow-plugin") + .arg("tests/test.js") + .arg(BUILD_VARIANT) + .output() + .unwrap(); + let stdout = std::str::from_utf8(&output.stdout).unwrap(); + let stderr = std::str::from_utf8(&output.stderr).unwrap(); + //println!("stdout {:?}", stdout); + //println!("stderr {:?}", stderr); + if !output.status.success() { + println!("stderr {}", stderr); + } + assert!(output.status.success()); + let expected = "Hello from native bindings. data: test | zero_copy: test\nNative Binding Sync Response: test\nHello from native bindings. data: test | zero_copy: test\nNative Binding Async Response: test\n"; + assert_eq!(stdout, expected); + assert_eq!(stderr, ""); } diff --git a/test_plugin/tests/test.js b/test_plugin/tests/test.js index 5add1a35bf2691..51c4258a3670fa 100644 --- a/test_plugin/tests/test.js +++ b/test_plugin/tests/test.js @@ -1,4 +1,4 @@ -const filenameBase = "test_native_plugin"; +const filenameBase = "test_plugin"; let filenameSuffix = ".so"; let filenamePrefix = "lib"; @@ -11,9 +11,9 @@ if (Deno.build.os === "mac") { filenameSuffix = ".dylib"; } -const filename = `${filenamePrefix}${filenameBase}${filenameSuffix}`; +const filename = `../target/${Deno.args[1]}/${filenamePrefix}${filenameBase}${filenameSuffix}`; -const plugin = Deno.openPlugin(`./../../target/${Deno.args[1]}/${filename}`); +const plugin = Deno.openPlugin(filename); // eslint-disable-next-line @typescript-eslint/camelcase const test_io_sync = plugin.ops.test_io_sync; From e0dcb0bc4a7f6c953316f28627662ff1a301e98e Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 28 Nov 2019 08:38:06 -0800 Subject: [PATCH 18/24] clean up --- test_plugin/src/lib.rs | 8 ++++---- test_plugin/tests/test.js | 26 ++++++++------------------ 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/test_plugin/src/lib.rs b/test_plugin/src/lib.rs index c78c600431caa3..c4173cbe21c18b 100644 --- a/test_plugin/src/lib.rs +++ b/test_plugin/src/lib.rs @@ -9,12 +9,12 @@ use deno::{Buf, PinnedBuf}; use futures::future::FutureExt; fn init(context: &mut dyn PluginInitContext) { - context.register_op("test_io_sync", Box::new(op_test_io_sync)); - context.register_op("test_io_async", Box::new(op_test_io_async)); + context.register_op("testSync", Box::new(op_test_sync)); + context.register_op("testAsync", Box::new(op_test_async)); } init_fn!(init); -pub fn op_test_io_sync(data: &[u8], zero_copy: Option) -> CoreOp { +pub fn op_test_sync(data: &[u8], zero_copy: Option) -> CoreOp { if let Some(buf) = zero_copy { let data_str = std::str::from_utf8(&data[..]).unwrap(); let buf_str = std::str::from_utf8(&buf[..]).unwrap(); @@ -28,7 +28,7 @@ pub fn op_test_io_sync(data: &[u8], zero_copy: Option) -> CoreOp { Op::Sync(result_box) } -pub fn op_test_io_async(data: &[u8], zero_copy: Option) -> CoreOp { +pub fn op_test_async(data: &[u8], zero_copy: Option) -> CoreOp { let data_str = std::str::from_utf8(&data[..]).unwrap().to_string(); let fut = async move { if let Some(buf) = zero_copy { diff --git a/test_plugin/tests/test.js b/test_plugin/tests/test.js index 51c4258a3670fa..537ed802f8890a 100644 --- a/test_plugin/tests/test.js +++ b/test_plugin/tests/test.js @@ -15,30 +15,20 @@ const filename = `../target/${Deno.args[1]}/${filenamePrefix}${filenameBase}${fi const plugin = Deno.openPlugin(filename); -// eslint-disable-next-line @typescript-eslint/camelcase -const test_io_sync = plugin.ops.test_io_sync; -// eslint-disable-next-line @typescript-eslint/camelcase -const test_io_async = plugin.ops.test_io_async; +const { testSync, testAsync } = plugin.ops; const textDecoder = new TextDecoder(); -{ - // eslint-disable-next-line @typescript-eslint/camelcase - const response = test_io_sync.dispatch( - new Uint8Array([116, 101, 115, 116]), - new Uint8Array([116, 101, 115, 116]) - ); - - console.log(`Native Binding Sync Response: ${textDecoder.decode(response)}`); -} +const response = testSync.dispatch( + new Uint8Array([116, 101, 115, 116]), + new Uint8Array([116, 101, 115, 116]) +); +console.log(`Native Binding Sync Response: ${textDecoder.decode(response)}`); -// eslint-disable-next-line @typescript-eslint/camelcase -test_io_async.setAsyncHandler(response => { +testAsync.setAsyncHandler(response => { console.log(`Native Binding Async Response: ${textDecoder.decode(response)}`); }); - -// eslint-disable-next-line @typescript-eslint/camelcase -const response = test_io_async.dispatch( +const response = testAsync.dispatch( new Uint8Array([116, 101, 115, 116]), new Uint8Array([116, 101, 115, 116]) ); From 26bd330a18b2b87370bc3ebe85d4614946f3316b Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 28 Nov 2019 09:00:15 -0800 Subject: [PATCH 19/24] clean up --- test_plugin/Cargo.toml | 2 -- test_plugin/tests/integration_tests.rs | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test_plugin/Cargo.toml b/test_plugin/Cargo.toml index 00e7eb1c9c3820..d6c4eb1de3e75c 100644 --- a/test_plugin/Cargo.toml +++ b/test_plugin/Cargo.toml @@ -11,6 +11,4 @@ crate-type = ["cdylib"] [dependencies] futures = "0.3" deno = { path = "../core" } - -[dev-dependencies] deno_cli = { path = "../cli" } \ No newline at end of file diff --git a/test_plugin/tests/integration_tests.rs b/test_plugin/tests/integration_tests.rs index 07f20830e9c6a9..8bacf80de03fa3 100644 --- a/test_plugin/tests/integration_tests.rs +++ b/test_plugin/tests/integration_tests.rs @@ -2,6 +2,7 @@ use deno_cli::test_util::*; use std::process::Command; fn deno_cmd() -> Command { + assert!(deno_exe_path().exists()); Command::new(deno_exe_path()) } @@ -21,9 +22,8 @@ fn basic() { .unwrap(); let stdout = std::str::from_utf8(&output.stdout).unwrap(); let stderr = std::str::from_utf8(&output.stderr).unwrap(); - //println!("stdout {:?}", stdout); - //println!("stderr {:?}", stderr); if !output.status.success() { + println!("stdout {}", stdout); println!("stderr {}", stderr); } assert!(output.status.success()); From 33b749f4af288bd514ae015f537ff06f33b7088d Mon Sep 17 00:00:00 2001 From: afinch7 Date: Tue, 3 Dec 2019 09:25:48 -0500 Subject: [PATCH 20/24] fix scoping in plugin test --- test_plugin/tests/test.js | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/test_plugin/tests/test.js b/test_plugin/tests/test.js index 537ed802f8890a..56703caf392fab 100644 --- a/test_plugin/tests/test.js +++ b/test_plugin/tests/test.js @@ -19,20 +19,29 @@ const { testSync, testAsync } = plugin.ops; const textDecoder = new TextDecoder(); -const response = testSync.dispatch( - new Uint8Array([116, 101, 115, 116]), - new Uint8Array([116, 101, 115, 116]) -); -console.log(`Native Binding Sync Response: ${textDecoder.decode(response)}`); +function testSync() { + const response = testSync.dispatch( + new Uint8Array([116, 101, 115, 116]), + new Uint8Array([116, 101, 115, 116]) + ); + + console.log(`Native Binding Sync Response: ${textDecoder.decode(response)}`); +} testAsync.setAsyncHandler(response => { console.log(`Native Binding Async Response: ${textDecoder.decode(response)}`); }); -const response = testAsync.dispatch( - new Uint8Array([116, 101, 115, 116]), - new Uint8Array([116, 101, 115, 116]) -); -if (response != null || response != undefined) { - throw new Error("Expected null response!"); +function testAsync() { + const response = testAsync.dispatch( + new Uint8Array([116, 101, 115, 116]), + new Uint8Array([116, 101, 115, 116]) + ); + + if (response != null || response != undefined) { + throw new Error("Expected null response!"); + } } + +testSync(); +testAsync(); From 10ad75c46c260dc18fe710a2f8ba4c03861ed7a3 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Tue, 3 Dec 2019 09:50:36 -0500 Subject: [PATCH 21/24] fix another issue in plugin test --- test_plugin/tests/test.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_plugin/tests/test.js b/test_plugin/tests/test.js index 56703caf392fab..61abd8c22fb58a 100644 --- a/test_plugin/tests/test.js +++ b/test_plugin/tests/test.js @@ -19,7 +19,7 @@ const { testSync, testAsync } = plugin.ops; const textDecoder = new TextDecoder(); -function testSync() { +function runTestSync() { const response = testSync.dispatch( new Uint8Array([116, 101, 115, 116]), new Uint8Array([116, 101, 115, 116]) @@ -32,7 +32,7 @@ testAsync.setAsyncHandler(response => { console.log(`Native Binding Async Response: ${textDecoder.decode(response)}`); }); -function testAsync() { +function runTestAsync() { const response = testAsync.dispatch( new Uint8Array([116, 101, 115, 116]), new Uint8Array([116, 101, 115, 116]) @@ -43,5 +43,5 @@ function testAsync() { } } -testSync(); -testAsync(); +runTestSync(); +runTestAsync(); From 230e9264e26892062faa3f159c4b9b9cbb8d9ade Mon Sep 17 00:00:00 2001 From: afinch7 Date: Tue, 3 Dec 2019 10:55:24 -0500 Subject: [PATCH 22/24] fixes * Removed "native" wording from plugins * Build `test_plugin` before running debug tests in CI --- .github/workflows/ci.yml | 1 + cli/js/deno.ts | 2 +- cli/js/dispatch.ts | 2 +- cli/js/lib.deno_runtime.d.ts | 6 ++--- cli/js/{native_plugins.ts => plugins.ts} | 0 cli/ops/mod.rs | 2 +- cli/ops/{native_plugins.rs => plugins.rs} | 31 +++++++++++------------ cli/permissions.rs | 10 +++----- cli/worker.rs | 2 +- core/plugins.rs | 2 +- test_plugin/mod.ts | 0 test_plugin/src/lib.rs | 4 +-- test_plugin/tests/integration_tests.rs | 2 +- test_plugin/tests/test.js | 4 +-- 14 files changed, 33 insertions(+), 35 deletions(-) rename cli/js/{native_plugins.ts => plugins.ts} (100%) rename cli/ops/{native_plugins.rs => plugins.rs} (71%) delete mode 100644 test_plugin/mod.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbdf02d57dde7c..fb7f3dad99520e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,6 +130,7 @@ jobs: if: matrix.kind == 'test_debug' run: | echo ::set-env name=DENO_BUILD_MODE::debug + cargo build --locked -p test_plugin cargo test --locked --all-targets - name: Run Benchmarks diff --git a/cli/js/deno.ts b/cli/js/deno.ts index 7fd89db53f8bc4..27a7bb3bd0dcc8 100644 --- a/cli/js/deno.ts +++ b/cli/js/deno.ts @@ -76,7 +76,7 @@ export { } from "./permissions.ts"; export { truncateSync, truncate } from "./truncate.ts"; export { FileInfo } from "./file_info.ts"; -export { openPlugin } from "./native_plugins.ts"; +export { openPlugin } from "./plugins.ts"; export { connect, dial, listen, Listener, Conn } from "./net.ts"; export { dialTLS, listenTLS } from "./tls.ts"; export { metrics, Metrics } from "./metrics.ts"; diff --git a/cli/js/dispatch.ts b/cli/js/dispatch.ts index 93b0948b884ef5..ed6f5705200076 100644 --- a/cli/js/dispatch.ts +++ b/cli/js/dispatch.ts @@ -1,7 +1,7 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. import * as minimal from "./dispatch_minimal.ts"; import * as json from "./dispatch_json.ts"; -import { AsyncHandler } from "./native_plugins.ts"; +import { AsyncHandler } from "./plugins.ts"; // These consts are shared with Rust. Update with care. export let OP_READ: number; diff --git a/cli/js/lib.deno_runtime.d.ts b/cli/js/lib.deno_runtime.d.ts index 623be438b96763..0d346386962aa3 100644 --- a/cli/js/lib.deno_runtime.d.ts +++ b/cli/js/lib.deno_runtime.d.ts @@ -987,7 +987,7 @@ declare namespace Deno { */ export function truncate(name: string, len?: number): Promise; - // @url js/native_plugins.d.ts + // @url js/plugins.d.ts export interface AsyncHandler { (msg: Uint8Array): void; @@ -1007,13 +1007,13 @@ declare namespace Deno { }; } - /** Open and initalize a native plugin. + /** Open and initalize a plugin. * Requires the `--allow-plugin` flag. * * const plugin = Deno.openPlugin("./path/to/some/plugin.so"); * const some_op = plugin.ops.some_op; * const response = some_op.dispatch(new Uint8Array([1,2,3,4])); - * console.log(`Response from native plugin ${response}`); + * console.log(`Response from plugin ${response}`); */ export function openPlugin(filename: string): Plugin; diff --git a/cli/js/native_plugins.ts b/cli/js/plugins.ts similarity index 100% rename from cli/js/native_plugins.ts rename to cli/js/plugins.ts diff --git a/cli/ops/mod.rs b/cli/ops/mod.rs index d039d5e132680c..28bf7e21762b85 100644 --- a/cli/ops/mod.rs +++ b/cli/ops/mod.rs @@ -13,10 +13,10 @@ pub mod fetch; pub mod files; pub mod fs; pub mod io; -pub mod native_plugins; pub mod net; pub mod os; pub mod permissions; +pub mod plugins; pub mod process; pub mod random; pub mod repl; diff --git a/cli/ops/native_plugins.rs b/cli/ops/plugins.rs similarity index 71% rename from cli/ops/native_plugins.rs rename to cli/ops/plugins.rs index 68ae08d511bf2a..2673e3d6aa9c34 100644 --- a/cli/ops/native_plugins.rs +++ b/cli/ops/plugins.rs @@ -11,25 +11,25 @@ use std::sync::Arc; pub fn init(i: &mut Isolate, s: &ThreadSafeState, r: Arc) { let r_ = r.clone(); i.register_op( - "open_native_plugin", + "open_plugin", s.core_op(json_op(s.stateful_op(move |state, args, zero_copy| { - op_open_native_plugin(&r_, state, args, zero_copy) + op_open_plugin(&r_, state, args, zero_copy) }))), ); } fn open_plugin>(lib_path: P) -> Result { - debug!("Loading Native Plugin: {:#?}", lib_path.as_ref()); + debug!("Loading Plugin: {:#?}", lib_path.as_ref()); Library::open(lib_path).map_err(ErrBox::from) } -struct NativePluginResource { +struct PluginResource { lib: Library, ops: HashMap, } -impl Resource for NativePluginResource {} +impl Resource for PluginResource {} struct InitContext { ops: HashMap>, @@ -47,34 +47,34 @@ impl PluginInitContext for InitContext { #[derive(Deserialize)] #[serde(rename_all = "camelCase")] -struct OpenNativePluginArgs { +struct OpenPluginArgs { filename: String, } -pub fn op_open_native_plugin( +pub fn op_open_plugin( registry: &Arc, state: &ThreadSafeState, args: Value, _zero_copy: Option, ) -> Result { - let args: OpenNativePluginArgs = serde_json::from_value(args)?; + let args: OpenPluginArgs = serde_json::from_value(args)?; let (filename, filename_) = deno_fs::resolve_from_cwd(&args.filename)?; state.check_plugin(&filename_)?; let lib = open_plugin(filename)?; - let plugin_resource = NativePluginResource { + let plugin_resource = PluginResource { lib, ops: HashMap::new(), }; let mut table = state.lock_resource_table(); - let rid = table.add("native_plugin", Box::new(plugin_resource)); - let plugin_resource = table.get_mut::(rid).unwrap(); + let rid = table.add("plugin", Box::new(plugin_resource)); + let plugin_resource = table.get_mut::(rid).unwrap(); let init_fn = *unsafe { plugin_resource .lib - .symbol::("native_plugin_init") + .symbol::("deno_plugin_init") }?; let mut init_context = InitContext { ops: HashMap::new(), @@ -82,12 +82,11 @@ pub fn op_open_native_plugin( init_fn(&mut init_context); for op in init_context.ops { // Register each plugin op in the `OpRegistry` with the name - // formated like this `native_plugin_{plugin_rid}_{name}`. + // formated like this `plugin_{plugin_rid}_{name}`. // The inclusion of prefix and rid is designed to avoid any // op name collision beyond the bound of a single loaded - // native plugin instance. - let op_id = - registry.register(&format!("native_plugin_{}_{}", rid, op.0), op.1); + // plugin instance. + let op_id = registry.register(&format!("plugin_{}_{}", rid, op.0), op.1); plugin_resource.ops.insert(op.0, op_id); } diff --git a/cli/permissions.rs b/cli/permissions.rs index 3ef205ed85160a..72a1a928acc09d 100644 --- a/cli/permissions.rs +++ b/cli/permissions.rs @@ -211,7 +211,7 @@ impl DenoPermissions { pub fn check_plugin(&self, filename: &str) -> Result<(), ErrBox> { self.allow_plugin.check( - &format!("access to open a native plugin: {}", filename), + &format!("access to open a plugin: {}", filename), "run again with the --allow-plugin flag", ) } @@ -226,7 +226,7 @@ impl DenoPermissions { if check_path_white_list(path, &self.read_whitelist) { return PermissionState::Allow; }; - self.allow_write.request(&match path { + self.allow_read.request(&match path { None => "Deno requests read access.".to_string(), Some(path) => format!("Deno requests read access to \"{}\".", path), }) @@ -247,7 +247,7 @@ impl DenoPermissions { url: &Option<&str>, ) -> Result { if self.get_state_net_url(url)? == PermissionState::Ask { - return Ok(self.allow_run.request(&match url { + return Ok(self.allow_net.request(&match url { None => "Deno requests network access.".to_string(), Some(url) => format!("Deno requests network access to \"{}\".", url), })); @@ -268,9 +268,7 @@ impl DenoPermissions { } pub fn request_plugin(&mut self) -> PermissionState { - self - .allow_plugin - .request("Deno requests to open native plugins.") + self.allow_plugin.request("Deno requests to open plugins.") } pub fn get_permission_state( diff --git a/cli/worker.rs b/cli/worker.rs index 55a3fb666e2c2d..ed778423088924 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -58,7 +58,7 @@ impl Worker { ops::files::init(&mut i, &state); ops::fs::init(&mut i, &state); ops::io::init(&mut i, &state); - ops::native_plugins::init(&mut i, &state, op_registry); + ops::plugins::init(&mut i, &state, op_registry); ops::net::init(&mut i, &state); ops::tls::init(&mut i, &state); ops::os::init(&mut i, &state); diff --git a/core/plugins.rs b/core/plugins.rs index 92b1036f9f37dc..50271d53a4869b 100644 --- a/core/plugins.rs +++ b/core/plugins.rs @@ -15,7 +15,7 @@ pub trait PluginInitContext { macro_rules! init_fn { ($fn:path) => { #[no_mangle] - pub fn native_plugin_init(context: &mut dyn PluginInitContext) { + pub fn deno_plugin_init(context: &mut dyn PluginInitContext) { $fn(context) } }; diff --git a/test_plugin/mod.ts b/test_plugin/mod.ts deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/test_plugin/src/lib.rs b/test_plugin/src/lib.rs index c4173cbe21c18b..30df114b94563e 100644 --- a/test_plugin/src/lib.rs +++ b/test_plugin/src/lib.rs @@ -19,7 +19,7 @@ pub fn op_test_sync(data: &[u8], zero_copy: Option) -> CoreOp { let data_str = std::str::from_utf8(&data[..]).unwrap(); let buf_str = std::str::from_utf8(&buf[..]).unwrap(); println!( - "Hello from native bindings. data: {} | zero_copy: {}", + "Hello from plugin. data: {} | zero_copy: {}", data_str, buf_str ); } @@ -34,7 +34,7 @@ pub fn op_test_async(data: &[u8], zero_copy: Option) -> CoreOp { if let Some(buf) = zero_copy { let buf_str = std::str::from_utf8(&buf[..]).unwrap(); println!( - "Hello from native bindings. data: {} | zero_copy: {}", + "Hello from plugin. data: {} | zero_copy: {}", data_str, buf_str ); } diff --git a/test_plugin/tests/integration_tests.rs b/test_plugin/tests/integration_tests.rs index 8bacf80de03fa3..ee74f8062f0cf7 100644 --- a/test_plugin/tests/integration_tests.rs +++ b/test_plugin/tests/integration_tests.rs @@ -27,7 +27,7 @@ fn basic() { println!("stderr {}", stderr); } assert!(output.status.success()); - let expected = "Hello from native bindings. data: test | zero_copy: test\nNative Binding Sync Response: test\nHello from native bindings. data: test | zero_copy: test\nNative Binding Async Response: test\n"; + let expected = "Hello from plugin. data: test | zero_copy: test\nPlugin Sync Response: test\nHello from plugin. data: test | zero_copy: test\nPlugin Async Response: test\n"; assert_eq!(stdout, expected); assert_eq!(stderr, ""); } diff --git a/test_plugin/tests/test.js b/test_plugin/tests/test.js index 61abd8c22fb58a..5a127d32863ea5 100644 --- a/test_plugin/tests/test.js +++ b/test_plugin/tests/test.js @@ -25,11 +25,11 @@ function runTestSync() { new Uint8Array([116, 101, 115, 116]) ); - console.log(`Native Binding Sync Response: ${textDecoder.decode(response)}`); + console.log(`Plugin Sync Response: ${textDecoder.decode(response)}`); } testAsync.setAsyncHandler(response => { - console.log(`Native Binding Async Response: ${textDecoder.decode(response)}`); + console.log(`Plugin Async Response: ${textDecoder.decode(response)}`); }); function runTestAsync() { From b96ec38ae721083f70921becb8bb693d42c908e9 Mon Sep 17 00:00:00 2001 From: afinch7 Date: Tue, 3 Dec 2019 11:28:39 -0500 Subject: [PATCH 23/24] build test plugin during test --- .github/workflows/ci.yml | 1 - test_plugin/tests/integration_tests.rs | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb7f3dad99520e..bbdf02d57dde7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,7 +130,6 @@ jobs: if: matrix.kind == 'test_debug' run: | echo ::set-env name=DENO_BUILD_MODE::debug - cargo build --locked -p test_plugin cargo test --locked --all-targets - name: Run Benchmarks diff --git a/test_plugin/tests/integration_tests.rs b/test_plugin/tests/integration_tests.rs index ee74f8062f0cf7..86a631694cecc9 100644 --- a/test_plugin/tests/integration_tests.rs +++ b/test_plugin/tests/integration_tests.rs @@ -14,6 +14,13 @@ const BUILD_VARIANT: &str = "release"; #[test] fn basic() { + let mut build_plugin_base = Command::new("cargo"); + let mut build_plugin = + build_plugin_base.arg("build").arg("-p").arg("test_plugin"); + if BUILD_VARIANT == "release" { + build_plugin = build_plugin.arg("--release"); + } + let _build_plugin_output = build_plugin.output().unwrap(); let output = deno_cmd() .arg("--allow-plugin") .arg("tests/test.js") From ad9f9e21532da6d65853a940c07ef1e1d3e1de8c Mon Sep 17 00:00:00 2001 From: afinch7 Date: Tue, 3 Dec 2019 11:36:39 -0500 Subject: [PATCH 24/24] fix windows issue --- test_plugin/tests/integration_tests.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test_plugin/tests/integration_tests.rs b/test_plugin/tests/integration_tests.rs index 86a631694cecc9..a6790d013b9a81 100644 --- a/test_plugin/tests/integration_tests.rs +++ b/test_plugin/tests/integration_tests.rs @@ -34,7 +34,11 @@ fn basic() { println!("stderr {}", stderr); } assert!(output.status.success()); - let expected = "Hello from plugin. data: test | zero_copy: test\nPlugin Sync Response: test\nHello from plugin. data: test | zero_copy: test\nPlugin Async Response: test\n"; + let expected = if cfg!(target_os = "windows") { + "Hello from plugin. data: test | zero_copy: test\nPlugin Sync Response: test\r\nHello from plugin. data: test | zero_copy: test\nPlugin Async Response: test\r\n" + } else { + "Hello from plugin. data: test | zero_copy: test\nPlugin Sync Response: test\nHello from plugin. data: test | zero_copy: test\nPlugin Async Response: test\n" + }; assert_eq!(stdout, expected); assert_eq!(stderr, ""); }