From 232982cdbb995ad4da8102cb04f29fc9fc29b37d Mon Sep 17 00:00:00 2001 From: Matthew Paras <34500476+mattwparas@users.noreply.github.com> Date: Wed, 17 Jan 2024 20:39:48 -0800 Subject: [PATCH] Make steel-core compile for wasm32-unknown-unknown (#141) --- crates/steel-core/Cargo.toml | 5 ++- crates/steel-core/src/compiler/compiler.rs | 7 +++ crates/steel-core/src/compiler/modules.rs | 44 ++++++++++++++----- .../src/compiler/passes/analysis.rs | 9 ++++ .../steel-core/src/compiler/passes/begin.rs | 7 ++- crates/steel-core/src/compiler/program.rs | 41 +++++++++++++++-- crates/steel-core/src/parser/kernel.rs | 2 + crates/steel-core/src/primitives/process.rs | 4 ++ crates/steel-core/src/primitives/time.rs | 3 +- crates/steel-core/src/steel_vm/engine.rs | 24 ++++++++-- crates/steel-core/src/steel_vm/vm/threads.rs | 4 ++ crates/steel-core/src/values/closed.rs | 4 ++ 12 files changed, 131 insertions(+), 23 deletions(-) diff --git a/crates/steel-core/Cargo.toml b/crates/steel-core/Cargo.toml index 57ecb843b..3077b6899 100644 --- a/crates/steel-core/Cargo.toml +++ b/crates/steel-core/Cargo.toml @@ -42,7 +42,6 @@ weak-table = "0.3.2" rand = "0.8.5" # TODO: Consider only depending on the sub crate num = "0.4.0" -which = "4.4.0" radix_fmt = "1.0.0" # For structs @@ -67,6 +66,9 @@ anyhow = { version = "1", optional = true } stacker = { version = "0.1.15", optional = true } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +which = { version = "4.4.0" } + [dev-dependencies] proptest = "1.1.0" criterion = "0.5.1" @@ -76,6 +78,7 @@ env_logger = "0.10.0" steel-gen = { path = "../steel-gen", version = "0.2.0" } [features] +# TODO: Deprecate the modules feature flag, it no longer does anything default = ["modules"] modules = [] jit = ["dep:cranelift", "dep:cranelift-module", "dep:cranelift-jit"] diff --git a/crates/steel-core/src/compiler/compiler.rs b/crates/steel-core/src/compiler/compiler.rs index 45fa46783..956d9841d 100644 --- a/crates/steel-core/src/compiler/compiler.rs +++ b/crates/steel-core/src/compiler/compiler.rs @@ -54,6 +54,7 @@ use super::{ use im_rc::HashMap as ImmutableHashMap; +#[cfg(feature = "profiling")] use std::time::Instant; // use itertools::Itertools; @@ -782,13 +783,16 @@ impl Compiler { path: Option, sources: &mut Sources, ) -> Result> { + #[cfg(feature = "profiling")] let now = Instant::now(); let mut expanded_statements = self.expand_expressions(exprs, path, sources, builtin_modules.clone())?; + #[cfg(feature = "profiling")] log::debug!(target: "pipeline_time", "Phase 1 module expansion time: {:?}", now.elapsed()); + #[cfg(feature = "profiling")] let now = Instant::now(); log::debug!(target: "expansion-phase", "Expanding macros -> phase 1"); @@ -848,6 +852,7 @@ impl Compiler { // TODO: Check that defines are in legal positions, post expansion. + #[cfg(feature = "profiling")] log::debug!(target: "pipeline_time", "Top level macro expansion time: {:?}", now.elapsed()); log::debug!(target: "expansion-phase", "Beginning constant folding"); @@ -855,6 +860,7 @@ impl Compiler { let mut expanded_statements = self.apply_const_evaluation(constants.clone(), expanded_statements, false)?; + #[cfg(feature = "profiling")] let now = Instant::now(); // RenameShadowedVariables::rename_shadowed_vars(&mut expanded_statements); @@ -955,6 +961,7 @@ impl Compiler { semantic.replace_anonymous_function_calls_with_plain_lets(); + #[cfg(feature = "profiling")] log::info!(target: "pipeline_time", "CAT time: {:?}", now.elapsed()); self.analysis = semantic.into_analysis(); diff --git a/crates/steel-core/src/compiler/modules.rs b/crates/steel-core/src/compiler/modules.rs index 3fd260005..4ef7c1dd0 100644 --- a/crates/steel-core/src/compiler/modules.rs +++ b/crates/steel-core/src/compiler/modules.rs @@ -51,10 +51,12 @@ use super::{ macro_rules! time { ($label:expr, $e:expr) => {{ + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); let e = $e; + #[cfg(feature = "profiling")] log::debug!(target: "pipeline_time", "{}: {:?}", $label, now.elapsed()); e @@ -114,8 +116,12 @@ create_prelude!( // for_syntax "#%private/steel/match" ); +#[cfg(not(target_arch = "wasm32"))] pub static STEEL_HOME: Lazy> = Lazy::new(|| std::env::var("STEEL_HOME").ok()); +#[cfg(target_arch = "wasm32")] +pub static STEEL_HOME: Lazy> = Lazy::new(|| None); + /// Manages the modules /// keeps some visited state on the manager for traversal /// Also keeps track of the metadata for each file in order to determine @@ -944,18 +950,18 @@ impl ModuleManager { (module, in_scope_macros, name_mangler) } - #[cfg(not(feature = "modules"))] - pub(crate) fn expand_expressions( - &mut self, - global_macro_map: &mut HashMap, - mut exprs: Vec, - ) -> Result> { - extract_macro_defs(&mut exprs, global_macro_map)?; - exprs - .into_iter() - .map(|x| expand(x, global_macro_map)) - .collect() - } + // #[cfg(not(feature = "modules"))] + // pub(crate) fn expand_expressions( + // &mut self, + // global_macro_map: &mut HashMap, + // mut exprs: Vec, + // ) -> Result> { + // extract_macro_defs(&mut exprs, global_macro_map)?; + // exprs + // .into_iter() + // .map(|x| expand(x, global_macro_map)) + // .collect() + // } } // Pre-compile module to bytecode? Is it even possible? @@ -1665,12 +1671,16 @@ impl<'a> ModuleBuilder<'a> { // change the path to not always be required // if its not required we know its not coming in + #[cfg(not(target_arch = "wasm32"))] let name = if let Some(p) = name { std::fs::canonicalize(p)? } else { std::env::current_dir()? }; + #[cfg(target_arch = "wasm32")] + let name = PathBuf::new(); + Ok(ModuleBuilder { name, main: true, @@ -1826,6 +1836,10 @@ impl<'a> ModuleBuilder<'a> { .filter(|x| matches!(x.path, PathOrBuiltIn::Path(_))) .map(|x| x.path.get_path()) { + if cfg!(target_arch = "wasm32") { + stop!(Generic => "requiring modules is not supported for wasm"); + } + let last_modified = std::fs::metadata(module.as_ref())?.modified()?; // Check if we should compile based on the last time modified @@ -2494,6 +2508,10 @@ impl<'a> ModuleBuilder<'a> { return Ok(()); } + if cfg!(target_arch = "wasm32") { + stop!(Generic => "requiring modules is not supported for wasm"); + } + let mut current = self.name.clone(); if current.is_file() { current.pop(); @@ -2777,6 +2795,7 @@ impl<'a> ModuleBuilder<'a> { } fn parse_builtin(mut self, input: Cow<'static, str>) -> Result { + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); let id = self @@ -2790,6 +2809,7 @@ impl<'a> ModuleBuilder<'a> { self.source_ast = parsed; + #[cfg(feature = "profiling")] log::debug!(target: "pipeline_time", "Parsing: {:?} - {:?}", self.name, now.elapsed()); // self.source_ast.pretty_print(); diff --git a/crates/steel-core/src/compiler/passes/analysis.rs b/crates/steel-core/src/compiler/passes/analysis.rs index 673c75ab3..d7d86ef59 100644 --- a/crates/steel-core/src/compiler/passes/analysis.rs +++ b/crates/steel-core/src/compiler/passes/analysis.rs @@ -325,10 +325,12 @@ impl Analysis { pub fn fresh_from_exprs(&mut self, exprs: &[ExprKind]) { self.clear(); + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); self.run(exprs); + #[cfg(feature = "profiling")] log::debug!(target: "pipeline_time", "Analysis time: {:?}", now.elapsed()); } @@ -3896,6 +3898,7 @@ impl<'a> SemanticAnalysis<'a> { module_manager: &mut ModuleManager, table: &mut FxHashSet, ) -> &mut Self { + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); let mut replacer = @@ -3941,6 +3944,7 @@ impl<'a> SemanticAnalysis<'a> { macro_replacer.visit(expr); } + #[cfg(feature = "profiling")] log::info!( target: "pipeline_time", "Replacing non shadowed globals time: {:?}", @@ -3961,6 +3965,7 @@ impl<'a> SemanticAnalysis<'a> { macros: &FxHashMap, module_manager: &ModuleManager, ) -> &mut Self { + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); let module_get_interned: InternedString = "%module-get%".into(); @@ -4038,6 +4043,7 @@ impl<'a> SemanticAnalysis<'a> { } if !found { + #[cfg(feature = "profiling")] log::debug!(target: "pipeline_time", "Removing unused globals time: {:?}", now.elapsed()); return self; @@ -4165,6 +4171,7 @@ impl<'a> SemanticAnalysis<'a> { return true; }); + #[cfg(feature = "profiling")] log::debug!(target: "pipeline_time", "Removing unused globals time: {:?}", now.elapsed()); // self.exprs.push(ExprKind::ident("void")); @@ -4312,6 +4319,7 @@ impl<'a> SemanticAnalysis<'a> { } pub fn replace_anonymous_function_calls_with_plain_lets(&mut self) -> &mut Self { + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); let mut re_run_analysis = false; @@ -4360,6 +4368,7 @@ impl<'a> SemanticAnalysis<'a> { self.find_anonymous_function_calls_and_mutate_with(func); + #[cfg(feature = "profiling")] log::debug!(target: "pipeline_time", "Anonymous function calls -> lets time: {:?}", now.elapsed()); if re_run_analysis { diff --git a/crates/steel-core/src/compiler/passes/begin.rs b/crates/steel-core/src/compiler/passes/begin.rs index d2c4ebd23..3249194f5 100644 --- a/crates/steel-core/src/compiler/passes/begin.rs +++ b/crates/steel-core/src/compiler/passes/begin.rs @@ -1,4 +1,3 @@ -use log::debug; use steel_parser::{ ast::{Define, If, Let, Macro, Quote, Require, Return, SyntaxRules}, tokens::MaybeBigInt, @@ -10,6 +9,8 @@ use crate::parser::{ visitors::VisitorMutRef, }; use crate::parser::{interner::InternedString, tokens::TokenType}; + +#[cfg(feature = "profiling")] use std::time::Instant; use super::{Folder, VisitorMutRefUnit, VisitorMutUnit}; @@ -284,6 +285,7 @@ impl VisitorMutRefUnit for FlattenBegin { pub fn flatten_begins_and_expand_defines( exprs: Vec, ) -> crate::rvals::Result> { + #[cfg(feature = "profiling")] let flatten_begins_and_expand_defines_time = Instant::now(); let res = exprs @@ -300,7 +302,8 @@ pub fn flatten_begins_and_expand_defines( }) .collect(); - debug!( + #[cfg(feature = "profiling")] + log::debug!( target: "pipeline_time", "Flatten begins and expand defines time: {:?}", flatten_begins_and_expand_defines_time.elapsed() diff --git a/crates/steel-core/src/compiler/program.rs b/crates/steel-core/src/compiler/program.rs index 451d5b3db..c61b8b44f 100644 --- a/crates/steel-core/src/compiler/program.rs +++ b/crates/steel-core/src/compiler/program.rs @@ -916,6 +916,38 @@ impl RawProgramWithSymbols { self } + pub fn debug_generate_instructions( + mut self, + symbol_map: &mut SymbolMap, + ) -> Result> { + let mut interner = DebruijnIndicesInterner::default(); + + for expression in &mut self.instructions { + interner.collect_first_pass_defines(expression, symbol_map)? + } + + for expression in &mut self.instructions { + interner.collect_second_pass_defines(expression, symbol_map)? + } + + // TODO try here - the loop condition local const arity two seems to rely on the + // existence of having been already adjusted by the interner + for instructions in &mut self.instructions { + // loop_condition_local_const_arity_two(instructions); + specialize_constants(instructions)?; + } + + // Put the new struct functions at the front + // struct_instructions.append(&mut self.instructions); + // self.instructions = struct_instructions; + + Ok(self + .instructions + .into_iter() + .map(|i| crate::core::instructions::disassemble(&i)) + .collect()) + } + pub fn debug_build(mut self, _name: String, symbol_map: &mut SymbolMap) -> Result<()> { #[cfg(feature = "profiling")] let now = Instant::now(); @@ -1017,7 +1049,10 @@ impl RawProgramWithSymbols { Ok(Executable { name: Rc::new(name), version: Rc::new(self.version), - time_stamp: SystemTime::now(), + #[cfg(not(target_arch = "wasm32"))] + time_stamp: Some(SystemTime::now()), + #[cfg(target_arch = "wasm32")] + time_stamp: None, instructions: instructions .into_iter() .map(|x| Rc::from(x.into_boxed_slice())) @@ -1091,7 +1126,7 @@ fn extract_spans( pub struct Executable { pub(crate) name: Rc, pub(crate) version: Rc, - pub(crate) time_stamp: SystemTime, // TODO -> don't use system time, probably not as portable, prefer date time + pub(crate) time_stamp: Option, // TODO -> don't use system time, probably not as portable, prefer date time pub(crate) instructions: Vec>, pub(crate) constant_map: ConstantMap, pub(crate) spans: Vec>, @@ -1102,7 +1137,7 @@ impl Executable { &self.name } - pub fn time_stamp(&self) -> &SystemTime { + pub fn time_stamp(&self) -> &Option { &self.time_stamp } } diff --git a/crates/steel-core/src/parser/kernel.rs b/crates/steel-core/src/parser/kernel.rs index 71880085f..c1f7ea8d1 100644 --- a/crates/steel-core/src/parser/kernel.rs +++ b/crates/steel-core/src/parser/kernel.rs @@ -517,6 +517,7 @@ impl Kernel { expr: ExprKind, environment: &str, ) -> Result { + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); let span = get_span(&expr); @@ -550,6 +551,7 @@ impl Kernel { // This shouldn't be lowering all the way. It should just be back to list right? let res = TryFromSteelValVisitorForExprKind::root(&result); + #[cfg(feature = "profiling")] log::debug!(target: "pipeline_time", "Kernel expansion time: {:?}", now.elapsed()); res diff --git a/crates/steel-core/src/primitives/process.rs b/crates/steel-core/src/primitives/process.rs index 9fa0f5c63..03faccf0a 100644 --- a/crates/steel-core/src/primitives/process.rs +++ b/crates/steel-core/src/primitives/process.rs @@ -42,10 +42,14 @@ struct ProcessExitStatus { } fn binary_exists_on_path(binary: String) -> Option { + #[cfg(not(target_arch = "wasm32"))] match which::which(binary) { Ok(v) => Some(v.into_os_string().into_string().unwrap()), Err(_) => None, } + + #[cfg(target_arch = "wasm32")] + None } impl ProcessExitStatus { diff --git a/crates/steel-core/src/primitives/time.rs b/crates/steel-core/src/primitives/time.rs index 9771f85c1..ed005942b 100644 --- a/crates/steel-core/src/primitives/time.rs +++ b/crates/steel-core/src/primitives/time.rs @@ -2,7 +2,8 @@ use crate::gc::Gc; use crate::SteelVal; use crate::{rvals::Custom, steel_vm::builtin::MarkdownDoc}; use chrono::Local; -use std::{time::Duration, time::Instant}; +use std::time::Duration; +use std::time::Instant; use steel_derive::function; use crate::steel_vm::builtin::BuiltInModule; diff --git a/crates/steel-core/src/steel_vm/engine.rs b/crates/steel-core/src/steel_vm/engine.rs index b70e76254..c6be5dff3 100644 --- a/crates/steel-core/src/steel_vm/engine.rs +++ b/crates/steel-core/src/steel_vm/engine.rs @@ -300,10 +300,12 @@ pub fn load_module_noop(target: &crate::rvals::SteelString) -> crate::rvals::Res macro_rules! time { ($target:expr, $label:expr, $e:expr) => {{ + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); let e = $e; + #[cfg(feature = "profiling")] log::debug!(target: $target, "{}: {:?}", $label, now.elapsed()); e @@ -323,7 +325,9 @@ impl Engine { /// kernel in the compiler pub(crate) fn new_kernel() -> Self { log::debug!(target:"kernel", "Instantiating a new kernel"); + #[cfg(feature = "profiling")] let mut total_time = std::time::Instant::now(); + #[cfg(feature = "profiling")] let mut now = std::time::Instant::now(); let mut vm = Engine { @@ -382,22 +386,25 @@ impl Engine { // } // ) // } else { + time!( "engine-creation", "Loading the ALL_MODULES prelude code", vm.compile_and_run_raw_program(crate::steel_vm::primitives::ALL_MODULES) - .unwrap() + .expect("loading ALL_MODULES failed") ); // } // log::debug!(target: "kernel", "Registered modules in the kernel!: {:?}", now.elapsed()); + #[cfg(feature = "profiling")] let mut now = std::time::Instant::now(); let core_libraries = [crate::stdlib::PRELUDE]; for core in core_libraries.into_iter() { - vm.compile_and_run_raw_program(core).unwrap(); + vm.compile_and_run_raw_program(core) + .expect("Loading the standard library failed"); } // Initialize the global macro environment with the default one. This way @@ -409,8 +416,10 @@ impl Engine { *guard = vm.in_scope_macros().clone(); }); + #[cfg(feature = "profiling")] log::debug!(target: "kernel", "Loaded prelude in the kernel!: {:?}", now.elapsed()); + #[cfg(feature = "profiling")] log::debug!(target: "pipeline_time", "Total kernel loading time: {:?}", total_time.elapsed()); vm @@ -1168,6 +1177,7 @@ impl Engine { engine.compiler.kernel = Some(Kernel::new()); + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); if let Err(e) = engine.run(PRELUDE_WITHOUT_BASE) { @@ -1175,6 +1185,7 @@ impl Engine { panic!("This shouldn't happen!"); } + #[cfg(feature = "profiling")] log::info!(target: "engine-creation", "Engine Creation: {:?}", now.elapsed()); engine @@ -1303,6 +1314,11 @@ impl Engine { ) } + #[doc(hidden)] + pub fn debug_build_strings(&mut self, program: RawProgramWithSymbols) -> Result> { + program.debug_generate_instructions(&mut self.compiler.symbol_map) + } + pub fn debug_print_build( &mut self, name: String, @@ -1919,11 +1935,11 @@ impl Engine { .compiler .get_idx(constant) .ok_or_else(throw!( - Generic => format!("unreachable") + Generic => format!("Constants: unreachable") )) .and_then(|idx| { self.virtual_machine.extract_value(idx).ok_or_else(throw!( - Generic => "unreachable" + Generic => "Constants: unreachable" )) }); diff --git a/crates/steel-core/src/steel_vm/vm/threads.rs b/crates/steel-core/src/steel_vm/vm/threads.rs index b6cf91140..aba7de996 100644 --- a/crates/steel-core/src/steel_vm/vm/threads.rs +++ b/crates/steel-core/src/steel_vm/vm/threads.rs @@ -13,10 +13,12 @@ use super::*; // TODO: Do proper logging here for thread spawning macro_rules! time { ($label:expr, $e:expr) => {{ + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); let e = $e; + #[cfg(feature = "profiling")] log::debug!(target: "threads", "{}: {:?}", $label, now.elapsed()); e @@ -132,6 +134,7 @@ struct MovableFunctionInterner { fn spawn_thread_result(ctx: &mut VmCore, args: &[SteelVal]) -> Result { use crate::rvals::SerializableSteelVal; + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); // Need a new: @@ -379,6 +382,7 @@ fn spawn_thread_result(ctx: &mut VmCore, args: &[SteelVal]) -> Result constant_map, }; + #[cfg(feature = "profiling")] log::info!(target: "threads", "Time taken to spawn thread: {:?}", now.elapsed()); // Call the function! diff --git a/crates/steel-core/src/values/closed.rs b/crates/steel-core/src/values/closed.rs index 816d6b018..b6aa250c6 100644 --- a/crates/steel-core/src/values/closed.rs +++ b/crates/steel-core/src/values/closed.rs @@ -390,6 +390,7 @@ impl Heap { ) { log::debug!(target: "gc", "Marking the heap"); + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); let mut context = MarkAndSweepContext { @@ -439,8 +440,10 @@ impl Heap { context.visit(); + #[cfg(feature = "profiling")] log::debug!(target: "gc", "Mark: Time taken: {:?}", now.elapsed()); + #[cfg(feature = "profiling")] let now = std::time::Instant::now(); log::debug!(target: "gc", "--- Sweeping ---"); @@ -463,6 +466,7 @@ impl Heap { ROOTS.with(|x| x.borrow_mut().increment_generation()); + #[cfg(feature = "profiling")] log::debug!(target: "gc", "Sweep: Time taken: {:?}", now.elapsed()); } }