Adds perf jitdump support

Patch adds support for the perf jitdump file specification.
With this patch it should be possible to see profile data for code
generated and maped at runtime. Specifically the patch adds support
for the JIT_CODE_LOAD and the JIT_DEBUG_INFO record as described in
the specification. Dumping jitfiles is enabled with the --jitdump
flag. When the -g flag is also used there is an attempt to dump file
and line number information where this option would be most useful
when the WASM file already includes DWARF debug information.

The generation of the jitdump files has been tested on only a few wasm
files. This patch is expected to be useful/serviceable where currently
there is no means for jit profiling, but future patches may benefit
line mapping and add support for additional jitdump record types.

Usage Example:
Record
  sudo perf record -k 1 -e instructions:u target/debug/wasmtime -g
  --jitdump test.wasm
Combine
  sudo perf inject -v -j -i perf.data -o perf.jit.data
Report
  sudo perf report -i perf.jit.data -F+period,srcline

Cargo.toml

@@ -29,6 +29,7 @@ wasmtime-environ = { path = "crates/environ" }
 wasmtime-interface-types = { path = "crates/interface-types" }
 wasmtime-jit = { path = "crates/jit" }
 wasmtime-obj = { path = "crates/obj" }
+wasmtime-profiling = { path = "crates/profiling" }
 wasmtime-wast = { path = "crates/wast" }
 wasmtime-wasi = { path = "crates/wasi" }
 wasi-common = { path = "crates/wasi-common" }
@@ -74,6 +75,7 @@ lightbeam = [
     "wasmtime-wast/lightbeam",
     "wasmtime/lightbeam",
 ]
+profiling = ["wasmtime-jit/profiling", "wasmtime/profiling", "wasmtime-profiling/profiling"]
 test_programs = ["test-programs/test_programs"]
 
 [badges]

crates/api/Cargo.toml

@@ -12,6 +12,7 @@ edition = "2018"
 wasmtime-runtime = { path = "../runtime", version = "0.9.0" }
 wasmtime-environ = { path = "../environ", version = "0.9.0" }
 wasmtime-jit = { path = "../jit", version = "0.9.0" }
+wasmtime-profiling = { path = "../profiling" }
 wasmparser = "0.51.1"
 target-lexicon = { version = "0.10.0", default-features = false }
 anyhow = "1.0.19"
@@ -44,3 +45,4 @@ default = ['wat']
 # to cranelift. Requires Nightly Rust currently, and this is not enabled by
 # default.
 lightbeam = ["wasmtime-jit/lightbeam"]
+profiling = []

crates/api/src/module.rs

@@ -253,6 +253,7 @@ impl Module {
             &mut store.compiler_mut(),
             binary,
             store.engine().config().debug_info,
+            store.engine().config().profiler.as_ref(),
         )?;
 
         let names = Arc::new(Names {

crates/api/src/runtime.rs

@@ -3,11 +3,12 @@ use std::cell::RefCell;
 use std::fmt;
 use std::path::Path;
 use std::rc::Rc;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use wasmparser::{OperatorValidatorConfig, ValidatingParserConfig};
 use wasmtime_environ::settings::{self, Configurable};
 use wasmtime_environ::CacheConfig;
 use wasmtime_jit::{native, CompilationStrategy, Compiler};
+use wasmtime_profiling::{JitDumpAgent, ProfilingAgent, ProfilingStrategy};
 
 // Runtime Environment
 
@@ -25,6 +26,7 @@ pub struct Config {
     pub(crate) debug_info: bool,
     pub(crate) strategy: CompilationStrategy,
     pub(crate) cache_config: CacheConfig,
+    pub(crate) profiler: Option<Arc<Mutex<Box<dyn ProfilingAgent + Send>>>>,
 }
 
 impl Config {
@@ -58,6 +60,7 @@ impl Config {
             flags,
             strategy: CompilationStrategy::Auto,
             cache_config: CacheConfig::new_cache_disabled(),
+            profiler: None,
         }
     }
 
@@ -212,6 +215,20 @@ impl Config {
         Ok(self)
     }
 
+    /// Creates a default profiler based on the profiling strategy choosen
+    ///
+    /// Profiler creation calls the type's default initializer where the purpose is
+    /// really just to put in place the type used for profiling.
+    pub fn profiler(&mut self, profile: ProfilingStrategy) -> Result<&mut Self> {
+        match profile {
+            ProfilingStrategy::JitDumpProfiler => {
+                self.profiler = { Some(Arc::new(Mutex::new(Box::new(JitDumpAgent::default())))) }
+            }
+            _ => self.profiler = { None },
+        };
+        Ok(self)
+    }
+
     /// Configures whether the debug verifier of Cranelift is enabled or not.
     ///
     /// When Cranelift is used as a code generation backend this will configure

crates/jit/Cargo.toml

@@ -19,6 +19,7 @@ cranelift-frontend = "0.58.0"
 wasmtime-environ = { path = "../environ", version = "0.9.0" }
 wasmtime-runtime = { path = "../runtime", version = "0.9.0" }
 wasmtime-debug = { path = "../debug", version = "0.9.0" }
+wasmtime-profiling = { path = "../profiling" }
 region = "2.0.0"
 thiserror = "1.0.4"
 target-lexicon = { version = "0.10.0", default-features = false }
@@ -32,6 +33,7 @@ winapi = { version = "0.3.7", features = ["winnt", "impl-default"] }
 
 [features]
 lightbeam = ["wasmtime-environ/lightbeam"]
+profiling = ["wasmtime-profiling/profiling"]
 
 [badges]
 maintenance = { status = "actively-developed" }

crates/jit/src/code_memory.rs

@@ -5,6 +5,7 @@ use region;
 use std::mem::ManuallyDrop;
 use std::{cmp, mem};
 use wasmtime_environ::{Compilation, CompiledFunction};
+use wasmtime_profiling::ProfilingAgent;
 use wasmtime_runtime::{Mmap, VMFunctionBody};
 
 struct CodeMemoryEntry {
@@ -230,4 +231,22 @@ impl CodeMemory {
 
         Ok(())
     }
+
+    /// Calls the module_load for a given ProfilerAgent. Includes
+    /// all memory address and length for the given module.
+    /// TODO: Properly handle the possibilities of multiple mmapped regions
+    /// which may, amongst other things, influence being more specific about
+    /// the module name.
+    pub fn profiler_module_load(
+        &mut self,
+        profiler: &mut Box<dyn ProfilingAgent + Send>,
+        module_name: &str,
+        dbg_image: Option<&[u8]>,
+    ) -> () {
+        for CodeMemoryEntry { mmap: m, table: _t } in &mut self.entries {
+            if m.len() > 0 {
+                profiler.module_load(module_name, m.as_ptr(), m.len(), dbg_image);
+            }
+        }
+    }
 }

crates/jit/src/compiler.rs

@@ -20,6 +20,7 @@ use wasmtime_environ::{
     Compiler as _C, FunctionBodyData, Module, ModuleMemoryOffset, ModuleVmctxInfo, Relocations,
     Traps, Tunables, VMOffsets,
 };
+use wasmtime_profiling::ProfilingAgent;
 use wasmtime_runtime::{
     InstantiationError, SignatureRegistry, TrapRegistration, TrapRegistry, VMFunctionBody,
 };
@@ -240,6 +241,16 @@ impl Compiler {
         self.code_memory.publish();
     }
 
+    pub(crate) fn profiler_module_load(
+        &mut self,
+        profiler: &mut Box<dyn ProfilingAgent + Send>,
+        module_name: &str,
+        dbg_image: Option<&[u8]>,
+    ) -> () {
+        self.code_memory
+            .profiler_module_load(profiler, module_name, dbg_image);
+    }
+
     /// Shared signature registry.
     pub fn signatures(&self) -> &SignatureRegistry {
         &self.signatures

crates/jit/src/instantiate.rs

@@ -9,14 +9,16 @@ use crate::link::link_module;
 use crate::resolver::Resolver;
 use std::io::Write;
 use std::rc::Rc;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use thiserror::Error;
 use wasmtime_debug::read_debuginfo;
 use wasmtime_environ::entity::{BoxedSlice, PrimaryMap};
 use wasmtime_environ::wasm::{DefinedFuncIndex, SignatureIndex};
 use wasmtime_environ::{
     CompileError, DataInitializer, DataInitializerLocation, Module, ModuleEnvironment,
 };
+
+use wasmtime_profiling::ProfilingAgent;
 use wasmtime_runtime::{
     GdbJitImageRegistration, InstanceHandle, InstantiationError, TrapRegistration, VMFunctionBody,
     VMSharedSignatureIndex,
@@ -61,6 +63,7 @@ impl<'data> RawCompiledModule<'data> {
         compiler: &mut Compiler,
         data: &'data [u8],
         debug_info: bool,
+        profiler: Option<&Arc<Mutex<Box<dyn ProfilingAgent + Send>>>>,
     ) -> Result<Self, SetupError> {
         let environ = ModuleEnvironment::new(compiler.frontend_config(), compiler.tunables());
 
@@ -103,6 +106,23 @@ impl<'data> RawCompiledModule<'data> {
         // Make all code compiled thus far executable.
         compiler.publish_compiled_code();
 
+        // Initialize profiler and load the wasm module
+        let _: Option<Box<dyn ProfilingAgent>> = match profiler {
+            Some(_) => {
+                let region_name = String::from("wasm_module");
+                let mut profiler_new = profiler.unwrap().lock().unwrap().clone();
+                profiler_new.init().ok();
+                match &dbg_image {
+                    Some(dbg) => {
+                        compiler.profiler_module_load(&mut profiler_new, &region_name, Some(&dbg))
+                    }
+                    _ => compiler.profiler_module_load(&mut profiler_new, &region_name, None),
+                };
+                Some(profiler_new)
+            }
+            _ => None,
+        };
+
         let dbg_jit_registration = if let Some(img) = dbg_image {
             let mut bytes = Vec::new();
             bytes.write_all(&img).expect("all written");
@@ -139,8 +159,9 @@ impl CompiledModule {
         compiler: &mut Compiler,
         data: &'data [u8],
         debug_info: bool,
+        profiler: Option<&Arc<Mutex<Box<dyn ProfilingAgent + Send>>>>,
     ) -> Result<Self, SetupError> {
-        let raw = RawCompiledModule::<'data>::new(compiler, data, debug_info)?;
+        let raw = RawCompiledModule::<'data>::new(compiler, data, debug_info, profiler)?;
 
         Ok(Self::from_parts(
             raw.module,
@@ -246,7 +267,7 @@ impl OwnedDataInitializer {
 
 /// Create a new wasm instance by compiling the wasm module in `data` and instatiating it.
 ///
-/// This is equivalent to createing a `CompiledModule` and calling `instantiate()` on it,
+/// This is equivalent to creating a `CompiledModule` and calling `instantiate()` on it,
 /// but avoids creating an intermediate copy of the data initializers.
 ///
 /// # Unsafety
@@ -258,7 +279,9 @@ pub unsafe fn instantiate(
     data: &[u8],
     resolver: &mut dyn Resolver,
     debug_info: bool,
+    profiler: Option<&Arc<Mutex<Box<dyn ProfilingAgent + Send>>>>,
 ) -> Result<InstanceHandle, SetupError> {
-    let instance = CompiledModule::new(compiler, data, debug_info)?.instantiate(resolver)?;
+    let instance =
+        CompiledModule::new(compiler, data, debug_info, profiler)?.instantiate(resolver)?;
     Ok(instance)
 }

crates/profiling/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "wasmtime-profiling"
+version = "0.1.0"
+authors = ["The Wasmtime Project Developers"]
+description = "Runtime library support for Wasmtime"
+license = "Apache-2.0 WITH LLVM-exception"
+categories = ["wasm"]
+keywords = ["webassembly", "wasm"]
+repository = "https://github.com/bytecodealliance/wasmtime"
+readme = "README.md"
+edition = "2018"
+
+[dependencies]
+libc = { version = "0.2.60", default-features = false }
+goblin = "0.0.24"
+serde = { version = "1.0.99", features = ["derive"] }
+scroll = "0.9.2"
+gimli = "0.19.0"
+object = "0.12.0"
+target-lexicon = "0.4.0"
+lazy_static = "1.4"
+
+[badges]
+maintenance = { status = "actively-developed" }
+
+[features]
+profiling = []