From 1c529a56d33d81180ae812721d59b19e574fcc35 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Wed, 20 Nov 2024 15:05:16 +0100 Subject: [PATCH 01/11] no need for custom INI parsing --- profiling/build.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/profiling/build.rs b/profiling/build.rs index 77f5349b8f..662a9411cc 100644 --- a/profiling/build.rs +++ b/profiling/build.rs @@ -363,6 +363,7 @@ fn cfg_php_feature_flags(vernum: u64) { fn cfg_zts() { let output = Command::new("php") + .arg("-n") .arg("-r") .arg("echo PHP_ZTS, PHP_EOL;") .output() From 793b123a079a4e20f1eb25ae79a08ff243a8fbc9 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Wed, 20 Nov 2024 13:22:42 +0100 Subject: [PATCH 02/11] move allocation profiling into own mod --- profiling/build.rs | 1 + .../allocation_le83.rs} | 96 +------------- profiling/src/allocation/mod.rs | 120 ++++++++++++++++++ profiling/src/config.rs | 2 +- 4 files changed, 124 insertions(+), 95 deletions(-) rename profiling/src/{allocation.rs => allocation/allocation_le83.rs} (85%) create mode 100644 profiling/src/allocation/mod.rs diff --git a/profiling/build.rs b/profiling/build.rs index 662a9411cc..ea22fd0fcb 100644 --- a/profiling/build.rs +++ b/profiling/build.rs @@ -358,6 +358,7 @@ fn cfg_php_feature_flags(vernum: u64) { if vernum >= 80400 { println!("cargo:rustc-cfg=php_frameless"); println!("cargo:rustc-cfg=php_opcache_restart_hook"); + println!("cargo:rustc-cfg=php_new_zendmm_hooks"); } } diff --git a/profiling/src/allocation.rs b/profiling/src/allocation/allocation_le83.rs similarity index 85% rename from profiling/src/allocation.rs rename to profiling/src/allocation/allocation_le83.rs index 8c05881a94..2798ab964c 100644 --- a/profiling/src/allocation.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -1,41 +1,18 @@ +use crate::allocation::{ALLOCATION_PROFILING_COUNT, ALLOCATION_PROFILING_SIZE, ALLOCATION_PROFILING_STATS}; use crate::bindings::{ self as zend, datadog_php_install_handler, datadog_php_zif_handler, ddog_php_prof_copy_long_into_zval, }; -use crate::profiling::Profiler; use crate::{PROFILER_NAME, REQUEST_LOCALS}; use lazy_static::lazy_static; use libc::{c_char, c_int, c_void, size_t}; use log::{debug, error, trace, warn}; -use rand::rngs::ThreadRng; -use rand_distr::{Distribution, Poisson}; -use std::cell::{RefCell, UnsafeCell}; -use std::sync::atomic::AtomicU64; +use std::cell::UnsafeCell; use std::sync::atomic::Ordering::{Relaxed, SeqCst}; use std::{ffi, ptr}; static mut GC_MEM_CACHES_HANDLER: zend::InternalFunctionHandler = None; -/// take a sample every 4096 KiB -pub const ALLOCATION_PROFILING_INTERVAL: f64 = 1024.0 * 4096.0; - -/// This will store the count of allocations (including reallocations) during -/// a profiling period. This will overflow when doing more than u64::MAX -/// allocations, which seems big enough to ignore. -pub static ALLOCATION_PROFILING_COUNT: AtomicU64 = AtomicU64::new(0); - -/// This will store the accumulated size of all allocations in bytes during the -/// profiling period. This will overflow when allocating more than 18 exabyte -/// of memory (u64::MAX) which might not happen, so we can ignore this. -pub static ALLOCATION_PROFILING_SIZE: AtomicU64 = AtomicU64::new(0); - -pub struct AllocationProfilingStats { - /// number of bytes until next sample collection - next_sample: i64, - poisson: Poisson, - rng: ThreadRng, -} - type ZendHeapPrepareFn = unsafe fn(heap: *mut zend::_zend_mm_heap) -> c_int; type ZendHeapRestoreFn = unsafe fn(heap: *mut zend::_zend_mm_heap, custom_heap: c_int); @@ -69,49 +46,7 @@ struct ZendMMState { free: unsafe fn(*mut c_void), } -impl AllocationProfilingStats { - fn new() -> AllocationProfilingStats { - // Safety: this will only error if lambda <= 0 - let poisson = Poisson::new(ALLOCATION_PROFILING_INTERVAL).unwrap(); - let mut stats = AllocationProfilingStats { - next_sample: 0, - poisson, - rng: rand::thread_rng(), - }; - stats.next_sampling_interval(); - stats - } - - fn next_sampling_interval(&mut self) { - self.next_sample = self.poisson.sample(&mut self.rng) as i64; - } - - fn track_allocation(&mut self, len: size_t) { - self.next_sample -= len as i64; - - if self.next_sample > 0 { - return; - } - - self.next_sampling_interval(); - - if let Some(profiler) = Profiler::get() { - // Safety: execute_data was provided by the engine, and the profiler doesn't mutate it. - unsafe { - profiler.collect_allocations( - zend::ddog_php_prof_get_current_execute_data(), - 1_i64, - len as i64, - ) - }; - } - } -} - thread_local! { - static ALLOCATION_PROFILING_STATS: RefCell = - RefCell::new(AllocationProfilingStats::new()); - /// Using an `UnsafeCell` here should be okay. There might not be any /// synchronisation issues, as it is used in as thread local and only /// mutated in RINIT and RSHUTDOWN. @@ -167,23 +102,6 @@ pub fn first_rinit_should_disable_due_to_jit() -> bool { } pub fn alloc_prof_rinit() { - let allocation_profiling: bool = REQUEST_LOCALS.with(|cell| { - match cell.try_borrow() { - Ok(locals) => { - let system_settings = locals.system_settings(); - system_settings.profiling_allocation_enabled - }, - Err(_err) => { - error!("Memory allocation was not initialized correctly due to a borrow error. Please report this to Datadog."); - false - } - } - }); - - if !allocation_profiling { - return; - } - ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); @@ -247,16 +165,6 @@ pub fn alloc_prof_rinit() { } pub fn alloc_prof_rshutdown() { - let allocation_profiling = REQUEST_LOCALS.with(|cell| { - cell.try_borrow() - .map(|locals| locals.system_settings().profiling_allocation_enabled) - .unwrap_or(false) - }); - - if !allocation_profiling { - return; - } - // If `is_zend_mm()` is true, the custom handlers have been reset to `None` // already. This is unexpected, therefore we will not touch the ZendMM // handlers anymore as resetting to prev handlers might result in segfaults diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs new file mode 100644 index 0000000000..0b2c3a12e2 --- /dev/null +++ b/profiling/src/allocation/mod.rs @@ -0,0 +1,120 @@ +use crate::bindings::{self as zend}; +use crate::profiling::Profiler; +use crate::REQUEST_LOCALS; +use libc::size_t; +use log::{error, trace}; +use rand::rngs::ThreadRng; +use rand_distr::{Distribution, Poisson}; +use std::cell::RefCell; +use std::sync::atomic::AtomicU64; + +pub mod allocation_le83; + +/// take a sample every 4096 KiB +pub const ALLOCATION_PROFILING_INTERVAL: f64 = 1024.0 * 4096.0; + +/// This will store the count of allocations (including reallocations) during +/// a profiling period. This will overflow when doing more than u64::MAX +/// allocations, which seems big enough to ignore. +pub static ALLOCATION_PROFILING_COUNT: AtomicU64 = AtomicU64::new(0); + +/// This will store the accumulated size of all allocations in bytes during the +/// profiling period. This will overflow when allocating more than 18 exabyte +/// of memory (u64::MAX) which might not happen, so we can ignore this. +pub static ALLOCATION_PROFILING_SIZE: AtomicU64 = AtomicU64::new(0); + +pub struct AllocationProfilingStats { + /// number of bytes until next sample collection + next_sample: i64, + poisson: Poisson, + rng: ThreadRng, +} + +impl AllocationProfilingStats { + fn new() -> AllocationProfilingStats { + // Safety: this will only error if lambda <= 0 + let poisson = Poisson::new(ALLOCATION_PROFILING_INTERVAL).unwrap(); + let mut stats = AllocationProfilingStats { + next_sample: 0, + poisson, + rng: rand::thread_rng(), + }; + stats.next_sampling_interval(); + stats + } + + fn next_sampling_interval(&mut self) { + self.next_sample = self.poisson.sample(&mut self.rng) as i64; + } + + fn track_allocation(&mut self, len: size_t) { + self.next_sample -= len as i64; + + if self.next_sample > 0 { + return; + } + + self.next_sampling_interval(); + + if let Some(profiler) = Profiler::get() { + // Safety: execute_data was provided by the engine, and the profiler doesn't mutate it. + unsafe { + profiler.collect_allocations( + zend::ddog_php_prof_get_current_execute_data(), + 1_i64, + len as i64, + ) + }; + } + } +} + +thread_local! { + static ALLOCATION_PROFILING_STATS: RefCell = + RefCell::new(AllocationProfilingStats::new()); +} + +pub fn alloc_prof_minit() { + allocation_le83::alloc_prof_minit(); +} + +pub fn alloc_prof_startup() { + allocation_le83::alloc_prof_startup(); +} + +pub fn alloc_prof_rinit() { + let allocation_profiling: bool = REQUEST_LOCALS.with(|cell| { + match cell.try_borrow() { + Ok(locals) => { + let system_settings = locals.system_settings(); + system_settings.profiling_allocation_enabled + }, + Err(_err) => { + error!("Memory allocation was not initialized correctly due to a borrow error. Please report this to Datadog."); + false + } + } + }); + + if !allocation_profiling { + return; + } + + allocation_le83::alloc_prof_rinit(); + + trace!("Memory allocation profiling enabled.") +} + +pub fn alloc_prof_rshutdown() { + let allocation_profiling = REQUEST_LOCALS.with(|cell| { + cell.try_borrow() + .map(|locals| locals.system_settings().profiling_allocation_enabled) + .unwrap_or(false) + }); + + if !allocation_profiling { + return; + } + + allocation_le83::alloc_prof_rshutdown(); +} diff --git a/profiling/src/config.rs b/profiling/src/config.rs index d8a9c4fb18..a64ca0d94d 100644 --- a/profiling/src/config.rs +++ b/profiling/src/config.rs @@ -101,7 +101,7 @@ impl SystemSettings { } // Work around version-specific issues. - if allocation::first_rinit_should_disable_due_to_jit() { + if allocation::allocation_le83::first_rinit_should_disable_due_to_jit() { system_settings.profiling_allocation_enabled = false; } swap(&mut system_settings, SYSTEM_SETTINGS.assume_init_mut()); From 8ebcaf6a527747f664a79dc4c1b14ee1ecc2dfca Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Wed, 20 Nov 2024 14:47:23 +0100 Subject: [PATCH 03/11] Add new ZendMM hooks --- profiling/src/allocation/allocation_ge84.rs | 375 ++++++++++++++++++++ profiling/src/allocation/allocation_le83.rs | 4 +- profiling/src/allocation/mod.rs | 11 + profiling/src/bindings/mod.rs | 4 + profiling/src/config.rs | 2 + 5 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 profiling/src/allocation/allocation_ge84.rs diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs new file mode 100644 index 0000000000..0cddf0736e --- /dev/null +++ b/profiling/src/allocation/allocation_ge84.rs @@ -0,0 +1,375 @@ +use crate::allocation::ALLOCATION_PROFILING_COUNT; +use crate::allocation::ALLOCATION_PROFILING_SIZE; +use crate::allocation::ALLOCATION_PROFILING_STATS; +use crate::bindings::{self as zend}; +use crate::PROFILER_NAME; +use libc::{c_char, c_void, size_t}; +use log::{debug, trace, warn}; +use std::cell::UnsafeCell; +use std::ptr; +use std::sync::atomic::Ordering::SeqCst; + +struct ZendMMState { + /// The heap we create and set as the current heap in ZendMM + heap: Option<*mut zend::zend_mm_heap>, + /// The heap installed in ZendMM at the time we install our custom handlers + prev_heap: Option<*mut zend::zend_mm_heap>, + /// The engine's previous custom allocation function, if there is one. + prev_custom_mm_alloc: Option, + /// The engine's previous custom reallocation function, if there is one. + prev_custom_mm_realloc: Option, + /// The engine's previous custom free function, if there is one. + prev_custom_mm_free: Option, + /// The engine's previous custom gc function, if there is one. + prev_custom_mm_gc: Option, + /// The engine's previous custom shutdown function, if there is one. + prev_custom_mm_shutdown: Option, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_alloc()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_alloc` is initialised to a valid function + /// pointer, otherwise there will be dragons. + alloc: unsafe fn(size_t) -> *mut c_void, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_realloc()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_realloc` is initialised to a valid + /// function pointer, otherwise there will be dragons. + realloc: unsafe fn(*mut c_void, size_t) -> *mut c_void, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_free()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_free` is initialised to a valid function + /// pointer, otherwise there will be dragons. + free: unsafe fn(*mut c_void), + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_gc()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_gc` is initialised to a valid function + /// pointer, otherwise there will be dragons. + gc: unsafe fn() -> size_t, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_shutdown()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_shutdown` is initialised to a valid function + /// pointer, otherwise there will be dragons. + shutdown: unsafe fn(bool, bool), +} + +thread_local! { + /// Using an `UnsafeCell` here should be okay. There might not be any + /// synchronisation issues, as it is used in as thread local and only + /// mutated in RINIT and RSHUTDOWN. + static ZEND_MM_STATE: UnsafeCell = const { + UnsafeCell::new(ZendMMState { + heap: None, + prev_heap: None, + prev_custom_mm_alloc: None, + prev_custom_mm_realloc: None, + prev_custom_mm_free: None, + prev_custom_mm_gc: None, + prev_custom_mm_shutdown: None, + alloc: alloc_prof_orig_alloc, + realloc: alloc_prof_orig_realloc, + free: alloc_prof_orig_free, + gc: alloc_prof_orig_gc, + shutdown: alloc_prof_orig_shutdown, + }) + }; +} + +macro_rules! tls_zend_mm_state { + ($x:ident) => { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + (*zend_mm_state).$x + }) + }; +} + +pub fn alloc_prof_rinit() { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + + // Only need to create an observed heap once per thread. When we have it, we can just + // install the observed hook via `zend::zend_mm_set_heap()` + if unsafe { (*zend_mm_state).heap.is_none() } { + // Safety: `zend_mm_get_heap()` always returns a non-null pointer to a valid heap structure + let prev_heap = unsafe { zend::zend_mm_get_heap() }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(Some(prev_heap)) }; + + if !is_zend_mm() { + // Neighboring custom memory handlers found in the currently used ZendMM heap + debug!("Found another extension using the ZendMM custom handler hook"); + unsafe { + zend::zend_mm_get_custom_handlers_ex( + prev_heap, + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_alloc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_free), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_realloc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown), + ); + ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_prev_alloc); + ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_prev_free); + ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_prev_realloc); + // `gc` handler can be NULL + if (*zend_mm_state).prev_custom_mm_gc.is_none() { + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_orig_gc); + } else { + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_prev_gc); + } + // `shutdown` handler can be NULL + if (*zend_mm_state).prev_custom_mm_shutdown.is_none() { + ptr::addr_of_mut!((*zend_mm_state).shutdown) + .write(alloc_prof_orig_shutdown); + } else { + ptr::addr_of_mut!((*zend_mm_state).shutdown) + .write(alloc_prof_prev_shutdown); + } + } + } + + // Create our observed heap and prepare custom handlers + let heap = unsafe { zend::zend_mm_startup() }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(Some(heap)) }; + + // install our custom handler to ZendMM + unsafe { + zend::zend_mm_set_custom_handlers_ex( + (*zend_mm_state).heap.unwrap(), + Some(alloc_prof_malloc), + Some(alloc_prof_free), + Some(alloc_prof_realloc), + Some(alloc_prof_gc), + Some(alloc_prof_shutdown), + ); + } + } + + // install the observed heap into ZendMM + unsafe { + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + } + }); + + // `is_zend_mm()` should be false now, as we installed our custom handlers + if is_zend_mm() { + // Can't proceed with it being disabled, because that's a system-wide + // setting, not per-request. + panic!("Memory allocation profiling could not be enabled. Please feel free to fill an issue stating the PHP version and installed modules. Most likely the reason is your PHP binary was compiled with `ZEND_MM_CUSTOM` being disabled."); + } + trace!("Memory allocation profiling enabled.") +} + +pub fn alloc_prof_rshutdown() { + // If `is_zend_mm()` is true, the custom handlers have been reset to `None` or our observed + // heap has been uninstalled. This is unexpected, therefore we will not touch the ZendMM + // handlers anymore as resetting to prev handlers might result in segfaults and other undefined + // behaviour. + if is_zend_mm() { + return; + } + + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + + // Do a sanity check and see if something played with our heap + let mut custom_mm_malloc: Option = None; + let mut custom_mm_free: Option = None; + let mut custom_mm_realloc: Option = None; + let mut custom_mm_gc: Option = None; + let mut custom_mm_shutdown: Option = None; + + // Safety: `unwrap()` is safe here, as `heap` is initialized in `RINIT` + let heap = unsafe { (*zend_mm_state).heap.unwrap() }; + unsafe { + zend::zend_mm_get_custom_handlers_ex( + heap, + &mut custom_mm_malloc, + &mut custom_mm_free, + &mut custom_mm_realloc, + &mut custom_mm_gc, + &mut custom_mm_shutdown, + ); + } + if custom_mm_free != Some(alloc_prof_free) + || custom_mm_malloc != Some(alloc_prof_malloc) + || custom_mm_realloc != Some(alloc_prof_realloc) + || custom_mm_gc != Some(alloc_prof_gc) + || custom_mm_shutdown != Some(alloc_prof_shutdown) + { + // Custom handlers are installed, but it's not us. Someone, somewhere might have + // function pointers to our custom handlers. Best bet to avoid segfaults is to not + // touch custom handlers in ZendMM and make sure our extension will not be + // `dlclose()`-ed so the pointers stay valid + let zend_extension = + unsafe { zend::zend_get_extension(PROFILER_NAME.as_ptr() as *const c_char) }; + if !zend_extension.is_null() { + // Safety: Checked for null pointer above. + unsafe { ptr::addr_of_mut!((*zend_extension).handle).write(ptr::null_mut()) }; + } + warn!("Found another extension using the custom heap which is unexpected at this point, so the extension handle was `null`'ed to avoid being `dlclose()`'ed."); + } else { + // This is the happy path. Restore previous heap. + unsafe { + zend::zend_mm_set_heap( + (*zend_mm_state).prev_heap.unwrap() + ); + } + trace!("Memory allocation profiling shutdown gracefully."); + } + }); +} + +unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { + ALLOCATION_PROFILING_COUNT.fetch_add(1, SeqCst); + ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, SeqCst); + + let ptr = tls_zend_mm_state!(alloc)(len); + + // during startup, minit, rinit, ... current_execute_data is null + // we are only interested in allocations during userland operations + if zend::ddog_php_prof_get_current_execute_data().is_null() { + return ptr; + } + + ALLOCATION_PROFILING_STATS.with(|cell| { + let mut allocations = cell.borrow_mut(); + allocations.track_allocation(len) + }); + + ptr +} + +unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + // Safety: `ZEND_MM_STATE.alloc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_alloc` is also initialised + let ptr = ((*zend_mm_state).prev_custom_mm_alloc.unwrap())(len); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + ptr + }) +} + +unsafe fn alloc_prof_orig_alloc(len: size_t) -> *mut c_void { + let ptr: *mut c_void = zend::_zend_mm_alloc(tls_zend_mm_state!(prev_heap).unwrap(), len); + ptr +} + +/// This function exists because when calling `zend_mm_set_custom_handlers()`, +/// you need to pass a pointer to a `free()` function as well, otherwise your +/// custom handlers won't be installed. We can not just point to the original +/// `zend::_zend_mm_free()` as the function definitions differ. +unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { + tls_zend_mm_state!(free)(ptr); +} + +unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + // Safety: `ZEND_MM_STATE.free` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_free` is also initialised + ((*zend_mm_state).prev_custom_mm_free.unwrap())(ptr); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + }) +} + +unsafe fn alloc_prof_orig_free(ptr: *mut c_void) { + zend::_zend_mm_free(tls_zend_mm_state!(prev_heap).unwrap(), ptr); +} + +unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { + ALLOCATION_PROFILING_COUNT.fetch_add(1, SeqCst); + ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, SeqCst); + + let ptr = tls_zend_mm_state!(realloc)(prev_ptr, len); + + // during startup, minit, rinit, ... current_execute_data is null + // we are only interested in allocations during userland operations + if zend::ddog_php_prof_get_current_execute_data().is_null() || ptr == prev_ptr { + return ptr; + } + + ALLOCATION_PROFILING_STATS.with(|cell| { + let mut allocations = cell.borrow_mut(); + allocations.track_allocation(len) + }); + + ptr +} + +unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + // Safety: `ZEND_MM_STATE.realloc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_realloc` is also initialised + let ptr = ((*zend_mm_state).prev_custom_mm_realloc.unwrap())(prev_ptr, len); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + ptr + }) +} + +unsafe fn alloc_prof_orig_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { + zend::_zend_mm_realloc(tls_zend_mm_state!(prev_heap).unwrap(), prev_ptr, len) +} + +unsafe extern "C" fn alloc_prof_gc() -> size_t { + tls_zend_mm_state!(gc)() +} + +unsafe fn alloc_prof_prev_gc() -> size_t { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + // Safety: `ZEND_MM_STATE.gc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_gc` is also initialised + let freed = ((*zend_mm_state).prev_custom_mm_gc.unwrap())(); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + freed + }) +} + +unsafe fn alloc_prof_orig_gc() -> size_t { + zend::zend_mm_gc(tls_zend_mm_state!(prev_heap).unwrap()) +} + +unsafe extern "C" fn alloc_prof_shutdown(full: bool, silent: bool) { + tls_zend_mm_state!(shutdown)(full, silent); +} + +unsafe fn alloc_prof_prev_shutdown(full: bool, silent: bool) { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + // Safety: `ZEND_MM_STATE.shutdown` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_shutdown` is also initialised + ((*zend_mm_state).prev_custom_mm_shutdown.unwrap())(full, silent); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + }) +} + +unsafe fn alloc_prof_orig_shutdown(full: bool, silent: bool) { + zend::zend_mm_shutdown(tls_zend_mm_state!(prev_heap).unwrap(), full, silent) +} + +/// safe wrapper for `zend::is_zend_mm()`. +/// `true` means the internal ZendMM is being used, `false` means that a custom memory manager is +/// installed +fn is_zend_mm() -> bool { + unsafe { zend::is_zend_mm() } +} diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index 2798ab964c..6a8a8b58a2 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -1,4 +1,6 @@ -use crate::allocation::{ALLOCATION_PROFILING_COUNT, ALLOCATION_PROFILING_SIZE, ALLOCATION_PROFILING_STATS}; +use crate::allocation::{ + ALLOCATION_PROFILING_COUNT, ALLOCATION_PROFILING_SIZE, ALLOCATION_PROFILING_STATS, +}; use crate::bindings::{ self as zend, datadog_php_install_handler, datadog_php_zif_handler, ddog_php_prof_copy_long_into_zval, diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index 0b2c3a12e2..f7303d112d 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -8,6 +8,9 @@ use rand_distr::{Distribution, Poisson}; use std::cell::RefCell; use std::sync::atomic::AtomicU64; +#[cfg(php_new_zendmm_hooks)] +mod allocation_ge84; +#[cfg(not(php_new_zendmm_hooks))] pub mod allocation_le83; /// take a sample every 4096 KiB @@ -75,10 +78,12 @@ thread_local! { } pub fn alloc_prof_minit() { + #[cfg(not(php_new_zendmm_hooks))] allocation_le83::alloc_prof_minit(); } pub fn alloc_prof_startup() { + #[cfg(not(php_new_zendmm_hooks))] allocation_le83::alloc_prof_startup(); } @@ -100,7 +105,10 @@ pub fn alloc_prof_rinit() { return; } + #[cfg(not(php_new_zendmm_hooks))] allocation_le83::alloc_prof_rinit(); + #[cfg(php_new_zendmm_hooks)] + allocation_ge84::alloc_prof_rinit(); trace!("Memory allocation profiling enabled.") } @@ -116,5 +124,8 @@ pub fn alloc_prof_rshutdown() { return; } + #[cfg(not(php_new_zendmm_hooks))] allocation_le83::alloc_prof_rshutdown(); + #[cfg(php_new_zendmm_hooks)] + allocation_ge84::alloc_prof_rshutdown(); } diff --git a/profiling/src/bindings/mod.rs b/profiling/src/bindings/mod.rs index 8258ec3da6..bfb178a1af 100644 --- a/profiling/src/bindings/mod.rs +++ b/profiling/src/bindings/mod.rs @@ -46,6 +46,10 @@ pub type VmMmCustomAllocFn = unsafe extern "C" fn(size_t) -> *mut c_void; pub type VmMmCustomReallocFn = unsafe extern "C" fn(*mut c_void, size_t) -> *mut c_void; #[cfg(feature = "allocation_profiling")] pub type VmMmCustomFreeFn = unsafe extern "C" fn(*mut c_void); +#[cfg(all(feature = "allocation_profiling", php_new_zendmm_hooks))] +pub type VmMmCustomGcFn = unsafe extern "C" fn() -> size_t; +#[cfg(all(feature = "allocation_profiling", php_new_zendmm_hooks))] +pub type VmMmCustomShutdownFn = unsafe extern "C" fn(bool, bool); // todo: this a lie on some PHP versions; is it a problem even though zend_bool // was always supposed to be 0 or 1 anyway? diff --git a/profiling/src/config.rs b/profiling/src/config.rs index a64ca0d94d..b84650fbb2 100644 --- a/profiling/src/config.rs +++ b/profiling/src/config.rs @@ -1,3 +1,4 @@ +#[cfg(not(php_new_zendmm_hooks))] use crate::allocation; use crate::bindings::zai_config_type::*; use crate::bindings::{ @@ -101,6 +102,7 @@ impl SystemSettings { } // Work around version-specific issues. + #[cfg(not(php_new_zendmm_hooks))] if allocation::allocation_le83::first_rinit_should_disable_due_to_jit() { system_settings.profiling_allocation_enabled = false; } From eca51e067eb156ef1c0ba91f57e046179d2e74ec Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Thu, 21 Nov 2024 10:59:30 +0100 Subject: [PATCH 04/11] fix comments --- profiling/src/allocation/allocation_le83.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index 6a8a8b58a2..7b27485d3a 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -311,7 +311,7 @@ unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { } unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { - // Safety: `ALLOCATION_PROFILING_ALLOC` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_alloc` is also initialised let alloc = tls_zend_mm_state!(prev_custom_mm_alloc).unwrap(); @@ -336,8 +336,8 @@ unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { } unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { - // Safety: `ALLOCATION_PROFILING_FREE` will be initialised in - // `alloc_prof_free()` and only point to this function when + // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in + // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_free` is also initialised let free = tls_zend_mm_state!(prev_custom_mm_free).unwrap(); free(ptr) @@ -369,8 +369,8 @@ unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> * } unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - // Safety: `ALLOCATION_PROFILING_REALLOC` will be initialised in - // `alloc_prof_realloc()` and only point to this function when + // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_realloc` is also initialised let realloc = tls_zend_mm_state!(prev_custom_mm_realloc).unwrap(); realloc(prev_ptr, len) From 3e5d472ddec261fb086c99e6bc5e4fe708b49201 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 25 Nov 2024 14:16:55 +0100 Subject: [PATCH 05/11] Remove some `unwrap()` calls --- profiling/src/allocation/allocation_ge84.rs | 57 +++++++++++---------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 0cddf0736e..10266b8156 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -11,9 +11,9 @@ use std::sync::atomic::Ordering::SeqCst; struct ZendMMState { /// The heap we create and set as the current heap in ZendMM - heap: Option<*mut zend::zend_mm_heap>, + heap: *mut zend::zend_mm_heap, /// The heap installed in ZendMM at the time we install our custom handlers - prev_heap: Option<*mut zend::zend_mm_heap>, + prev_heap: *mut zend::zend_mm_heap, /// The engine's previous custom allocation function, if there is one. prev_custom_mm_alloc: Option, /// The engine's previous custom reallocation function, if there is one. @@ -57,8 +57,13 @@ thread_local! { /// mutated in RINIT and RSHUTDOWN. static ZEND_MM_STATE: UnsafeCell = const { UnsafeCell::new(ZendMMState { - heap: None, - prev_heap: None, + // Safety: Using `ptr::null_mut()` might seem dangerous but actually it is okay in this + // case. The `heap` and `prev_heap` fields will be initialized in the first call to + // RINIT and only used after that. By using this "trick" we can get rid of all + // `unwrap()` calls when using the `heap` or `prev_heap` field. Alternatively we could + // use `unwrap_unchecked()` for the same performance characteristics. + heap: ptr::null_mut(), + prev_heap: ptr::null_mut(), prev_custom_mm_alloc: None, prev_custom_mm_realloc: None, prev_custom_mm_free: None, @@ -88,10 +93,10 @@ pub fn alloc_prof_rinit() { // Only need to create an observed heap once per thread. When we have it, we can just // install the observed hook via `zend::zend_mm_set_heap()` - if unsafe { (*zend_mm_state).heap.is_none() } { + if unsafe { (*zend_mm_state).heap.is_null() } { // Safety: `zend_mm_get_heap()` always returns a non-null pointer to a valid heap structure let prev_heap = unsafe { zend::zend_mm_get_heap() }; - unsafe { ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(Some(prev_heap)) }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(prev_heap) }; if !is_zend_mm() { // Neighboring custom memory handlers found in the currently used ZendMM heap @@ -127,12 +132,12 @@ pub fn alloc_prof_rinit() { // Create our observed heap and prepare custom handlers let heap = unsafe { zend::zend_mm_startup() }; - unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(Some(heap)) }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(heap) }; // install our custom handler to ZendMM unsafe { zend::zend_mm_set_custom_handlers_ex( - (*zend_mm_state).heap.unwrap(), + (*zend_mm_state).heap, Some(alloc_prof_malloc), Some(alloc_prof_free), Some(alloc_prof_realloc), @@ -144,7 +149,7 @@ pub fn alloc_prof_rinit() { // install the observed heap into ZendMM unsafe { - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); } }); @@ -177,7 +182,7 @@ pub fn alloc_prof_rshutdown() { let mut custom_mm_shutdown: Option = None; // Safety: `unwrap()` is safe here, as `heap` is initialized in `RINIT` - let heap = unsafe { (*zend_mm_state).heap.unwrap() }; + let heap = unsafe { (*zend_mm_state).heap }; unsafe { zend::zend_mm_get_custom_handlers_ex( heap, @@ -209,7 +214,7 @@ pub fn alloc_prof_rshutdown() { // This is the happy path. Restore previous heap. unsafe { zend::zend_mm_set_heap( - (*zend_mm_state).prev_heap.unwrap() + (*zend_mm_state).prev_heap ); } trace!("Memory allocation profiling shutdown gracefully."); @@ -241,19 +246,19 @@ unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); // Safety: `ZEND_MM_STATE.alloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_alloc` is also initialised let ptr = ((*zend_mm_state).prev_custom_mm_alloc.unwrap())(len); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); ptr }) } unsafe fn alloc_prof_orig_alloc(len: size_t) -> *mut c_void { - let ptr: *mut c_void = zend::_zend_mm_alloc(tls_zend_mm_state!(prev_heap).unwrap(), len); + let ptr: *mut c_void = zend::_zend_mm_alloc(tls_zend_mm_state!(prev_heap), len); ptr } @@ -269,18 +274,18 @@ unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); // Safety: `ZEND_MM_STATE.free` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_free` is also initialised ((*zend_mm_state).prev_custom_mm_free.unwrap())(ptr); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); }) } unsafe fn alloc_prof_orig_free(ptr: *mut c_void) { - zend::_zend_mm_free(tls_zend_mm_state!(prev_heap).unwrap(), ptr); + zend::_zend_mm_free(tls_zend_mm_state!(prev_heap), ptr); } unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { @@ -307,19 +312,19 @@ unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_ ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); // Safety: `ZEND_MM_STATE.realloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_realloc` is also initialised let ptr = ((*zend_mm_state).prev_custom_mm_realloc.unwrap())(prev_ptr, len); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); ptr }) } unsafe fn alloc_prof_orig_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - zend::_zend_mm_realloc(tls_zend_mm_state!(prev_heap).unwrap(), prev_ptr, len) + zend::_zend_mm_realloc(tls_zend_mm_state!(prev_heap), prev_ptr, len) } unsafe extern "C" fn alloc_prof_gc() -> size_t { @@ -330,19 +335,19 @@ unsafe fn alloc_prof_prev_gc() -> size_t { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); // Safety: `ZEND_MM_STATE.gc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_gc` is also initialised let freed = ((*zend_mm_state).prev_custom_mm_gc.unwrap())(); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); freed }) } unsafe fn alloc_prof_orig_gc() -> size_t { - zend::zend_mm_gc(tls_zend_mm_state!(prev_heap).unwrap()) + zend::zend_mm_gc(tls_zend_mm_state!(prev_heap)) } unsafe extern "C" fn alloc_prof_shutdown(full: bool, silent: bool) { @@ -353,18 +358,18 @@ unsafe fn alloc_prof_prev_shutdown(full: bool, silent: bool) { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); // Safety: `ZEND_MM_STATE.shutdown` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_shutdown` is also initialised ((*zend_mm_state).prev_custom_mm_shutdown.unwrap())(full, silent); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); }) } unsafe fn alloc_prof_orig_shutdown(full: bool, silent: bool) { - zend::zend_mm_shutdown(tls_zend_mm_state!(prev_heap).unwrap(), full, silent) + zend::zend_mm_shutdown(tls_zend_mm_state!(prev_heap), full, silent) } /// safe wrapper for `zend::is_zend_mm()`. From 9aa404cb5f20149be3b257d1a0945cc6bba3021f Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 25 Nov 2024 14:35:22 +0100 Subject: [PATCH 06/11] fix comments --- profiling/src/allocation/allocation_ge84.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 10266b8156..32eac825ae 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -247,7 +247,7 @@ unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).prev_heap); - // Safety: `ZEND_MM_STATE.alloc` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_alloc` is also initialised let ptr = ((*zend_mm_state).prev_custom_mm_alloc.unwrap())(len); @@ -275,7 +275,7 @@ unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).prev_heap); - // Safety: `ZEND_MM_STATE.free` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_free` is also initialised ((*zend_mm_state).prev_custom_mm_free.unwrap())(ptr); @@ -313,7 +313,7 @@ unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_ let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).prev_heap); - // Safety: `ZEND_MM_STATE.realloc` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_realloc` is also initialised let ptr = ((*zend_mm_state).prev_custom_mm_realloc.unwrap())(prev_ptr, len); @@ -336,7 +336,7 @@ unsafe fn alloc_prof_prev_gc() -> size_t { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).prev_heap); - // Safety: `ZEND_MM_STATE.gc` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_gc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_gc` is also initialised let freed = ((*zend_mm_state).prev_custom_mm_gc.unwrap())(); @@ -359,7 +359,7 @@ unsafe fn alloc_prof_prev_shutdown(full: bool, silent: bool) { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).prev_heap); - // Safety: `ZEND_MM_STATE.shutdown` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_shutdown` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_shutdown` is also initialised ((*zend_mm_state).prev_custom_mm_shutdown.unwrap())(full, silent); From 44f29123c97c5330bfa2fb35f78bb9ea82982928 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 25 Nov 2024 17:23:29 +0100 Subject: [PATCH 07/11] These `unwrap()`'s can go unchecked :tada: --- profiling/src/allocation/allocation_ge84.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 32eac825ae..6a8dafda87 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -250,7 +250,7 @@ unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_alloc` is also initialised - let ptr = ((*zend_mm_state).prev_custom_mm_alloc.unwrap())(len); + let ptr = ((*zend_mm_state).prev_custom_mm_alloc.unwrap_unchecked())(len); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).heap); ptr @@ -278,7 +278,7 @@ unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_free` is also initialised - ((*zend_mm_state).prev_custom_mm_free.unwrap())(ptr); + ((*zend_mm_state).prev_custom_mm_free.unwrap_unchecked())(ptr); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).heap); }) @@ -316,7 +316,7 @@ unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_ // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_realloc` is also initialised - let ptr = ((*zend_mm_state).prev_custom_mm_realloc.unwrap())(prev_ptr, len); + let ptr = ((*zend_mm_state).prev_custom_mm_realloc.unwrap_unchecked())(prev_ptr, len); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).heap); ptr @@ -339,7 +339,7 @@ unsafe fn alloc_prof_prev_gc() -> size_t { // Safety: `ZEND_MM_STATE.prev_custom_mm_gc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_gc` is also initialised - let freed = ((*zend_mm_state).prev_custom_mm_gc.unwrap())(); + let freed = ((*zend_mm_state).prev_custom_mm_gc.unwrap_unchecked())(); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).heap); freed @@ -362,7 +362,7 @@ unsafe fn alloc_prof_prev_shutdown(full: bool, silent: bool) { // Safety: `ZEND_MM_STATE.prev_custom_mm_shutdown` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_shutdown` is also initialised - ((*zend_mm_state).prev_custom_mm_shutdown.unwrap())(full, silent); + ((*zend_mm_state).prev_custom_mm_shutdown.unwrap_unchecked())(full, silent); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).heap); }) From b5ff8de7e4baaa3d3f148c2e493e32d65e4ceca9 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 2 Dec 2024 16:11:46 +0100 Subject: [PATCH 08/11] cleanup --- profiling/build.rs | 2 +- profiling/src/allocation/allocation_ge84.rs | 67 +++++++++++++++++++-- profiling/src/allocation/mod.rs | 39 +++++++++--- profiling/src/bindings/mod.rs | 4 +- profiling/src/config.rs | 4 +- profiling/src/lib.rs | 15 ++++- 6 files changed, 110 insertions(+), 21 deletions(-) diff --git a/profiling/build.rs b/profiling/build.rs index ea22fd0fcb..66c6b63266 100644 --- a/profiling/build.rs +++ b/profiling/build.rs @@ -358,7 +358,7 @@ fn cfg_php_feature_flags(vernum: u64) { if vernum >= 80400 { println!("cargo:rustc-cfg=php_frameless"); println!("cargo:rustc-cfg=php_opcache_restart_hook"); - println!("cargo:rustc-cfg=php_new_zendmm_hooks"); + println!("cargo:rustc-cfg=php_zend_mm_set_custom_handlers_ex"); } } diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 6a8dafda87..c3fb11ae52 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -87,12 +87,24 @@ macro_rules! tls_zend_mm_state { }; } -pub fn alloc_prof_rinit() { +#[allow(dead_code)] +pub fn alloc_prof_minit() { + #[cfg(not(php_zts))] + alloc_prof_ginit(); +} + +#[allow(dead_code)] +pub fn alloc_prof_mshutdown() { + #[cfg(not(php_zts))] + alloc_prof_gshutdown(); +} + +pub fn alloc_prof_ginit() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Only need to create an observed heap once per thread. When we have it, we can just - // install the observed hook via `zend::zend_mm_set_heap()` + // install the observed heap via `zend::zend_mm_set_heap()` if unsafe { (*zend_mm_state).heap.is_null() } { // Safety: `zend_mm_get_heap()` always returns a non-null pointer to a valid heap structure let prev_heap = unsafe { zend::zend_mm_get_heap() }; @@ -130,7 +142,7 @@ pub fn alloc_prof_rinit() { } } - // Create our observed heap and prepare custom handlers + // Create a new (to be observed) heap and prepare custom handlers let heap = unsafe { zend::zend_mm_startup() }; unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(heap) }; @@ -145,10 +157,55 @@ pub fn alloc_prof_rinit() { Some(alloc_prof_shutdown), ); } + debug!("New observed heap created"); } + }); +} - // install the observed heap into ZendMM +pub fn alloc_prof_gshutdown() { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + unsafe { + // remove custom handlers to allow for ZendMM internal shutdown + zend::zend_mm_set_custom_handlers_ex( + (*zend_mm_state).heap, + None, + None, + None, + None, + None, + ); + + // reset to defaults + ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_orig_alloc); + ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_orig_free); + ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_orig_realloc); + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_orig_gc); + ptr::addr_of_mut!((*zend_mm_state).shutdown).write(alloc_prof_orig_shutdown); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_alloc).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_free).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_realloc).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown).write(None); + + // This shutdown will free the observed heap we created in minit + zend::zend_mm_shutdown((*zend_mm_state).heap, true, true); + + ptr::addr_of_mut!((*zend_mm_state).heap).write(ptr::null_mut()); + ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(ptr::null_mut()); + + debug!("Observed heap was freed and `zend_mm_state` reset"); + } + }); +} + +pub fn alloc_prof_rinit() { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `zend_mm_state.heap` got initialized in `MINIT` and is guaranteed to + // be a non null pointer to a valid `zend::zend_mm_heap` struct unsafe { + // Install our observed heap into ZendMM zend::zend_mm_set_heap((*zend_mm_state).heap); } }); @@ -181,7 +238,7 @@ pub fn alloc_prof_rshutdown() { let mut custom_mm_gc: Option = None; let mut custom_mm_shutdown: Option = None; - // Safety: `unwrap()` is safe here, as `heap` is initialized in `RINIT` + // Safety: `unwrap()` is safe here, as `heap` is initialized in `MINIT` let heap = unsafe { (*zend_mm_state).heap }; unsafe { zend::zend_mm_get_custom_handlers_ex( diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index f7303d112d..9980db0adf 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -8,9 +8,9 @@ use rand_distr::{Distribution, Poisson}; use std::cell::RefCell; use std::sync::atomic::AtomicU64; -#[cfg(php_new_zendmm_hooks)] +#[cfg(php_zend_mm_set_custom_handlers_ex)] mod allocation_ge84; -#[cfg(not(php_new_zendmm_hooks))] +#[cfg(not(php_zend_mm_set_custom_handlers_ex))] pub mod allocation_le83; /// take a sample every 4096 KiB @@ -78,12 +78,35 @@ thread_local! { } pub fn alloc_prof_minit() { - #[cfg(not(php_new_zendmm_hooks))] + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] allocation_le83::alloc_prof_minit(); + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_minit(); } +#[allow(dead_code)] +pub fn alloc_prof_mshutdown() { + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_mshutdown(); +} + +#[allow(dead_code)] +#[cfg(php_zts)] +pub fn alloc_prof_ginit() { + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_ginit(); +} + +#[allow(dead_code)] +#[cfg(php_zts)] +pub fn alloc_prof_gshutdown() { + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_gshutdown(); +} + +#[allow(dead_code)] pub fn alloc_prof_startup() { - #[cfg(not(php_new_zendmm_hooks))] + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] allocation_le83::alloc_prof_startup(); } @@ -105,9 +128,9 @@ pub fn alloc_prof_rinit() { return; } - #[cfg(not(php_new_zendmm_hooks))] + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] allocation_le83::alloc_prof_rinit(); - #[cfg(php_new_zendmm_hooks)] + #[cfg(php_zend_mm_set_custom_handlers_ex)] allocation_ge84::alloc_prof_rinit(); trace!("Memory allocation profiling enabled.") @@ -124,8 +147,8 @@ pub fn alloc_prof_rshutdown() { return; } - #[cfg(not(php_new_zendmm_hooks))] + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] allocation_le83::alloc_prof_rshutdown(); - #[cfg(php_new_zendmm_hooks)] + #[cfg(php_zend_mm_set_custom_handlers_ex)] allocation_ge84::alloc_prof_rshutdown(); } diff --git a/profiling/src/bindings/mod.rs b/profiling/src/bindings/mod.rs index bfb178a1af..77cd127071 100644 --- a/profiling/src/bindings/mod.rs +++ b/profiling/src/bindings/mod.rs @@ -46,9 +46,9 @@ pub type VmMmCustomAllocFn = unsafe extern "C" fn(size_t) -> *mut c_void; pub type VmMmCustomReallocFn = unsafe extern "C" fn(*mut c_void, size_t) -> *mut c_void; #[cfg(feature = "allocation_profiling")] pub type VmMmCustomFreeFn = unsafe extern "C" fn(*mut c_void); -#[cfg(all(feature = "allocation_profiling", php_new_zendmm_hooks))] +#[cfg(all(feature = "allocation_profiling", php_zend_mm_set_custom_handlers_ex))] pub type VmMmCustomGcFn = unsafe extern "C" fn() -> size_t; -#[cfg(all(feature = "allocation_profiling", php_new_zendmm_hooks))] +#[cfg(all(feature = "allocation_profiling", php_zend_mm_set_custom_handlers_ex))] pub type VmMmCustomShutdownFn = unsafe extern "C" fn(bool, bool); // todo: this a lie on some PHP versions; is it a problem even though zend_bool diff --git a/profiling/src/config.rs b/profiling/src/config.rs index b84650fbb2..c21185e838 100644 --- a/profiling/src/config.rs +++ b/profiling/src/config.rs @@ -1,4 +1,4 @@ -#[cfg(not(php_new_zendmm_hooks))] +#[cfg(not(php_zend_mm_set_custom_handlers_ex))] use crate::allocation; use crate::bindings::zai_config_type::*; use crate::bindings::{ @@ -102,7 +102,7 @@ impl SystemSettings { } // Work around version-specific issues. - #[cfg(not(php_new_zendmm_hooks))] + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] if allocation::allocation_le83::first_rinit_should_disable_due_to_jit() { system_settings.profiling_allocation_enabled = false; } diff --git a/profiling/src/lib.rs b/profiling/src/lib.rs index 3b3db288a0..e15efb26b1 100644 --- a/profiling/src/lib.rs +++ b/profiling/src/lib.rs @@ -203,12 +203,18 @@ pub extern "C" fn get_module() -> &'static mut zend::ModuleEntry { unsafe extern "C" fn ginit(_globals_ptr: *mut c_void) { #[cfg(all(feature = "timeline", php_zts))] timeline::timeline_ginit(); + + #[cfg(feature = "allocation_profiling")] + allocation::alloc_prof_ginit(); } #[cfg(php_zts)] unsafe extern "C" fn gshutdown(_globals_ptr: *mut c_void) { #[cfg(all(feature = "timeline", php_zts))] timeline::timeline_gshutdown(); + + #[cfg(feature = "allocation_profiling")] + allocation::alloc_prof_gshutdown(); } /* Important note on the PHP lifecycle: @@ -337,15 +343,15 @@ extern "C" fn minit(_type: c_int, module_number: c_int) -> ZendResult { */ unsafe { zend::zend_register_extension(&extension, handle) }; - #[cfg(feature = "allocation_profiling")] - allocation::alloc_prof_minit(); - #[cfg(feature = "timeline")] timeline::timeline_minit(); #[cfg(feature = "exception_profiling")] exception::exception_profiling_minit(); + #[cfg(feature = "allocation_profiling")] + allocation::alloc_prof_minit(); + // There are a few things which need to do something on the first rinit of // each minit/mshutdown cycle. In Apache, when doing `apachectl graceful`, // there can be more than one of these cycles per process. @@ -851,6 +857,9 @@ extern "C" fn mshutdown(_type: c_int, _module_number: c_int) -> ZendResult { #[cfg(feature = "exception_profiling")] exception::exception_profiling_mshutdown(); + #[cfg(feature = "allocation_profiling")] + allocation::alloc_prof_mshutdown(); + Profiler::stop(Duration::from_secs(1)); ZendResult::Success From 21d6bf2adf76da7bd82d577554f4a0ccc7ecb7cc Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Tue, 3 Dec 2024 16:56:16 +0100 Subject: [PATCH 09/11] cleanup --- profiling/src/allocation/allocation_ge84.rs | 40 +++++++++++++++------ 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index c3fb11ae52..b9dd34d2e6 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -51,12 +51,9 @@ struct ZendMMState { shutdown: unsafe fn(bool, bool), } -thread_local! { - /// Using an `UnsafeCell` here should be okay. There might not be any - /// synchronisation issues, as it is used in as thread local and only - /// mutated in RINIT and RSHUTDOWN. - static ZEND_MM_STATE: UnsafeCell = const { - UnsafeCell::new(ZendMMState { +impl ZendMMState { + const fn new() -> ZendMMState { + ZendMMState { // Safety: Using `ptr::null_mut()` might seem dangerous but actually it is okay in this // case. The `heap` and `prev_heap` fields will be initialized in the first call to // RINIT and only used after that. By using this "trick" we can get rid of all @@ -74,7 +71,18 @@ thread_local! { free: alloc_prof_orig_free, gc: alloc_prof_orig_gc, shutdown: alloc_prof_orig_shutdown, - }) + } + } +} + +impl ZendMMState {} + +thread_local! { + /// Using an `UnsafeCell` here should be okay. There might not be any + /// synchronisation issues, as it is used in as thread local and only + /// mutated in RINIT and RSHUTDOWN. + static ZEND_MM_STATE: UnsafeCell = const { + UnsafeCell::new(ZendMMState::new()) }; } @@ -90,16 +98,28 @@ macro_rules! tls_zend_mm_state { #[allow(dead_code)] pub fn alloc_prof_minit() { #[cfg(not(php_zts))] - alloc_prof_ginit(); + alloc_prof_hook(); } #[allow(dead_code)] pub fn alloc_prof_mshutdown() { #[cfg(not(php_zts))] - alloc_prof_gshutdown(); + alloc_prof_unhook(); } +#[allow(dead_code)] pub fn alloc_prof_ginit() { + #[cfg(php_zts)] + alloc_prof_hook(); +} + +#[allow(dead_code)] +pub fn alloc_prof_gshutdown() { + #[cfg(php_zts)] + alloc_prof_unhook(); +} + +fn alloc_prof_hook() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); @@ -162,7 +182,7 @@ pub fn alloc_prof_ginit() { }); } -pub fn alloc_prof_gshutdown() { +fn alloc_prof_unhook() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); unsafe { From 6e13b73a0c6164859a365bf8e2ff576324180f16 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Wed, 4 Dec 2024 12:48:58 +0100 Subject: [PATCH 10/11] add moar comments --- profiling/src/allocation/allocation_ge84.rs | 129 +++++++++++--------- 1 file changed, 70 insertions(+), 59 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index b9dd34d2e6..1d19bc6181 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -98,95 +98,105 @@ macro_rules! tls_zend_mm_state { #[allow(dead_code)] pub fn alloc_prof_minit() { #[cfg(not(php_zts))] - alloc_prof_hook(); + alloc_prof_custom_heap_init(); } #[allow(dead_code)] pub fn alloc_prof_mshutdown() { #[cfg(not(php_zts))] - alloc_prof_unhook(); + alloc_prof_custom_heap_reset(); } #[allow(dead_code)] pub fn alloc_prof_ginit() { #[cfg(php_zts)] - alloc_prof_hook(); + alloc_prof_custom_heap_init(); } #[allow(dead_code)] pub fn alloc_prof_gshutdown() { #[cfg(php_zts)] - alloc_prof_unhook(); + alloc_prof_custom_heap_reset(); } -fn alloc_prof_hook() { +/// This initializes the thread locale variable `ZEND_MM_STATE` with respect to the currently +/// installed `zend_mm_heap` in ZendMM. It guarantees compliance with the safety guarantees +/// described in the `ZendMMState` structure, specifically for `ZendMMState::alloc`, +/// `ZendMMState::realloc`, `ZendMMState::free`, `ZendMMState::gc` and `ZendMMState::shutdown`. +/// This function may panic if called out of order! +fn alloc_prof_custom_heap_init() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Only need to create an observed heap once per thread. When we have it, we can just // install the observed heap via `zend::zend_mm_set_heap()` - if unsafe { (*zend_mm_state).heap.is_null() } { - // Safety: `zend_mm_get_heap()` always returns a non-null pointer to a valid heap structure - let prev_heap = unsafe { zend::zend_mm_get_heap() }; - unsafe { ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(prev_heap) }; - - if !is_zend_mm() { - // Neighboring custom memory handlers found in the currently used ZendMM heap - debug!("Found another extension using the ZendMM custom handler hook"); - unsafe { - zend::zend_mm_get_custom_handlers_ex( - prev_heap, - ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_alloc), - ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_free), - ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_realloc), - ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc), - ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown), - ); - ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_prev_alloc); - ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_prev_free); - ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_prev_realloc); - // `gc` handler can be NULL - if (*zend_mm_state).prev_custom_mm_gc.is_none() { - ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_orig_gc); - } else { - ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_prev_gc); - } - // `shutdown` handler can be NULL - if (*zend_mm_state).prev_custom_mm_shutdown.is_none() { - ptr::addr_of_mut!((*zend_mm_state).shutdown) - .write(alloc_prof_orig_shutdown); - } else { - ptr::addr_of_mut!((*zend_mm_state).shutdown) - .write(alloc_prof_prev_shutdown); - } - } - } + if unsafe { !(*zend_mm_state).heap.is_null() } { + // This can only happen if either MINIT or GINIT is being called out of order. + panic!("MINIT/GINIT was called with an already initialized allocation profiler. Most likely the SAPI did this without going through MSHUTDOWN/GSHUTDOWN before."); + } - // Create a new (to be observed) heap and prepare custom handlers - let heap = unsafe { zend::zend_mm_startup() }; - unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(heap) }; + // Safety: `zend_mm_get_heap()` always returns a non-null pointer to a valid heap structure + let prev_heap = unsafe { zend::zend_mm_get_heap() }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(prev_heap) }; - // install our custom handler to ZendMM + if !is_zend_mm() { + // Neighboring custom memory handlers found in the currently used ZendMM heap + debug!("Found another extension using the ZendMM custom handler hook"); unsafe { - zend::zend_mm_set_custom_handlers_ex( - (*zend_mm_state).heap, - Some(alloc_prof_malloc), - Some(alloc_prof_free), - Some(alloc_prof_realloc), - Some(alloc_prof_gc), - Some(alloc_prof_shutdown), + zend::zend_mm_get_custom_handlers_ex( + prev_heap, + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_alloc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_free), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_realloc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown), ); + ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_prev_alloc); + ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_prev_free); + ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_prev_realloc); + // `gc` handler can be NULL + if (*zend_mm_state).prev_custom_mm_gc.is_none() { + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_orig_gc); + } else { + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_prev_gc); + } + // `shutdown` handler can be NULL + if (*zend_mm_state).prev_custom_mm_shutdown.is_none() { + ptr::addr_of_mut!((*zend_mm_state).shutdown).write(alloc_prof_orig_shutdown); + } else { + ptr::addr_of_mut!((*zend_mm_state).shutdown).write(alloc_prof_prev_shutdown); + } } - debug!("New observed heap created"); } + + // Create a new (to be observed) heap and prepare custom handlers + let heap = unsafe { zend::zend_mm_startup() }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(heap) }; + + // install our custom handler to ZendMM + unsafe { + zend::zend_mm_set_custom_handlers_ex( + (*zend_mm_state).heap, + Some(alloc_prof_malloc), + Some(alloc_prof_free), + Some(alloc_prof_realloc), + Some(alloc_prof_gc), + Some(alloc_prof_shutdown), + ); + } + debug!("New observed heap created"); }); } -fn alloc_prof_unhook() { +/// This resets the thread locale variable `ZEND_MM_STATE` and frees allocated memory. It +/// guarantees compliance with the safety guarantees described in the `ZendMMState` structure, +/// specifically for `ZendMMState::alloc`, `ZendMMState::realloc`, `ZendMMState::free`, +/// `ZendMMState::gc` and `ZendMMState::shutdown`. +fn alloc_prof_custom_heap_reset() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); unsafe { - // remove custom handlers to allow for ZendMM internal shutdown + // Remove custom handlers to allow for ZendMM internal shutdown zend::zend_mm_set_custom_handlers_ex( (*zend_mm_state).heap, None, @@ -196,7 +206,8 @@ fn alloc_prof_unhook() { None, ); - // reset to defaults + // Reset ZEND_MM_STATE to defaults, now that the pointer are not know to the observed + // heap anymore. ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_orig_alloc); ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_orig_free); ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_orig_realloc); @@ -208,14 +219,14 @@ fn alloc_prof_unhook() { ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc).write(None); ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown).write(None); - // This shutdown will free the observed heap we created in minit + // This shutdown call will free the observed heap we created in `alloc_prof_custom_heap_init` zend::zend_mm_shutdown((*zend_mm_state).heap, true, true); + // Now that the heap is gone, we need to NULL the pointer ptr::addr_of_mut!((*zend_mm_state).heap).write(ptr::null_mut()); ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(ptr::null_mut()); - - debug!("Observed heap was freed and `zend_mm_state` reset"); } + trace!("Observed heap was freed and `zend_mm_state` reset"); }); } From 9c9742c685683d0341553094ea91c09c0ddd7161 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Thu, 5 Dec 2024 11:19:54 +0100 Subject: [PATCH 11/11] cleanup --- profiling/src/allocation/allocation_ge84.rs | 28 ++------------------- profiling/src/allocation/allocation_le83.rs | 2 +- profiling/src/allocation/mod.rs | 22 +++------------- profiling/src/bindings/mod.rs | 4 +-- profiling/src/lib.rs | 16 +++--------- 5 files changed, 11 insertions(+), 61 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 1d19bc6181..6be8ba5fcc 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -95,36 +95,12 @@ macro_rules! tls_zend_mm_state { }; } -#[allow(dead_code)] -pub fn alloc_prof_minit() { - #[cfg(not(php_zts))] - alloc_prof_custom_heap_init(); -} - -#[allow(dead_code)] -pub fn alloc_prof_mshutdown() { - #[cfg(not(php_zts))] - alloc_prof_custom_heap_reset(); -} - -#[allow(dead_code)] -pub fn alloc_prof_ginit() { - #[cfg(php_zts)] - alloc_prof_custom_heap_init(); -} - -#[allow(dead_code)] -pub fn alloc_prof_gshutdown() { - #[cfg(php_zts)] - alloc_prof_custom_heap_reset(); -} - /// This initializes the thread locale variable `ZEND_MM_STATE` with respect to the currently /// installed `zend_mm_heap` in ZendMM. It guarantees compliance with the safety guarantees /// described in the `ZendMMState` structure, specifically for `ZendMMState::alloc`, /// `ZendMMState::realloc`, `ZendMMState::free`, `ZendMMState::gc` and `ZendMMState::shutdown`. /// This function may panic if called out of order! -fn alloc_prof_custom_heap_init() { +pub fn alloc_prof_ginit() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); @@ -192,7 +168,7 @@ fn alloc_prof_custom_heap_init() { /// guarantees compliance with the safety guarantees described in the `ZendMMState` structure, /// specifically for `ZendMMState::alloc`, `ZendMMState::realloc`, `ZendMMState::free`, /// `ZendMMState::gc` and `ZendMMState::shutdown`. -fn alloc_prof_custom_heap_reset() { +pub fn alloc_prof_gshutdown() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); unsafe { diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index 7b27485d3a..7f8b12617d 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -87,7 +87,7 @@ lazy_static! { static ref JIT_ENABLED: bool = unsafe { zend::ddog_php_jit_enabled() }; } -pub fn alloc_prof_minit() { +pub fn alloc_prof_ginit() { unsafe { zend::ddog_php_opcache_init_handle() }; } diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index 9980db0adf..f20e2e31be 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -77,36 +77,20 @@ thread_local! { RefCell::new(AllocationProfilingStats::new()); } -pub fn alloc_prof_minit() { - #[cfg(not(php_zend_mm_set_custom_handlers_ex))] - allocation_le83::alloc_prof_minit(); - #[cfg(php_zend_mm_set_custom_handlers_ex)] - allocation_ge84::alloc_prof_minit(); -} - -#[allow(dead_code)] -pub fn alloc_prof_mshutdown() { - #[cfg(php_zend_mm_set_custom_handlers_ex)] - allocation_ge84::alloc_prof_mshutdown(); -} - -#[allow(dead_code)] -#[cfg(php_zts)] pub fn alloc_prof_ginit() { + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] + allocation_le83::alloc_prof_ginit(); #[cfg(php_zend_mm_set_custom_handlers_ex)] allocation_ge84::alloc_prof_ginit(); } -#[allow(dead_code)] -#[cfg(php_zts)] pub fn alloc_prof_gshutdown() { #[cfg(php_zend_mm_set_custom_handlers_ex)] allocation_ge84::alloc_prof_gshutdown(); } -#[allow(dead_code)] +#[cfg(not(php_zend_mm_set_custom_handlers_ex))] pub fn alloc_prof_startup() { - #[cfg(not(php_zend_mm_set_custom_handlers_ex))] allocation_le83::alloc_prof_startup(); } diff --git a/profiling/src/bindings/mod.rs b/profiling/src/bindings/mod.rs index 77cd127071..5638dc8e91 100644 --- a/profiling/src/bindings/mod.rs +++ b/profiling/src/bindings/mod.rs @@ -179,13 +179,13 @@ pub struct ModuleEntry { /// thread for module globals. The function pointers in [`ModuleEntry::globals_ctor`] and /// [`ModuleEntry::globals_dtor`] will only be called if this is a non-zero. pub globals_size: size_t, - #[cfg(php_zts)] /// Pointer to a `ts_rsrc_id` (which is a [`i32`]). For C-Extension this is created using the /// `ZEND_DECLARE_MODULE_GLOBALS(module_name)` macro. /// See + #[cfg(php_zts)] pub globals_id_ptr: *mut ts_rsrc_id, - #[cfg(not(php_zts))] /// Pointer to the module globals struct in NTS mode + #[cfg(not(php_zts))] pub globals_ptr: *mut c_void, /// Constructor for module globals. /// Be aware this will only be called in case [`ModuleEntry::globals_size`] is non-zero and for diff --git a/profiling/src/lib.rs b/profiling/src/lib.rs index e15efb26b1..f8cf7c44be 100644 --- a/profiling/src/lib.rs +++ b/profiling/src/lib.rs @@ -31,7 +31,6 @@ use core::ptr; use ddcommon::{cstr, tag, tag::Tag}; use lazy_static::lazy_static; use libc::c_char; -#[cfg(php_zts)] use libc::c_void; use log::{debug, error, info, trace, warn}; use once_cell::sync::{Lazy, OnceCell}; @@ -184,14 +183,13 @@ pub extern "C" fn get_module() -> &'static mut zend::ModuleEntry { version: PROFILER_VERSION.as_ptr(), post_deactivate_func: Some(prshutdown), deps: DEPS.as_ptr(), - #[cfg(php_zts)] globals_ctor: Some(ginit), - #[cfg(php_zts)] globals_dtor: Some(gshutdown), - #[cfg(php_zts)] globals_size: 1, #[cfg(php_zts)] globals_id_ptr: unsafe { &mut GLOBALS_ID_PTR }, + #[cfg(not(php_zts))] + globals_ptr: ptr::null_mut(), ..Default::default() }); @@ -199,7 +197,6 @@ pub extern "C" fn get_module() -> &'static mut zend::ModuleEntry { unsafe { &mut *ptr::addr_of_mut!(MODULE) } } -#[cfg(php_zts)] unsafe extern "C" fn ginit(_globals_ptr: *mut c_void) { #[cfg(all(feature = "timeline", php_zts))] timeline::timeline_ginit(); @@ -208,7 +205,6 @@ unsafe extern "C" fn ginit(_globals_ptr: *mut c_void) { allocation::alloc_prof_ginit(); } -#[cfg(php_zts)] unsafe extern "C" fn gshutdown(_globals_ptr: *mut c_void) { #[cfg(all(feature = "timeline", php_zts))] timeline::timeline_gshutdown(); @@ -349,9 +345,6 @@ extern "C" fn minit(_type: c_int, module_number: c_int) -> ZendResult { #[cfg(feature = "exception_profiling")] exception::exception_profiling_minit(); - #[cfg(feature = "allocation_profiling")] - allocation::alloc_prof_minit(); - // There are a few things which need to do something on the first rinit of // each minit/mshutdown cycle. In Apache, when doing `apachectl graceful`, // there can be more than one of these cycles per process. @@ -857,9 +850,6 @@ extern "C" fn mshutdown(_type: c_int, _module_number: c_int) -> ZendResult { #[cfg(feature = "exception_profiling")] exception::exception_profiling_mshutdown(); - #[cfg(feature = "allocation_profiling")] - allocation::alloc_prof_mshutdown(); - Profiler::stop(Duration::from_secs(1)); ZendResult::Success @@ -884,7 +874,7 @@ extern "C" fn startup(extension: *mut ZendExtension) -> ZendResult { timeline::timeline_startup(); } - #[cfg(feature = "allocation_profiling")] + #[cfg(all(feature = "allocation_profiling", not(php_zend_mm_set_custom_handlers_ex)))] allocation::alloc_prof_startup(); ZendResult::Success