forked from solana-labs/solana
-
Notifications
You must be signed in to change notification settings - Fork 232
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
bpf_loader: use an explicit thread-local pool for stack and heap memory
Use a fixed thread-local pool to hold stack and heap memory. This mitigates the long standing issue of jemalloc causing TLB shootdowns to serve such frequent large allocations. Because we need 1 stack and 1 heap region per instruction, and the current max instruction nesting is hardcoded to 5, the pre-allocated size is (MAX_STACK + MAX_HEAP) * 5 * NUM_THREADS. With the current limits that's about 2.5MB per thread. Note that this is memory that would eventually get allocated anyway, we're just pre-allocating it now.
- Loading branch information
1 parent
b716121
commit 52eb624
Showing
7 changed files
with
190 additions
and
32 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| use { | ||
| solana_compute_budget::{ | ||
| compute_budget::{MAX_CALL_DEPTH, MAX_INSTRUCTION_STACK_DEPTH, STACK_FRAME_SIZE}, | ||
| compute_budget_processor::{MAX_HEAP_FRAME_BYTES, MIN_HEAP_FRAME_BYTES}, | ||
| }, | ||
| solana_rbpf::{aligned_memory::AlignedMemory, ebpf::HOST_ALIGN}, | ||
| std::array, | ||
| }; | ||
|
|
||
| trait Reset { | ||
| fn reset(&mut self); | ||
| } | ||
|
|
||
| struct Pool<T: Reset, const SIZE: usize> { | ||
| items: [Option<T>; SIZE], | ||
| next_empty: usize, | ||
| } | ||
|
|
||
| impl<T: Reset, const SIZE: usize> Pool<T, SIZE> { | ||
| fn new(items: [T; SIZE]) -> Self { | ||
| Self { | ||
| items: items.map(|i| Some(i)), | ||
| next_empty: SIZE, | ||
| } | ||
| } | ||
|
|
||
| fn len(&self) -> usize { | ||
| SIZE | ||
| } | ||
|
|
||
| fn get(&mut self) -> Option<T> { | ||
| if self.next_empty == 0 { | ||
| return None; | ||
| } | ||
| self.next_empty -= 1; | ||
| self.items[self.next_empty].take() | ||
| } | ||
|
|
||
| fn put(&mut self, mut value: T) -> bool { | ||
| if self.next_empty == self.items.len() { | ||
| return false; | ||
| } | ||
| value.reset(); | ||
| self.items[self.next_empty] = Some(value); | ||
| self.next_empty += 1; | ||
| true | ||
| } | ||
| } | ||
|
|
||
| impl Reset for AlignedMemory<{ HOST_ALIGN }> { | ||
| fn reset(&mut self) { | ||
| self.as_slice_mut().fill(0) | ||
| } | ||
| } | ||
|
|
||
| pub struct VmMemoryPool { | ||
| stack: Pool<AlignedMemory<{ HOST_ALIGN }>, MAX_INSTRUCTION_STACK_DEPTH>, | ||
| heap: Pool<AlignedMemory<{ HOST_ALIGN }>, MAX_INSTRUCTION_STACK_DEPTH>, | ||
| } | ||
|
|
||
| impl VmMemoryPool { | ||
| pub fn new() -> Self { | ||
| Self { | ||
| stack: Pool::new(array::from_fn(|_| { | ||
| AlignedMemory::zero_filled(STACK_FRAME_SIZE * MAX_CALL_DEPTH) | ||
| })), | ||
| heap: Pool::new(array::from_fn(|_| { | ||
| AlignedMemory::zero_filled(MAX_HEAP_FRAME_BYTES as usize) | ||
| })), | ||
| } | ||
| } | ||
|
|
||
| pub fn stack_len(&self) -> usize { | ||
| self.stack.len() | ||
| } | ||
|
|
||
| pub fn heap_len(&self) -> usize { | ||
| self.heap.len() | ||
| } | ||
|
|
||
| pub fn get_stack(&mut self, size: usize) -> AlignedMemory<{ HOST_ALIGN }> { | ||
| debug_assert!(size == STACK_FRAME_SIZE * MAX_CALL_DEPTH); | ||
| self.stack | ||
| .get() | ||
| .unwrap_or_else(|| AlignedMemory::zero_filled(size)) | ||
| } | ||
|
|
||
| pub fn put_stack(&mut self, stack: AlignedMemory<{ HOST_ALIGN }>) -> bool { | ||
| self.stack.put(stack) | ||
| } | ||
|
|
||
| pub fn get_heap(&mut self, heap_size: u32) -> AlignedMemory<{ HOST_ALIGN }> { | ||
| debug_assert!(heap_size >= MIN_HEAP_FRAME_BYTES && heap_size <= MAX_HEAP_FRAME_BYTES); | ||
| self.heap | ||
| .get() | ||
| .unwrap_or_else(|| AlignedMemory::zero_filled(MAX_HEAP_FRAME_BYTES as usize)) | ||
| } | ||
|
|
||
| pub fn put_heap(&mut self, heap: AlignedMemory<{ HOST_ALIGN }>) -> bool { | ||
| let heap_size = heap.len(); | ||
| debug_assert!( | ||
| heap_size >= MIN_HEAP_FRAME_BYTES as usize | ||
| && heap_size <= MAX_HEAP_FRAME_BYTES as usize | ||
| ); | ||
| self.heap.put(heap) | ||
| } | ||
| } | ||
|
|
||
| impl Default for VmMemoryPool { | ||
| fn default() -> Self { | ||
| Self::new() | ||
| } | ||
| } | ||
|
|
||
| #[cfg(test)] | ||
| mod test { | ||
| use super::*; | ||
|
|
||
| #[derive(Debug, Eq, PartialEq)] | ||
| struct Item(u8, u8); | ||
| impl Reset for Item { | ||
| fn reset(&mut self) { | ||
| self.1 = 0; | ||
| } | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_pool() { | ||
| let mut pool = Pool::<Item, 2>::new([Item(0, 1), Item(1, 1)]); | ||
| assert_eq!(pool.get(), Some(Item(1, 1))); | ||
| assert_eq!(pool.get(), Some(Item(0, 1))); | ||
| assert_eq!(pool.get(), None); | ||
| pool.put(Item(1, 1)); | ||
| assert_eq!(pool.get(), Some(Item(1, 0))); | ||
| pool.put(Item(2, 2)); | ||
| pool.put(Item(3, 3)); | ||
| assert!(!pool.put(Item(4, 4))); | ||
| assert_eq!(pool.get(), Some(Item(3, 0))); | ||
| assert_eq!(pool.get(), Some(Item(2, 0))); | ||
| assert_eq!(pool.get(), None); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters