diff --git a/julia/mmtk_julia_types.h b/julia/mmtk_julia_types.h index 9e1f6018..f23f86c4 100644 --- a/julia/mmtk_julia_types.h +++ b/julia/mmtk_julia_types.h @@ -470,6 +470,8 @@ typedef struct mmtk__jl_task_t { int8_t threadpoolid; // saved gc stack top for context switches mmtk_jl_gcframe_t *gcstack; + // stack of objects (not slots) that need to be transitively pinned + mmtk_jl_gcframe_t *gcpreserve_stack; size_t world_age; // quick lookup for current ptls void* ptls; // == jl_all_tls_states[tid] diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 97a569f9..4843e16d 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -10,7 +10,7 @@ edition = "2018" [package.metadata.julia] # Our CI matches the following line and extract mmtk/julia. If this line is updated, please check ci yaml files and make sure it works. julia_repo = "https://github.com/mmtk/julia.git" -julia_version = "88fea475d9639820488f6dd50fcb08c60a011899" +julia_version = "5c9b37044fd6e446141d29111fb6c894ba0a42ff" [lib] crate-type = ["cdylib"] diff --git a/mmtk/api/mmtk.h b/mmtk/api/mmtk.h index ce629092..b45549d5 100644 --- a/mmtk/api/mmtk.h +++ b/mmtk/api/mmtk.h @@ -52,6 +52,7 @@ extern void mmtk_unreachable(void); extern unsigned char mmtk_pin_object(void* obj); extern bool mmtk_is_pinned(void* obj); + extern void mmtk_set_vm_space(void* addr, size_t size); extern void mmtk_immortal_region_post_alloc(void* addr, size_t size); diff --git a/mmtk/src/julia_scanning.rs b/mmtk/src/julia_scanning.rs index 1ec76e7c..fbb817e6 100644 --- a/mmtk/src/julia_scanning.rs +++ b/mmtk/src/julia_scanning.rs @@ -355,11 +355,68 @@ pub unsafe fn scan_julia_object>(obj: Address, clos } } +pub unsafe fn mmtk_scan_gcpreserve_stack<'a, EV: SlotVisitor>( + ta: *const mmtk_jl_task_t, + closure: &'a mut EV, +) { + // process transitively pinning stack + let mut s = (*ta).gcpreserve_stack; + let (offset, lb, ub) = (0 as isize, 0 as u64, u64::MAX); + + if s != std::ptr::null_mut() { + let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); + let mut nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); + debug_assert!(nroots.as_usize() as u32 <= UINT32_MAX); + let mut nr = nroots >> 3; + + loop { + let rts = Address::from_mut_ptr(s).shift::
(2); + let mut i = 0; + + while i < nr { + let real_addr = get_stack_addr(rts.shift::
(i as isize), offset, lb, ub); + + let slot = read_stack(rts.shift::
(i as isize), offset, lb, ub); + use crate::julia_finalizer::gc_ptr_tag; + // malloced pointer tagged in jl_gc_add_quiescent + // skip both the next element (native function), and the object + if slot & 3usize == 3 { + i += 2; + continue; + } + + // pointer is not malloced but function is native, so skip it + if gc_ptr_tag(slot, 1) { + i += 2; + continue; + } + + process_slot(closure, real_addr); + i += 1; + } + + let s_prev_address = ::std::ptr::addr_of!((*s).prev); + let sprev = read_stack(Address::from_ptr(s_prev_address), offset, lb, ub); + if sprev.is_zero() { + break; + } + + s = sprev.to_mut_ptr::(); + let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); + let new_nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); + nroots = new_nroots; + nr = nroots >> 3; + continue; + } + } +} + pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor>( ta: *const mmtk_jl_task_t, - mut closure: &'a mut EV, - mut pclosure: Option<&'a mut EV>, + closure: &'a mut EV, + pclosure: Option<&'a mut EV>, ) { + // process Julia's standard shadow (GC) stack let stkbuf = (*ta).stkbuf; let copy_stack = (*ta).copy_stack_custom(); @@ -369,7 +426,6 @@ pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor>( process_slot(closure, stkbuf_slot); } - let mut s = (*ta).gcstack; let (mut offset, mut lb, mut ub) = (0 as isize, 0 as u64, u64::MAX); #[cfg(feature = "julia_copy_stack")] @@ -383,8 +439,30 @@ pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor>( offset = (*ta).stkbuf as isize - lb as isize; } - if s != std::ptr::null_mut() { - let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); + // process Julia's gc shadow stack + scan_stack((*ta).gcstack, lb, ub, offset, closure, pclosure); + + // just call into C, since the code is cold + if (*ta).excstack != std::ptr::null_mut() { + ((*UPCALLS).scan_julia_exc_obj)( + Address::from_ptr(ta), + Address::from_mut_ptr(closure), + process_slot:: as _, + ); + } +} + +#[inline(always)] +unsafe fn scan_stack<'a, EV: SlotVisitor>( + mut stack: *mut mmtk__jl_gcframe_t, + lb: u64, + ub: u64, + offset: isize, + mut closure: &'a mut EV, + mut pclosure: Option<&'a mut EV>, +) { + if stack != std::ptr::null_mut() { + let s_nroots_addr = ::std::ptr::addr_of!((*stack).nroots); let mut nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); debug_assert!(nroots.as_usize() as u32 <= UINT32_MAX); let mut nr = nroots >> 3; @@ -402,7 +480,7 @@ pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor>( } }; - let rts = Address::from_mut_ptr(s).shift::
(2); + let rts = Address::from_mut_ptr(stack).shift::
(2); let mut i = 0; while i < nr { if (nroots.as_usize() & 1) != 0 { @@ -435,29 +513,20 @@ pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor>( i += 1; } - let s_prev_address = ::std::ptr::addr_of!((*s).prev); + let s_prev_address = ::std::ptr::addr_of!((*stack).prev); let sprev = read_stack(Address::from_ptr(s_prev_address), offset, lb, ub); if sprev.is_zero() { break; } - s = sprev.to_mut_ptr::(); - let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); + stack = sprev.to_mut_ptr::(); + let s_nroots_addr = ::std::ptr::addr_of!((*stack).nroots); let new_nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); nroots = new_nroots; nr = nroots >> 3; continue; } } - - // just call into C, since the code is cold - if (*ta).excstack != std::ptr::null_mut() { - ((*UPCALLS).scan_julia_exc_obj)( - Address::from_ptr(ta), - Address::from_mut_ptr(closure), - process_slot:: as _, - ); - } } #[inline(always)] diff --git a/mmtk/src/julia_types.rs b/mmtk/src/julia_types.rs index 9fe2515e..4bc1a80a 100644 --- a/mmtk/src/julia_types.rs +++ b/mmtk/src/julia_types.rs @@ -6478,6 +6478,7 @@ pub struct mmtk__jl_task_t { pub tid: u16, pub threadpoolid: i8, pub gcstack: *mut mmtk_jl_gcframe_t, + pub gcpreserve_stack: *mut mmtk_jl_gcframe_t, pub world_age: usize, pub ptls: *mut ::std::os::raw::c_void, pub excstack: *mut mmtk_jl_excstack_t, @@ -6497,7 +6498,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() { let ptr = UNINIT.as_ptr(); assert_eq!( ::std::mem::size_of::(), - 376usize, + 384usize, concat!("Size of: ", stringify!(mmtk__jl_task_t)) ); assert_eq!( @@ -6656,8 +6657,18 @@ fn bindgen_test_layout_mmtk__jl_task_t() { ) ); assert_eq!( - unsafe { ::std::ptr::addr_of!((*ptr).world_age) as usize - ptr as usize }, + unsafe { ::std::ptr::addr_of!((*ptr).gcpreserve_stack) as usize - ptr as usize }, 112usize, + concat!( + "Offset of field: ", + stringify!(mmtk__jl_task_t), + "::", + stringify!(gcpreserve_stack) + ) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).world_age) as usize - ptr as usize }, + 120usize, concat!( "Offset of field: ", stringify!(mmtk__jl_task_t), @@ -6667,7 +6678,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).ptls) as usize - ptr as usize }, - 120usize, + 128usize, concat!( "Offset of field: ", stringify!(mmtk__jl_task_t), @@ -6677,7 +6688,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).excstack) as usize - ptr as usize }, - 128usize, + 136usize, concat!( "Offset of field: ", stringify!(mmtk__jl_task_t), @@ -6687,7 +6698,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).eh) as usize - ptr as usize }, - 136usize, + 144usize, concat!( "Offset of field: ", stringify!(mmtk__jl_task_t), @@ -6697,7 +6708,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).ctx) as usize - ptr as usize }, - 144usize, + 152usize, concat!( "Offset of field: ", stringify!(mmtk__jl_task_t), @@ -6707,7 +6718,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).stkbuf) as usize - ptr as usize }, - 344usize, + 352usize, concat!( "Offset of field: ", stringify!(mmtk__jl_task_t), @@ -6717,7 +6728,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).bufsz) as usize - ptr as usize }, - 352usize, + 360usize, concat!( "Offset of field: ", stringify!(mmtk__jl_task_t), @@ -6727,7 +6738,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).inference_start_time) as usize - ptr as usize }, - 360usize, + 368usize, concat!( "Offset of field: ", stringify!(mmtk__jl_task_t), @@ -6737,7 +6748,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).reentrant_inference) as usize - ptr as usize }, - 368usize, + 376usize, concat!( "Offset of field: ", stringify!(mmtk__jl_task_t), @@ -6747,7 +6758,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).reentrant_timing) as usize - ptr as usize }, - 370usize, + 378usize, concat!( "Offset of field: ", stringify!(mmtk__jl_task_t), diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index b5515f58..85779022 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -58,6 +58,10 @@ impl Scanning for VMScanning { let mut root_scan_task = |task: *const mmtk__jl_task_t, task_is_root: bool| { if !task.is_null() { unsafe { + // process gc preserve stack + mmtk_scan_gcpreserve_stack(task, &mut tpinning_slot_buffer); + + // process gc stack mmtk_scan_gcstack( task, &mut tpinning_slot_buffer,