Process GC preserve

julia/mmtk_julia_types.h

@@ -470,6 +470,8 @@ typedef struct mmtk__jl_task_t {
     int8_t threadpoolid;
     // saved gc stack top for context switches
     mmtk_jl_gcframe_t *gcstack;
+    // stack of objects (not slots) that need to be transitively pinned
+    mmtk_jl_gcframe_t *gcpreserve_stack;
     size_t world_age;
     // quick lookup for current ptls
     void* ptls; // == jl_all_tls_states[tid]

mmtk/Cargo.toml

@@ -10,7 +10,7 @@ edition = "2018"
 [package.metadata.julia]
 # Our CI matches the following line and extract mmtk/julia. If this line is updated, please check ci yaml files and make sure it works.
 julia_repo = "https://github.com/mmtk/julia.git"
-julia_version = "88fea475d9639820488f6dd50fcb08c60a011899"
+julia_version = "5c9b37044fd6e446141d29111fb6c894ba0a42ff"
 
 [lib]
 crate-type = ["cdylib"]

mmtk/api/mmtk.h

@@ -52,6 +52,7 @@ extern void mmtk_unreachable(void);
 extern unsigned char mmtk_pin_object(void* obj);
 extern bool mmtk_is_pinned(void* obj);
 
+
 extern void mmtk_set_vm_space(void* addr, size_t size);
 extern void mmtk_immortal_region_post_alloc(void* addr, size_t size);
 

mmtk/src/julia_scanning.rs

@@ -355,11 +355,68 @@ pub unsafe fn scan_julia_object<SV: SlotVisitor<JuliaVMSlot>>(obj: Address, clos
     }
 }
 
+pub unsafe fn mmtk_scan_gcpreserve_stack<'a, EV: SlotVisitor<JuliaVMSlot>>(
+    ta: *const mmtk_jl_task_t,
+    closure: &'a mut EV,
+) {
+    // process transitively pinning stack
+    let mut s = (*ta).gcpreserve_stack;
+    let (offset, lb, ub) = (0 as isize, 0 as u64, u64::MAX);
+
+    if s != std::ptr::null_mut() {
+        let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots);
+        let mut nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub);
+        debug_assert!(nroots.as_usize() as u32 <= UINT32_MAX);
+        let mut nr = nroots >> 3;
+
+        loop {
+            let rts = Address::from_mut_ptr(s).shift::<Address>(2);
+            let mut i = 0;
+
+            while i < nr {
+                let real_addr = get_stack_addr(rts.shift::<Address>(i as isize), offset, lb, ub);
+
+                let slot = read_stack(rts.shift::<Address>(i as isize), offset, lb, ub);
+                use crate::julia_finalizer::gc_ptr_tag;
+                // malloced pointer tagged in jl_gc_add_quiescent
+                // skip both the next element (native function), and the object
+                if slot & 3usize == 3 {
+                    i += 2;
+                    continue;
+                }
+
+                // pointer is not malloced but function is native, so skip it
+                if gc_ptr_tag(slot, 1) {
+                    i += 2;
+                    continue;
+                }
+
+                process_slot(closure, real_addr);
+                i += 1;
+            }
+
+            let s_prev_address = ::std::ptr::addr_of!((*s).prev);
+            let sprev = read_stack(Address::from_ptr(s_prev_address), offset, lb, ub);
+            if sprev.is_zero() {
+                break;
+            }
+
+            s = sprev.to_mut_ptr::<mmtk_jl_gcframe_t>();
+            let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots);
+            let new_nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub);
+            nroots = new_nroots;
+            nr = nroots >> 3;
+            continue;
+        }
+    }
+}
+
 pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor<JuliaVMSlot>>(
     ta: *const mmtk_jl_task_t,
-    mut closure: &'a mut EV,
-    mut pclosure: Option<&'a mut EV>,
+    closure: &'a mut EV,
+    pclosure: Option<&'a mut EV>,
 ) {
+    // process Julia's standard shadow (GC) stack
     let stkbuf = (*ta).stkbuf;
     let copy_stack = (*ta).copy_stack_custom();
 
@@ -369,7 +426,6 @@ pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor<JuliaVMSlot>>(
         process_slot(closure, stkbuf_slot);
     }
 
-    let mut s = (*ta).gcstack;
     let (mut offset, mut lb, mut ub) = (0 as isize, 0 as u64, u64::MAX);
 
     #[cfg(feature = "julia_copy_stack")]
@@ -383,8 +439,30 @@ pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor<JuliaVMSlot>>(
         offset = (*ta).stkbuf as isize - lb as isize;
     }
 
-    if s != std::ptr::null_mut() {
-        let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots);
+    // process Julia's gc shadow stack
+    scan_stack((*ta).gcstack, lb, ub, offset, closure, pclosure);
+
+    // just call into C, since the code is cold
+    if (*ta).excstack != std::ptr::null_mut() {
+        ((*UPCALLS).scan_julia_exc_obj)(
+            Address::from_ptr(ta),
+            Address::from_mut_ptr(closure),
+            process_slot::<EV> as _,
+        );
+    }
+}
+
+#[inline(always)]
+unsafe fn scan_stack<'a, EV: SlotVisitor<JuliaVMSlot>>(
+    mut stack: *mut mmtk__jl_gcframe_t,
+    lb: u64,
+    ub: u64,
+    offset: isize,
+    mut closure: &'a mut EV,
+    mut pclosure: Option<&'a mut EV>,
+) {
+    if stack != std::ptr::null_mut() {
+        let s_nroots_addr = ::std::ptr::addr_of!((*stack).nroots);
         let mut nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub);
         debug_assert!(nroots.as_usize() as u32 <= UINT32_MAX);
         let mut nr = nroots >> 3;
@@ -402,7 +480,7 @@ pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor<JuliaVMSlot>>(
                 }
             };
 
-            let rts = Address::from_mut_ptr(s).shift::<Address>(2);
+            let rts = Address::from_mut_ptr(stack).shift::<Address>(2);
             let mut i = 0;
             while i < nr {
                 if (nroots.as_usize() & 1) != 0 {
@@ -435,29 +513,20 @@ pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor<JuliaVMSlot>>(
                 i += 1;
             }
 
-            let s_prev_address = ::std::ptr::addr_of!((*s).prev);
+            let s_prev_address = ::std::ptr::addr_of!((*stack).prev);
             let sprev = read_stack(Address::from_ptr(s_prev_address), offset, lb, ub);
             if sprev.is_zero() {
                 break;
             }
 
-            s = sprev.to_mut_ptr::<mmtk_jl_gcframe_t>();
-            let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots);
+            stack = sprev.to_mut_ptr::<mmtk_jl_gcframe_t>();
+            let s_nroots_addr = ::std::ptr::addr_of!((*stack).nroots);
             let new_nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub);
             nroots = new_nroots;
             nr = nroots >> 3;
             continue;
         }
     }
-
-    // just call into C, since the code is cold
-    if (*ta).excstack != std::ptr::null_mut() {
-        ((*UPCALLS).scan_julia_exc_obj)(
-            Address::from_ptr(ta),
-            Address::from_mut_ptr(closure),
-            process_slot::<EV> as _,
-        );
-    }
 }
 
 #[inline(always)]

mmtk/src/julia_types.rs

@@ -6478,6 +6478,7 @@ pub struct mmtk__jl_task_t {
     pub tid: u16,
     pub threadpoolid: i8,
     pub gcstack: *mut mmtk_jl_gcframe_t,
+    pub gcpreserve_stack: *mut mmtk_jl_gcframe_t,
     pub world_age: usize,
     pub ptls: *mut ::std::os::raw::c_void,
     pub excstack: *mut mmtk_jl_excstack_t,
@@ -6497,7 +6498,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() {
     let ptr = UNINIT.as_ptr();
     assert_eq!(
         ::std::mem::size_of::<mmtk__jl_task_t>(),
-        376usize,
+        384usize,
         concat!("Size of: ", stringify!(mmtk__jl_task_t))
     );
     assert_eq!(
@@ -6656,8 +6657,18 @@ fn bindgen_test_layout_mmtk__jl_task_t() {
         )
     );
     assert_eq!(
-        unsafe { ::std::ptr::addr_of!((*ptr).world_age) as usize - ptr as usize },
+        unsafe { ::std::ptr::addr_of!((*ptr).gcpreserve_stack) as usize - ptr as usize },
         112usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(mmtk__jl_task_t),
+            "::",
+            stringify!(gcpreserve_stack)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).world_age) as usize - ptr as usize },
+        120usize,
         concat!(
             "Offset of field: ",
             stringify!(mmtk__jl_task_t),
@@ -6667,7 +6678,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() {
     );
     assert_eq!(
         unsafe { ::std::ptr::addr_of!((*ptr).ptls) as usize - ptr as usize },
-        120usize,
+        128usize,
         concat!(
             "Offset of field: ",
             stringify!(mmtk__jl_task_t),
@@ -6677,7 +6688,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() {
     );
     assert_eq!(
         unsafe { ::std::ptr::addr_of!((*ptr).excstack) as usize - ptr as usize },
-        128usize,
+        136usize,
         concat!(
             "Offset of field: ",
             stringify!(mmtk__jl_task_t),
@@ -6687,7 +6698,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() {
     );
     assert_eq!(
         unsafe { ::std::ptr::addr_of!((*ptr).eh) as usize - ptr as usize },
-        136usize,
+        144usize,
         concat!(
             "Offset of field: ",
             stringify!(mmtk__jl_task_t),
@@ -6697,7 +6708,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() {
     );
     assert_eq!(
         unsafe { ::std::ptr::addr_of!((*ptr).ctx) as usize - ptr as usize },
-        144usize,
+        152usize,
         concat!(
             "Offset of field: ",
             stringify!(mmtk__jl_task_t),
@@ -6707,7 +6718,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() {
     );
     assert_eq!(
         unsafe { ::std::ptr::addr_of!((*ptr).stkbuf) as usize - ptr as usize },
-        344usize,
+        352usize,
         concat!(
             "Offset of field: ",
             stringify!(mmtk__jl_task_t),
@@ -6717,7 +6728,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() {
     );
     assert_eq!(
         unsafe { ::std::ptr::addr_of!((*ptr).bufsz) as usize - ptr as usize },
-        352usize,
+        360usize,
         concat!(
             "Offset of field: ",
             stringify!(mmtk__jl_task_t),
@@ -6727,7 +6738,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() {
     );
     assert_eq!(
         unsafe { ::std::ptr::addr_of!((*ptr).inference_start_time) as usize - ptr as usize },
-        360usize,
+        368usize,
         concat!(
             "Offset of field: ",
             stringify!(mmtk__jl_task_t),
@@ -6737,7 +6748,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() {
     );
     assert_eq!(
         unsafe { ::std::ptr::addr_of!((*ptr).reentrant_inference) as usize - ptr as usize },
-        368usize,
+        376usize,
         concat!(
             "Offset of field: ",
             stringify!(mmtk__jl_task_t),
@@ -6747,7 +6758,7 @@ fn bindgen_test_layout_mmtk__jl_task_t() {
     );
     assert_eq!(
         unsafe { ::std::ptr::addr_of!((*ptr).reentrant_timing) as usize - ptr as usize },
-        370usize,
+        378usize,
         concat!(
             "Offset of field: ",
             stringify!(mmtk__jl_task_t),

mmtk/src/scanning.rs

@@ -58,6 +58,10 @@ impl Scanning<JuliaVM> for VMScanning {
         let mut root_scan_task = |task: *const mmtk__jl_task_t, task_is_root: bool| {
             if !task.is_null() {
                 unsafe {
+                    // process gc preserve stack
+                    mmtk_scan_gcpreserve_stack(task, &mut tpinning_slot_buffer);
+
+                    // process gc stack
                     mmtk_scan_gcstack(
                         task,
                         &mut tpinning_slot_buffer,