Conservative stack scanning (#203)

This PR ports #157 and
#184 to dev.

It also adds an optimization that if a task is not started, the
conservative stack scanning can be skipped for the task.

Merge with mmtk/julia#80.

---------

Co-authored-by: Yi Lin <qinsoon@gmail.com>

Author: udesou

mmtk/Cargo.toml

@@ -40,16 +40,17 @@ memoffset = "*"
 # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" }
 
 [features]
-default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning", "is_mmtk_object", "mmtk/vo_bit_access"]
+# We must build with default features
+default = ["mmtk/vm_space", "julia_copy_stack", "mmtk/object_pinning", "mmtk/is_mmtk_object", "mmtk/vo_bit_access"]
 
-# Plans
+# Default features
+julia_copy_stack = []
+
+# Plans: choose one
 nogc = []
 immix = []
 stickyimmix = ["mmtk/sticky_immix_non_moving_nursery", "mmtk/immix_smaller_block"]
 marksweep = []
-object_pinning = ["mmtk/object_pinning"]
-is_mmtk_object = ["mmtk/is_mmtk_object"]
 
 # This feature disables moving
 non_moving = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"]
-julia_copy_stack = []

mmtk/build.rs

@@ -54,6 +54,7 @@ fn main() {
         let bindings = bindgen::Builder::default()
             .header(format!("{}/src/julia.h", julia_dir))
             .header(format!("{}/src/julia_internal.h", julia_dir))
+            .header(format!("{}/src/gc-common.h", julia_dir))
             // Including the paths to depending .h files
             .clang_arg("-I")
             .clang_arg(format!("{}/mmtk/api", mmtk_dir))
@@ -77,6 +78,7 @@ fn main() {
             .allowlist_item("jl_bt_element_t")
             .allowlist_item("jl_taggedvalue_t")
             .allowlist_item("MMTkMutatorContext")
+            .allowlist_item("_bigval_t")
             // --opaque-type MMTkMutatorContext
             .opaque_type("MMTkMutatorContext")
             // compile using c++

mmtk/src/api.rs

@@ -350,7 +350,6 @@ pub extern "C" fn mmtk_set_vm_space(start: Address, size: usize) {
 
     #[cfg(feature = "stickyimmix")]
     set_side_log_bit_for_region(start, size);
-    #[cfg(feature = "is_mmtk_object")]
     set_side_vo_bit_for_region(start, size);
 }
 
@@ -383,7 +382,7 @@ pub extern "C" fn mmtk_memory_region_copy(
 pub extern "C" fn mmtk_immortal_region_post_alloc(start: Address, size: usize) {
     #[cfg(feature = "stickyimmix")]
     set_side_log_bit_for_region(start, size);
-    #[cfg(feature = "is_mmtk_object")]
+
     set_side_vo_bit_for_region(start, size);
 }
 
@@ -397,7 +396,8 @@ fn set_side_log_bit_for_region(start: Address, size: usize) {
     }
 }
 
-#[cfg(feature = "is_mmtk_object")]
+// We have to set VO bit even if this is a non_moving build. Otherwise, assertions in mmtk-core
+// will complain about seeing objects without VO bit.
 fn set_side_vo_bit_for_region(start: Address, size: usize) {
     debug!(
         "Bulk set VO bit {} to {} ({} bytes)",
@@ -485,9 +485,10 @@ pub extern "C" fn mmtk_get_obj_size(obj: ObjectReference) -> usize {
     }
 }
 
-#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))]
 #[no_mangle]
 pub extern "C" fn mmtk_pin_object(object: ObjectReference) -> bool {
+    crate::early_return_for_non_moving_build!(false);
+
     // We may in the future replace this with a check for the immix space (bound check), which should be much cheaper.
     if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) {
         memory_manager::pin_object(object)
@@ -497,9 +498,10 @@ pub extern "C" fn mmtk_pin_object(object: ObjectReference) -> bool {
     }
 }
 
-#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))]
 #[no_mangle]
 pub extern "C" fn mmtk_unpin_object(object: ObjectReference) -> bool {
+    crate::early_return_for_non_moving_build!(false);
+
     if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) {
         memory_manager::unpin_object(object)
     } else {
@@ -508,9 +510,10 @@ pub extern "C" fn mmtk_unpin_object(object: ObjectReference) -> bool {
     }
 }
 
-#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))]
 #[no_mangle]
 pub extern "C" fn mmtk_is_pinned(object: ObjectReference) -> bool {
+    crate::early_return_for_non_moving_build!(false);
+
     if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) {
         memory_manager::is_pinned(object)
     } else {
@@ -519,25 +522,6 @@ pub extern "C" fn mmtk_is_pinned(object: ObjectReference) -> bool {
     }
 }
 
-// If the `non-moving` feature is selected, pinning/unpinning is a noop and simply returns false
-#[cfg(all(feature = "object_pinning", feature = "non_moving"))]
-#[no_mangle]
-pub extern "C" fn mmtk_pin_object(_object: ObjectReference) -> bool {
-    false
-}
-
-#[cfg(all(feature = "object_pinning", feature = "non_moving"))]
-#[no_mangle]
-pub extern "C" fn mmtk_unpin_object(_object: ObjectReference) -> bool {
-    false
-}
-
-#[cfg(all(feature = "object_pinning", feature = "non_moving"))]
-#[no_mangle]
-pub extern "C" fn mmtk_is_pinned(_object: ObjectReference) -> bool {
-    false
-}
-
 #[no_mangle]
 pub extern "C" fn get_mmtk_version() -> *const c_char {
     crate::build_info::MMTK_JULIA_FULL_VERSION_STRING

mmtk/src/collection.rs

@@ -4,7 +4,7 @@ use crate::{
     jl_hrtime, jl_throw_out_of_memory_error,
 };
 use crate::{JuliaVM, USER_TRIGGERED_GC};
-use log::{info, trace};
+use log::{debug, trace};
 use mmtk::util::alloc::AllocationError;
 use mmtk::util::heap::GCTriggerPolicy;
 use mmtk::util::opaque_pointer::*;
@@ -15,6 +15,7 @@ use std::sync::atomic::{AtomicBool, AtomicIsize, AtomicU64, Ordering};
 use crate::{BLOCK_FOR_GC, STW_COND, WORLD_HAS_STOPPED};
 
 pub static GC_START: AtomicU64 = AtomicU64::new(0);
+static CURRENT_GC_MAY_MOVE: AtomicBool = AtomicBool::new(true);
 
 use std::collections::HashSet;
 use std::sync::RwLock;
@@ -52,11 +53,18 @@ impl Collection<JuliaVM> for VMCollection {
 
         trace!("Stopped the world!");
 
+        // Store if the current GC may move objects -- we will use it when the current GC finishes.
+        // We cache the value here just in case MMTk may clear it before we use the value.
+        CURRENT_GC_MAY_MOVE.store(
+            crate::SINGLETON.get_plan().current_gc_may_move_object(),
+            Ordering::SeqCst,
+        );
+
         // Tell MMTk the stacks are ready.
         {
             use mmtk::vm::ActivePlan;
             for mutator in crate::active_plan::VMActivePlan::mutators() {
-                info!("stop_all_mutators: visiting {:?}", mutator.mutator_tls);
+                debug!("stop_all_mutators: visiting {:?}", mutator.mutator_tls);
                 mutator_visitor(mutator);
             }
         }
@@ -68,6 +76,9 @@ impl Collection<JuliaVM> for VMCollection {
     }
 
     fn resume_mutators(_tls: VMWorkerThread) {
+        // unpin conservative roots
+        crate::conservative::unpin_conservative_roots();
+
         // Get the end time of the GC
         let end = unsafe { jl_hrtime() };
         trace!("gc_end = {}", end);
@@ -86,7 +97,7 @@ impl Collection<JuliaVM> for VMCollection {
         let (_, cvar) = &*STW_COND.clone();
         cvar.notify_all();
 
-        info!(
+        debug!(
             "Live bytes = {}, total bytes = {}",
             crate::api::mmtk_used_bytes(),
             crate::api::mmtk_total_bytes()
@@ -96,11 +107,11 @@ impl Collection<JuliaVM> for VMCollection {
     }
 
     fn block_for_gc(_tls: VMMutatorThread) {
-        info!("Triggered GC!");
+        debug!("Triggered GC!");
 
         unsafe { jl_gc_prepare_to_collect() };
 
-        info!("Finished blocking mutator for GC!");
+        debug!("Finished blocking mutator for GC!");
     }
 
     fn spawn_gc_thread(_tls: VMThread, ctx: GCThreadContext<JuliaVM>) {
@@ -158,14 +169,18 @@ pub fn is_current_gc_nursery() -> bool {
     }
 }
 
+pub fn is_current_gc_moving() -> bool {
+    CURRENT_GC_MAY_MOVE.load(Ordering::SeqCst)
+}
+
 #[no_mangle]
 pub extern "C" fn mmtk_block_thread_for_gc() {
     AtomicBool::store(&BLOCK_FOR_GC, true, Ordering::SeqCst);
 
     let (lock, cvar) = &*STW_COND.clone();
     let mut count = lock.lock().unwrap();
 
-    info!("Blocking for GC!");
+    debug!("Blocking for GC!");
 
     AtomicBool::store(&WORLD_HAS_STOPPED, true, Ordering::SeqCst);
 

mmtk/src/conservative.rs

@@ -0,0 +1,112 @@
+use crate::jl_task_stack_buffer;
+use crate::julia_types::*;
+use mmtk::memory_manager;
+use mmtk::util::constants::BYTES_IN_ADDRESS;
+use mmtk::util::{Address, ObjectReference};
+use std::collections::HashSet;
+use std::sync::Mutex;
+lazy_static! {
+    pub static ref CONSERVATIVE_ROOTS: Mutex<HashSet<ObjectReference>> = Mutex::new(HashSet::new());
+}
+pub fn pin_conservative_roots() {
+    crate::early_return_for_non_moving_build!(());
+    crate::early_return_for_current_gc!();
+
+    let mut roots = CONSERVATIVE_ROOTS.lock().unwrap();
+    let n_roots = roots.len();
+    roots.retain(|obj| mmtk::memory_manager::pin_object(*obj));
+    let n_pinned = roots.len();
+    log::debug!("Conservative roots: {}, pinned: {}", n_roots, n_pinned);
+}
+pub fn unpin_conservative_roots() {
+    crate::early_return_for_non_moving_build!(());
+    crate::early_return_for_current_gc!();
+
+    let mut roots = CONSERVATIVE_ROOTS.lock().unwrap();
+    let n_pinned = roots.len();
+    let mut n_live = 0;
+    roots.drain().for_each(|obj| {
+        if mmtk::memory_manager::is_live_object(obj) {
+            n_live += 1;
+            mmtk::memory_manager::unpin_object(obj);
+        }
+    });
+    log::debug!(
+        "Conservative roots: pinned: {}, unpinned/live {}",
+        n_pinned,
+        n_live
+    );
+}
+pub fn mmtk_conservative_scan_task_stack(ta: *const jl_task_t) {
+    crate::early_return_for_non_moving_build!(());
+    crate::early_return_for_current_gc!();
+
+    let mut size: u64 = 0;
+    let mut ptid: i32 = 0;
+    log::debug!("mmtk_conservative_scan_native_stack begin ta = {:?}", ta);
+    let stk = unsafe { jl_task_stack_buffer(ta, &mut size as *mut _, &mut ptid as *mut _) };
+    log::debug!(
+        "mmtk_conservative_scan_native_stack continue stk = {}, size = {}, ptid = {:x}",
+        stk,
+        size,
+        ptid
+    );
+    if !stk.is_zero() {
+        log::debug!("Conservatively scan the stack");
+        // See jl_guard_size
+        // TODO: Are we sure there are always guard pages we need to skip?
+        const JL_GUARD_PAGE: usize = 4096 * 8;
+        let guard_page_start = stk + JL_GUARD_PAGE;
+        log::debug!("Skip guard page: {}, {}", stk, guard_page_start);
+        conservative_scan_range(guard_page_start, stk + size as usize);
+    } else {
+        log::warn!("Skip stack for {:?}", ta);
+    }
+}
+pub fn mmtk_conservative_scan_task_registers(ta: *const jl_task_t) {
+    crate::early_return_for_non_moving_build!(());
+    crate::early_return_for_current_gc!();
+
+    let (lo, hi) = get_range(&unsafe { &*ta }.ctx);
+    conservative_scan_range(lo, hi);
+}
+pub fn mmtk_conservative_scan_ptls_registers(ptls: &mut _jl_tls_states_t) {
+    crate::early_return_for_non_moving_build!(());
+    crate::early_return_for_current_gc!();
+
+    let (lo, hi) = get_range(&((*ptls).gc_tls.ctx_at_the_time_gc_started));
+    conservative_scan_range(lo, hi);
+}
+// TODO: This scans the entire context type, which is slower.
+// We actually only need to scan registers.
+fn get_range<T>(ctx: &T) -> (Address, Address) {
+    let start = Address::from_ptr(ctx);
+    let ty_size = std::mem::size_of::<T>();
+    (start, start + ty_size)
+}
+fn conservative_scan_range(lo: Address, hi: Address) {
+    // The high address is exclusive
+    let hi = if hi.is_aligned_to(BYTES_IN_ADDRESS) {
+        hi - BYTES_IN_ADDRESS
+    } else {
+        hi.align_down(BYTES_IN_ADDRESS)
+    };
+    let lo = lo.align_up(BYTES_IN_ADDRESS);
+    log::trace!("Scan {} (lo) {} (hi)", lo, hi);
+    let mut cursor = hi;
+    while cursor >= lo {
+        let addr = unsafe { cursor.load::<Address>() };
+        if let Some(obj) = is_potential_mmtk_object(addr) {
+            CONSERVATIVE_ROOTS.lock().unwrap().insert(obj);
+        }
+        cursor -= BYTES_IN_ADDRESS;
+    }
+}
+fn is_potential_mmtk_object(addr: Address) -> Option<ObjectReference> {
+    if crate::object_model::is_addr_in_immixspace(addr) {
+        // We only care about immix space. If the object is in other spaces, we won't move them, and we don't need to pin them.
+        memory_manager::find_object_from_internal_pointer(addr, usize::MAX)
+    } else {
+        None
+    }
+}