From fded673d1a738b29184468139af64f9021cde725 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 10 Jul 2025 00:12:13 +0800 Subject: [PATCH 1/9] Add MultiLevelMapper This is a replacement for `FragmentedMapper`. It does not use hash tables and is lock-free when looking up or lazily initializing the slabs table. It is intended to eliminate the pathological case of `FragmentedMapper` where a hash collision can result in many locking operations to find the matching or vacant slab table entry. --- src/util/heap/layout/mod.rs | 4 +- src/util/heap/layout/multi_level_mapper.rs | 503 +++++++++++++++++++++ src/util/rust_util/atomic_box.rs | 48 ++ src/util/rust_util/mod.rs | 1 + 4 files changed, 555 insertions(+), 1 deletion(-) create mode 100644 src/util/heap/layout/multi_level_mapper.rs create mode 100644 src/util/rust_util/atomic_box.rs diff --git a/src/util/heap/layout/mod.rs b/src/util/heap/layout/mod.rs index c4eea74582..6611f88672 100644 --- a/src/util/heap/layout/mod.rs +++ b/src/util/heap/layout/mod.rs @@ -6,6 +6,7 @@ pub use self::mmapper::Mmapper; mod byte_map_mmapper; #[cfg(target_pointer_width = "64")] mod fragmented_mapper; +mod multi_level_mapper; mod map; pub(crate) use self::map::CreateFreeListResult; @@ -37,7 +38,8 @@ pub fn create_mmapper() -> Box { #[cfg(target_pointer_width = "64")] pub fn create_mmapper() -> Box { // TODO: ByteMapMmapper for 39-bit or less virtual space - Box::new(fragmented_mapper::FragmentedMapper::new()) + // Box::new(fragmented_mapper::FragmentedMapper::new()) + Box::new(multi_level_mapper::MultiLevelMapper::new()) } use crate::util::Address; diff --git a/src/util/heap/layout/multi_level_mapper.rs b/src/util/heap/layout/multi_level_mapper.rs new file mode 100644 index 0000000000..95cc638597 --- /dev/null +++ b/src/util/heap/layout/multi_level_mapper.rs @@ -0,0 +1,503 @@ +use super::mmapper::MapState; +use super::Mmapper; +use crate::util::constants::BYTES_IN_PAGE; +use crate::util::conversions; +use crate::util::heap::layout::vm_layout::*; +use crate::util::memory::{MmapAnnotation, MmapStrategy}; +use crate::util::rust_util::atomic_box::OnceOptionBox; +use crate::util::Address; +use atomic::{Atomic, Ordering}; +use std::fmt; +use std::io::Result; +use std::sync::Mutex; + +/// Logarithm of the address space size a user-space program is allowed to use. +/// This is the address space size of x86_64 and some other architectures, +/// although we don't usually have this much physical memory. +const LOG_MAPPABLE_BYTES: usize = 48; +/// Address space size a user-space program is allowed to use. +const MAPPABLE_BYTES: usize = 1 << LOG_MAPPABLE_BYTES; + +/// Log number of bytes per slab. +/// For a two-level array, it is advisable to choose the arithmetic mean of [`LOG_MAPPABLE_BYTES`] +/// and [`LOG_MMAP_CHUNK_BYTES`] in order to make [`MMAP_SLAB_BYTES`] the geometric mean of +/// [`MAPPABLE_BYTES`] and [`MMAP_CHUNK_BYTES`]. This will balance the array size of +/// [`MultiLevelMapper::slabs`] and [`Slab`]. +/// +/// TODO: Use `usize::midpoint` after bumping MSRV to 1.85 +const LOG_MMAP_SLAB_BYTES: usize = LOG_MMAP_CHUNK_BYTES + (LOG_MAPPABLE_BYTES - LOG_MMAP_CHUNK_BYTES) / 2; +/// Number of bytes per slab. +const MMAP_SLAB_BYTES: usize = 1 << LOG_MMAP_SLAB_BYTES; + +/// Log number of chunks per slab. +const LOG_MMAP_CHUNKS_PER_SLAB: usize = LOG_MMAP_SLAB_BYTES - LOG_MMAP_CHUNK_BYTES; +/// Number of chunks per slab. +const MMAP_CHUNKS_PER_SLAB: usize = 1 << LOG_MMAP_CHUNKS_PER_SLAB; + +/// Mask for getting in-slab bits from an address. +/// Invert this to get out-of-slab bits. +const MMAP_SLAB_MASK: usize = (1 << LOG_MMAP_SLAB_BYTES) - 1; + +/// Logarithm of maximum number of slabs, which determines the maximum mappable address space. +const LOG_MAX_SLABS: usize = LOG_MAPPABLE_BYTES - LOG_MMAP_SLAB_BYTES; +/// maximum number of slabs, which determines the maximum mappable address space. +const MAX_SLABS: usize = 1 << LOG_MAX_SLABS; + +/// The slab type. Each slab holds the `MapState` of multiple chunks. +type Slab = [Atomic; 1 << LOG_MMAP_CHUNKS_PER_SLAB]; + +/// A multi-level implementation of `Mmapper`. +pub struct MultiLevelMapper { + /// Lock for transitioning map states. + /// + /// FIXME: We only needs the lock when transitioning map states. + /// The `MultiLevelMapper` itself is completely lock-free even when allocating new slabs. + /// We should move the lock one leve above, to `MapState`. + transition_lock: Mutex<()>, + /// Slabs + slabs: Vec>, +} + +unsafe impl Send for MultiLevelMapper {} +unsafe impl Sync for MultiLevelMapper {} + +impl fmt::Debug for MultiLevelMapper { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "MultiLevelMapper({})", 1 << LOG_MAX_SLABS) + } +} + +impl Mmapper for MultiLevelMapper { + fn eagerly_mmap_all_spaces(&self, _space_map: &[Address]) {} + + fn mark_as_mapped(&self, mut start: Address, bytes: usize) { + let end = start + bytes; + // Iterate over the slabs covered + while start < end { + let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { + Self::slab_limit(start) + } else { + end + }; + let slab = Self::slab_align_down(start); + let start_chunk = Self::chunk_index(slab, start); + let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); + + let mapped = self.get_or_allocate_slab_table(start); + for entry in mapped.iter().take(end_chunk).skip(start_chunk) { + entry.store(MapState::Mapped, Ordering::Relaxed); + } + start = high; + } + } + + fn quarantine_address_range( + &self, + mut start: Address, + pages: usize, + strategy: MmapStrategy, + anno: &MmapAnnotation, + ) -> Result<()> { + debug_assert!(start.is_aligned_to(BYTES_IN_PAGE)); + + let end = start + conversions::pages_to_bytes(pages); + + // Each `MapState` entry governs a chunk. + // Align down to the chunk start because we only mmap multiples of whole chunks. + let mmap_start = conversions::mmap_chunk_align_down(start); + + // We collect the chunk states from slabs to process them in bulk. + let mut state_slices = vec![]; + + // Iterate over the slabs covered + while start < end { + let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { + Self::slab_limit(start) + } else { + end + }; + + let slab = Self::slab_align_down(start); + let start_chunk = Self::chunk_index(slab, start); + let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); + + let mapped = self.get_or_allocate_slab_table(start); + state_slices.push(&mapped[start_chunk..end_chunk]); + + start = high; + } + + #[cfg(debug_assertions)] + { + // Check if the number of entries are normal. + let mmap_end = conversions::mmap_chunk_align_up(end); + let num_slices = state_slices.iter().map(|s| s.len()).sum::(); + + debug_assert_eq!(mmap_start + BYTES_IN_CHUNK * num_slices, mmap_end); + } + + // Transition the chunks in bulk. + { + let _guard = self.transition_lock.lock().unwrap(); + MapState::bulk_transition_to_quarantined( + state_slices.as_slice(), + mmap_start, + strategy, + anno, + )?; + } + + Ok(()) + } + + fn ensure_mapped( + &self, + mut start: Address, + pages: usize, + strategy: MmapStrategy, + anno: &MmapAnnotation, + ) -> Result<()> { + let end = start + conversions::pages_to_bytes(pages); + // Iterate over the slabs covered + while start < end { + let base = Self::slab_align_down(start); + let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { + Self::slab_limit(start) + } else { + end + }; + + let slab = Self::slab_align_down(start); + let start_chunk = Self::chunk_index(slab, start); + let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); + + let mapped = self.get_or_allocate_slab_table(start); + + /* Iterate over the chunks within the slab */ + for (chunk, entry) in mapped.iter().enumerate().take(end_chunk).skip(start_chunk) { + if matches!(entry.load(Ordering::Relaxed), MapState::Mapped) { + continue; + } + + let mmap_start = Self::chunk_index_to_address(base, chunk); + let _guard = self.transition_lock.lock().unwrap(); + MapState::transition_to_mapped(entry, mmap_start, strategy, anno)?; + } + start = high; + } + Ok(()) + } + + /** + * Return {@code true} if the given address has been mmapped + * + * @param addr The address in question. + * @return {@code true} if the given address has been mmapped + */ + fn is_mapped_address(&self, addr: Address) -> bool { + let mapped = self.slab_table(addr); + match mapped { + Some(mapped) => { + mapped[Self::chunk_index(Self::slab_align_down(addr), addr)].load(Ordering::Relaxed) + == MapState::Mapped + } + _ => false, + } + } + + fn protect(&self, mut start: Address, pages: usize) { + let end = start + conversions::pages_to_bytes(pages); + let _guard = self.transition_lock.lock().unwrap(); + // Iterate over the slabs covered + while start < end { + let base = Self::slab_align_down(start); + let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { + Self::slab_limit(start) + } else { + end + }; + + let slab = Self::slab_align_down(start); + let start_chunk = Self::chunk_index(slab, start); + let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); + + let mapped = self.get_or_allocate_slab_table(start); + + for (chunk, entry) in mapped.iter().enumerate().take(end_chunk).skip(start_chunk) { + let mmap_start = Self::chunk_index_to_address(base, chunk); + MapState::transition_to_protected(entry, mmap_start).unwrap(); + } + start = high; + } + } +} + +impl MultiLevelMapper { + pub fn new() -> Self { + Self { + transition_lock: Default::default(), + slabs: unsafe { crate::util::rust_util::zeroed_alloc::new_zeroed_vec(MAX_SLABS) }, + } + } + + fn new_slab() -> Slab { + std::array::from_fn(|_| Atomic::new(MapState::Unmapped)) + } + + fn slab_table(&self, addr: Address) -> Option<&Slab> { + self.get_or_optionally_allocate_slab_table(addr, false) + } + + fn get_or_allocate_slab_table(&self, addr: Address) -> &Slab { + self.get_or_optionally_allocate_slab_table(addr, true) + .unwrap() + } + + fn get_or_optionally_allocate_slab_table( + &self, + addr: Address, + allocate: bool, + ) -> Option<&Slab> { + let index = addr >> LOG_MMAP_SLAB_BYTES; + if index > self.slabs.len() { + panic!("addr: {addr}, index: {index}, slabs.len: {sl}", sl = self.slabs.len()); + } + let slot = &self.slabs[index]; + if allocate { + slot.get_or_init(Ordering::Acquire, Ordering::Release, Self::new_slab) + } else { + slot.get(Ordering::Acquire) + } + } + + fn chunk_index_to_address(base: Address, chunk: usize) -> Address { + base + (chunk << LOG_MMAP_CHUNK_BYTES) + } + + /// Align `addr` down to slab size. + fn slab_align_down(addr: Address) -> Address { + addr.align_down(MMAP_SLAB_BYTES) + } + + /// Get the base address of the next slab after the slab that contains `addr`. + fn slab_limit(addr: Address) -> Address { + Self::slab_align_down(addr) + MMAP_SLAB_BYTES + } + + /// Return the index of the chunk that contains `addr` within the slab starting at `slab` + /// If `addr` is beyond the end of the slab, the result could be beyond the end of the slab. + fn chunk_index(slab: Address, addr: Address) -> usize { + let delta = addr - slab; + delta >> LOG_MMAP_CHUNK_BYTES + } +} + +impl Default for MultiLevelMapper { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mmap_anno_test; + use crate::util::constants::LOG_BYTES_IN_PAGE; + use crate::util::heap::layout::vm_layout::MMAP_CHUNK_BYTES; + use crate::util::memory; + use crate::util::test_util::FRAGMENTED_MMAPPER_TEST_REGION; + use crate::util::test_util::{serial_test, with_cleanup}; + use crate::util::{conversions, Address}; + + const FIXED_ADDRESS: Address = FRAGMENTED_MMAPPER_TEST_REGION.start; + const MAX_BYTES: usize = FRAGMENTED_MMAPPER_TEST_REGION.size; + + fn pages_to_chunks_up(pages: usize) -> usize { + conversions::raw_align_up(pages, MMAP_CHUNK_BYTES) / MMAP_CHUNK_BYTES + } + + fn get_chunk_map_state(mmapper: &MultiLevelMapper, chunk: Address) -> Option { + assert_eq!(conversions::mmap_chunk_align_up(chunk), chunk); + let mapped = mmapper.slab_table(chunk); + mapped.map(|m| { + m[MultiLevelMapper::chunk_index(MultiLevelMapper::slab_align_down(chunk), chunk)] + .load(Ordering::Relaxed) + }) + } + + #[test] + fn ensure_mapped_1page() { + serial_test(|| { + let pages = 1; + with_cleanup( + || { + let mmapper = MultiLevelMapper::new(); + mmapper + .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .unwrap(); + + let chunks = pages_to_chunks_up(pages); + for i in 0..chunks { + assert_eq!( + get_chunk_map_state( + &mmapper, + FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) + ), + Some(MapState::Mapped) + ); + } + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + #[test] + fn ensure_mapped_1chunk() { + serial_test(|| { + let pages = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; + with_cleanup( + || { + let mmapper = MultiLevelMapper::new(); + mmapper + .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .unwrap(); + + let chunks = pages_to_chunks_up(pages); + for i in 0..chunks { + assert_eq!( + get_chunk_map_state( + &mmapper, + FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) + ), + Some(MapState::Mapped) + ); + } + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + + #[test] + fn ensure_mapped_more_than_1chunk() { + serial_test(|| { + let pages = (MMAP_CHUNK_BYTES + MMAP_CHUNK_BYTES / 2) >> LOG_BYTES_IN_PAGE as usize; + with_cleanup( + || { + let mmapper = MultiLevelMapper::new(); + mmapper + .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .unwrap(); + + let chunks = pages_to_chunks_up(pages); + for i in 0..chunks { + assert_eq!( + get_chunk_map_state( + &mmapper, + FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) + ), + Some(MapState::Mapped) + ); + } + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + + #[test] + fn protect() { + serial_test(|| { + with_cleanup( + || { + // map 2 chunks + let mmapper = MultiLevelMapper::new(); + let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; + mmapper + .ensure_mapped( + FIXED_ADDRESS, + pages_per_chunk * 2, + MmapStrategy::TEST, + mmap_anno_test!(), + ) + .unwrap(); + + // protect 1 chunk + mmapper.protect(FIXED_ADDRESS, pages_per_chunk); + + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS), + Some(MapState::Protected) + ); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), + Some(MapState::Mapped) + ); + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + + #[test] + fn ensure_mapped_on_protected_chunks() { + serial_test(|| { + with_cleanup( + || { + // map 2 chunks + let mmapper = MultiLevelMapper::new(); + let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; + mmapper + .ensure_mapped( + FIXED_ADDRESS, + pages_per_chunk * 2, + MmapStrategy::TEST, + mmap_anno_test!(), + ) + .unwrap(); + + // protect 1 chunk + mmapper.protect(FIXED_ADDRESS, pages_per_chunk); + + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS), + Some(MapState::Protected) + ); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), + Some(MapState::Mapped) + ); + + // ensure mapped - this will unprotect the previously protected chunk + mmapper + .ensure_mapped( + FIXED_ADDRESS, + pages_per_chunk * 2, + MmapStrategy::TEST, + mmap_anno_test!(), + ) + .unwrap(); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS), + Some(MapState::Mapped) + ); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), + Some(MapState::Mapped) + ); + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } +} diff --git a/src/util/rust_util/atomic_box.rs b/src/util/rust_util/atomic_box.rs new file mode 100644 index 0000000000..20de1f3aa1 --- /dev/null +++ b/src/util/rust_util/atomic_box.rs @@ -0,0 +1,48 @@ +use std::sync::atomic::{AtomicPtr, Ordering}; + +/// A lazily initialized box. Similar to an `Option>`, but can be initialized atomically. +pub struct OnceOptionBox { + inner: AtomicPtr, +} + +impl OnceOptionBox { + pub fn new() -> OnceOptionBox { + Self { + inner: AtomicPtr::new(std::ptr::null_mut()), + } + } + + pub fn get(&self, order: Ordering) -> Option<&T> { + let ptr = self.inner.load(order); + unsafe { ptr.as_ref() } + } + + pub fn get_or_init( + &self, + order_load: Ordering, + order_store: Ordering, + init: impl FnOnce() -> T, + ) -> Option<&T> { + if let Some(get_result) = self.get(order_load) { + return Some(get_result); + } + + let new_inner = Box::into_raw(Box::new(init())); + let cas_result = self.inner.compare_exchange( + std::ptr::null_mut(), + new_inner, + order_store, + Ordering::Relaxed, + ); + match cas_result { + Ok(old_inner) => { + debug_assert_eq!(old_inner, std::ptr::null_mut()); + unsafe { new_inner.as_ref() } + } + Err(old_inner) => { + drop(unsafe { Box::from_raw(new_inner) }); + unsafe { old_inner.as_ref() } + } + } + } +} diff --git a/src/util/rust_util/mod.rs b/src/util/rust_util/mod.rs index f632fdb607..f31f68b5ec 100644 --- a/src/util/rust_util/mod.rs +++ b/src/util/rust_util/mod.rs @@ -2,6 +2,7 @@ //! functionalities that we may expect the Rust programming language and its standard libraries //! to provide. +pub mod atomic_box; pub mod rev_group; pub mod zeroed_alloc; From 96c76bd8e3bb10bd48a0c35de1ee9bd514bfc1d5 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 11 Jul 2025 14:22:13 +0800 Subject: [PATCH 2/9] Rename MultiLevelMapper to MultiLevelMmapper FragmentedMapper was FragmentedMmapper in JikesRVM. FragmentedMapper may be a misnomer introduced when porting it to Rust. --- src/util/heap/layout/mod.rs | 4 +-- ...level_mapper.rs => multi_level_mmapper.rs} | 28 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) rename src/util/heap/layout/{multi_level_mapper.rs => multi_level_mmapper.rs} (95%) diff --git a/src/util/heap/layout/mod.rs b/src/util/heap/layout/mod.rs index 6611f88672..ea43254ef5 100644 --- a/src/util/heap/layout/mod.rs +++ b/src/util/heap/layout/mod.rs @@ -6,7 +6,7 @@ pub use self::mmapper::Mmapper; mod byte_map_mmapper; #[cfg(target_pointer_width = "64")] mod fragmented_mapper; -mod multi_level_mapper; +mod multi_level_mmapper; mod map; pub(crate) use self::map::CreateFreeListResult; @@ -39,7 +39,7 @@ pub fn create_mmapper() -> Box { pub fn create_mmapper() -> Box { // TODO: ByteMapMmapper for 39-bit or less virtual space // Box::new(fragmented_mapper::FragmentedMapper::new()) - Box::new(multi_level_mapper::MultiLevelMapper::new()) + Box::new(multi_level_mmapper::MultiLevelMmapper::new()) } use crate::util::Address; diff --git a/src/util/heap/layout/multi_level_mapper.rs b/src/util/heap/layout/multi_level_mmapper.rs similarity index 95% rename from src/util/heap/layout/multi_level_mapper.rs rename to src/util/heap/layout/multi_level_mmapper.rs index 95cc638597..84fbc687b4 100644 --- a/src/util/heap/layout/multi_level_mapper.rs +++ b/src/util/heap/layout/multi_level_mmapper.rs @@ -47,7 +47,7 @@ const MAX_SLABS: usize = 1 << LOG_MAX_SLABS; type Slab = [Atomic; 1 << LOG_MMAP_CHUNKS_PER_SLAB]; /// A multi-level implementation of `Mmapper`. -pub struct MultiLevelMapper { +pub struct MultiLevelMmapper { /// Lock for transitioning map states. /// /// FIXME: We only needs the lock when transitioning map states. @@ -58,16 +58,16 @@ pub struct MultiLevelMapper { slabs: Vec>, } -unsafe impl Send for MultiLevelMapper {} -unsafe impl Sync for MultiLevelMapper {} +unsafe impl Send for MultiLevelMmapper {} +unsafe impl Sync for MultiLevelMmapper {} -impl fmt::Debug for MultiLevelMapper { +impl fmt::Debug for MultiLevelMmapper { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "MultiLevelMapper({})", 1 << LOG_MAX_SLABS) } } -impl Mmapper for MultiLevelMapper { +impl Mmapper for MultiLevelMmapper { fn eagerly_mmap_all_spaces(&self, _space_map: &[Address]) {} fn mark_as_mapped(&self, mut start: Address, bytes: usize) { @@ -232,7 +232,7 @@ impl Mmapper for MultiLevelMapper { } } -impl MultiLevelMapper { +impl MultiLevelMmapper { pub fn new() -> Self { Self { transition_lock: Default::default(), @@ -292,7 +292,7 @@ impl MultiLevelMapper { } } -impl Default for MultiLevelMapper { +impl Default for MultiLevelMmapper { fn default() -> Self { Self::new() } @@ -316,11 +316,11 @@ mod tests { conversions::raw_align_up(pages, MMAP_CHUNK_BYTES) / MMAP_CHUNK_BYTES } - fn get_chunk_map_state(mmapper: &MultiLevelMapper, chunk: Address) -> Option { + fn get_chunk_map_state(mmapper: &MultiLevelMmapper, chunk: Address) -> Option { assert_eq!(conversions::mmap_chunk_align_up(chunk), chunk); let mapped = mmapper.slab_table(chunk); mapped.map(|m| { - m[MultiLevelMapper::chunk_index(MultiLevelMapper::slab_align_down(chunk), chunk)] + m[MultiLevelMmapper::chunk_index(MultiLevelMmapper::slab_align_down(chunk), chunk)] .load(Ordering::Relaxed) }) } @@ -331,7 +331,7 @@ mod tests { let pages = 1; with_cleanup( || { - let mmapper = MultiLevelMapper::new(); + let mmapper = MultiLevelMmapper::new(); mmapper .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) .unwrap(); @@ -359,7 +359,7 @@ mod tests { let pages = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; with_cleanup( || { - let mmapper = MultiLevelMapper::new(); + let mmapper = MultiLevelMmapper::new(); mmapper .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) .unwrap(); @@ -388,7 +388,7 @@ mod tests { let pages = (MMAP_CHUNK_BYTES + MMAP_CHUNK_BYTES / 2) >> LOG_BYTES_IN_PAGE as usize; with_cleanup( || { - let mmapper = MultiLevelMapper::new(); + let mmapper = MultiLevelMmapper::new(); mmapper .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) .unwrap(); @@ -417,7 +417,7 @@ mod tests { with_cleanup( || { // map 2 chunks - let mmapper = MultiLevelMapper::new(); + let mmapper = MultiLevelMmapper::new(); let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; mmapper .ensure_mapped( @@ -453,7 +453,7 @@ mod tests { with_cleanup( || { // map 2 chunks - let mmapper = MultiLevelMapper::new(); + let mmapper = MultiLevelMmapper::new(); let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; mmapper .ensure_mapped( From e54a28ce0cc376ba37ff6d6d2f6a03cb148f123d Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 11 Jul 2025 14:24:12 +0800 Subject: [PATCH 3/9] Completely remove FragmentedMapper The hash table design should have been a mistake. Now that we are going to replace it, we replace it completely. --- src/util/heap/layout/fragmented_mapper.rs | 620 ---------------------- src/util/heap/layout/mod.rs | 3 - 2 files changed, 623 deletions(-) delete mode 100644 src/util/heap/layout/fragmented_mapper.rs diff --git a/src/util/heap/layout/fragmented_mapper.rs b/src/util/heap/layout/fragmented_mapper.rs deleted file mode 100644 index 0ebec07b83..0000000000 --- a/src/util/heap/layout/fragmented_mapper.rs +++ /dev/null @@ -1,620 +0,0 @@ -use super::mmapper::MapState; -use super::Mmapper; -use crate::util::constants::BYTES_IN_PAGE; -use crate::util::conversions; -use crate::util::heap::layout::vm_layout::*; -use crate::util::memory::{MmapAnnotation, MmapStrategy}; -use crate::util::Address; -use atomic::{Atomic, Ordering}; -use std::cell::UnsafeCell; -use std::fmt; -use std::io::Result; -use std::sync::Mutex; - -const MMAP_NUM_CHUNKS: usize = 1 << (33 - LOG_MMAP_CHUNK_BYTES); - -// 36 = 128G - physical memory larger than this is uncommon -// 40 = 2T. Increased to 2T. Though we probably won't use this much memory, we allow quarantine memory range, -// and that is usually used to quarantine a large amount of memory. -const LOG_MAPPABLE_BYTES: usize = 40; - -/* - * Size of a slab. The value 10 gives a slab size of 1GB, with 1024 - * chunks per slab, ie a 1k slab map. In a 64-bit address space, this - * will require 1M of slab maps. - */ -const LOG_MMAP_CHUNKS_PER_SLAB: usize = 8; -const LOG_MMAP_SLAB_BYTES: usize = LOG_MMAP_CHUNKS_PER_SLAB + LOG_MMAP_CHUNK_BYTES; -const MMAP_SLAB_EXTENT: usize = 1 << LOG_MMAP_SLAB_BYTES; -const MMAP_SLAB_MASK: usize = (1 << LOG_MMAP_SLAB_BYTES) - 1; -/** - * Maximum number of slabs, which determines the maximum mappable address space. - */ -const LOG_MAX_SLABS: usize = LOG_MAPPABLE_BYTES - LOG_MMAP_CHUNK_BYTES - LOG_MMAP_CHUNKS_PER_SLAB; -const MAX_SLABS: usize = 1 << LOG_MAX_SLABS; -/** - * Parameters for the slab table. The hash function requires it to be - * a power of 2. Must be larger than MAX_SLABS for hashing to work, - * and should be much larger for it to be efficient. - */ -const LOG_SLAB_TABLE_SIZE: usize = 1 + LOG_MAX_SLABS; -const HASH_MASK: usize = (1 << LOG_SLAB_TABLE_SIZE) - 1; -const SLAB_TABLE_SIZE: usize = 1 << LOG_SLAB_TABLE_SIZE; -const SENTINEL: Address = Address::MAX; - -type Slab = [Atomic; MMAP_NUM_CHUNKS]; - -pub struct FragmentedMapper { - lock: Mutex<()>, - inner: UnsafeCell, -} - -unsafe impl Send for FragmentedMapper {} -unsafe impl Sync for FragmentedMapper {} - -struct InnerFragmentedMapper { - free_slab_index: usize, - free_slabs: Vec>>, - slab_table: Vec>>, - slab_map: Vec
, -} - -impl fmt::Debug for FragmentedMapper { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "FragmentedMapper({})", MMAP_NUM_CHUNKS) - } -} - -impl Mmapper for FragmentedMapper { - fn eagerly_mmap_all_spaces(&self, _space_map: &[Address]) {} - - fn mark_as_mapped(&self, mut start: Address, bytes: usize) { - let end = start + bytes; - // Iterate over the slabs covered - while start < end { - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - for entry in mapped.iter().take(end_chunk).skip(start_chunk) { - entry.store(MapState::Mapped, Ordering::Relaxed); - } - start = high; - } - } - - fn quarantine_address_range( - &self, - mut start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - debug_assert!(start.is_aligned_to(BYTES_IN_PAGE)); - - let end = start + conversions::pages_to_bytes(pages); - - // Each `MapState` entry governs a chunk. - // Align down to the chunk start because we only mmap multiples of whole chunks. - let mmap_start = conversions::mmap_chunk_align_down(start); - - // We collect the chunk states from slabs to process them in bulk. - let mut state_slices = vec![]; - - // Iterate over the slabs covered - while start < end { - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - state_slices.push(&mapped[start_chunk..end_chunk]); - - start = high; - } - - #[cfg(debug_assertions)] - { - // Check if the number of entries are normal. - let mmap_end = conversions::mmap_chunk_align_up(end); - let num_slices = state_slices.iter().map(|s| s.len()).sum::(); - - debug_assert_eq!(mmap_start + BYTES_IN_CHUNK * num_slices, mmap_end); - } - - // Transition the chunks in bulk. - { - let _guard = self.lock.lock().unwrap(); - MapState::bulk_transition_to_quarantined( - state_slices.as_slice(), - mmap_start, - strategy, - anno, - )?; - } - - Ok(()) - } - - fn ensure_mapped( - &self, - mut start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - let end = start + conversions::pages_to_bytes(pages); - // Iterate over the slabs covered - while start < end { - let base = Self::slab_align_down(start); - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - - /* Iterate over the chunks within the slab */ - for (chunk, entry) in mapped.iter().enumerate().take(end_chunk).skip(start_chunk) { - if matches!(entry.load(Ordering::Relaxed), MapState::Mapped) { - continue; - } - - let mmap_start = Self::chunk_index_to_address(base, chunk); - let _guard = self.lock.lock().unwrap(); - MapState::transition_to_mapped(entry, mmap_start, strategy, anno)?; - } - start = high; - } - Ok(()) - } - - /** - * Return {@code true} if the given address has been mmapped - * - * @param addr The address in question. - * @return {@code true} if the given address has been mmapped - */ - fn is_mapped_address(&self, addr: Address) -> bool { - let mapped = self.slab_table(addr); - match mapped { - Some(mapped) => { - mapped[Self::chunk_index(Self::slab_align_down(addr), addr)].load(Ordering::Relaxed) - == MapState::Mapped - } - _ => false, - } - } - - fn protect(&self, mut start: Address, pages: usize) { - let end = start + conversions::pages_to_bytes(pages); - let _guard = self.lock.lock().unwrap(); - // Iterate over the slabs covered - while start < end { - let base = Self::slab_align_down(start); - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - - for (chunk, entry) in mapped.iter().enumerate().take(end_chunk).skip(start_chunk) { - let mmap_start = Self::chunk_index_to_address(base, chunk); - MapState::transition_to_protected(entry, mmap_start).unwrap(); - } - start = high; - } - } -} - -impl FragmentedMapper { - pub fn new() -> Self { - Self { - lock: Mutex::new(()), - inner: UnsafeCell::new(InnerFragmentedMapper { - free_slab_index: 0, - free_slabs: (0..MAX_SLABS).map(|_| Some(Self::new_slab())).collect(), - slab_table: (0..SLAB_TABLE_SIZE).map(|_| None).collect(), - slab_map: vec![SENTINEL; SLAB_TABLE_SIZE], - }), - } - } - - fn new_slab() -> Box { - // Because AtomicU8 does not implement Copy, it is a compilation error to usen the - // expression `[Atomic::new(MapState::Unmapped); MMAP_NUM_CHUNKS]` because that involves - // copying. We must define a constant for it. - // - // TODO: Use the inline const expression `const { Atomic::new(MapState::Unmapped) }` after - // we bump MSRV to 1.79. - - // If we declare a const Atomic, Clippy will warn about const items being interior mutable. - // Using inline const expression will eliminate this warning, but that is experimental until - // 1.79. Fix it after we bump MSRV. - #[allow(clippy::declare_interior_mutable_const)] - const INITIAL_ENTRY: Atomic = Atomic::new(MapState::Unmapped); - - let mapped: Box = Box::new([INITIAL_ENTRY; MMAP_NUM_CHUNKS]); - mapped - } - - fn hash(addr: Address) -> usize { - let mut initial = (addr & !MMAP_SLAB_MASK) >> LOG_MMAP_SLAB_BYTES; - let mut hash = 0; - while initial != 0 { - hash ^= initial & HASH_MASK; - initial >>= LOG_SLAB_TABLE_SIZE; - } - hash - } - - fn slab_table(&self, addr: Address) -> Option<&Slab> { - self.get_or_optionally_allocate_slab_table(addr, false) - } - - fn get_or_allocate_slab_table(&self, addr: Address) -> &Slab { - self.get_or_optionally_allocate_slab_table(addr, true) - .unwrap() - } - - fn inner(&self) -> &InnerFragmentedMapper { - unsafe { &*self.inner.get() } - } - #[allow(clippy::mut_from_ref)] - fn inner_mut(&self) -> &mut InnerFragmentedMapper { - unsafe { &mut *self.inner.get() } - } - - fn get_or_optionally_allocate_slab_table( - &self, - addr: Address, - allocate: bool, - ) -> Option<&Slab> { - debug_assert!(addr != SENTINEL); - let base = unsafe { Address::from_usize(addr & !MMAP_SLAB_MASK) }; - let hash = Self::hash(base); - let mut index = hash; // Use 'index' to iterate over the hash table so that we remember where we started - loop { - /* Check for a hash-table hit. Should be the frequent case. */ - if base == self.inner().slab_map[index] { - return self.slab_table_for(addr, index); - } - let _guard = self.lock.lock().unwrap(); - - /* Check whether another thread has allocated a slab while we were acquiring the lock */ - if base == self.inner().slab_map[index] { - // drop(guard); - return self.slab_table_for(addr, index); - } - - /* Check for a free slot */ - if self.inner().slab_map[index] == SENTINEL { - if !allocate { - // drop(guard); - return None; - } - unsafe { - self.commit_free_slab(index); - } - self.inner_mut().slab_map[index] = base; - return self.slab_table_for(addr, index); - } - // lock.release(); - index += 1; - index %= SLAB_TABLE_SIZE; - assert!(index != hash, "MMAP slab table is full!"); - } - } - - fn slab_table_for(&self, _addr: Address, index: usize) -> Option<&Slab> { - debug_assert!(self.inner().slab_table[index].is_some()); - self.inner().slab_table[index].as_ref().map(|x| x as &Slab) - } - - /** - * Take a free slab of chunks from the freeSlabs array, and insert it - * at the correct index in the slabTable. - * @param index slab table index - */ - /// # Safety - /// - /// Caller must ensure that only one thread is calling this function at a time. - unsafe fn commit_free_slab(&self, index: usize) { - assert!( - self.inner().free_slab_index < MAX_SLABS, - "All free slabs used: virtual address space is exhausled." - ); - debug_assert!(self.inner().slab_table[index].is_none()); - debug_assert!(self.inner().free_slabs[self.inner().free_slab_index].is_some()); - ::std::mem::swap( - &mut self.inner_mut().slab_table[index], - &mut self.inner_mut().free_slabs[self.inner().free_slab_index], - ); - self.inner_mut().free_slab_index += 1; - } - - fn chunk_index_to_address(base: Address, chunk: usize) -> Address { - base + (chunk << LOG_MMAP_CHUNK_BYTES) - } - - /** - * @param addr an address - * @return the base address of the enclosing slab - */ - fn slab_align_down(addr: Address) -> Address { - unsafe { Address::from_usize(addr & !MMAP_SLAB_MASK) } - } - - /** - * @param addr an address - * @return the base address of the next slab - */ - fn slab_limit(addr: Address) -> Address { - Self::slab_align_down(addr) + MMAP_SLAB_EXTENT - } - - /** - * @param slab Address of the slab - * @param addr Address within a chunk (could be in the next slab) - * @return The index of the chunk within the slab (could be beyond the end of the slab) - */ - fn chunk_index(slab: Address, addr: Address) -> usize { - let delta = addr - slab; - delta >> LOG_MMAP_CHUNK_BYTES - } -} - -impl Default for FragmentedMapper { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::mmap_anno_test; - use crate::util::constants::LOG_BYTES_IN_PAGE; - use crate::util::heap::layout::vm_layout::MMAP_CHUNK_BYTES; - use crate::util::memory; - use crate::util::test_util::FRAGMENTED_MMAPPER_TEST_REGION; - use crate::util::test_util::{serial_test, with_cleanup}; - use crate::util::{conversions, Address}; - - const FIXED_ADDRESS: Address = FRAGMENTED_MMAPPER_TEST_REGION.start; - const MAX_BYTES: usize = FRAGMENTED_MMAPPER_TEST_REGION.size; - - fn pages_to_chunks_up(pages: usize) -> usize { - conversions::raw_align_up(pages, MMAP_CHUNK_BYTES) / MMAP_CHUNK_BYTES - } - - fn get_chunk_map_state(mmapper: &FragmentedMapper, chunk: Address) -> Option { - assert_eq!(conversions::mmap_chunk_align_up(chunk), chunk); - let mapped = mmapper.slab_table(chunk); - mapped.map(|m| { - m[FragmentedMapper::chunk_index(FragmentedMapper::slab_align_down(chunk), chunk)] - .load(Ordering::Relaxed) - }) - } - - #[test] - fn address_hashing() { - for i in 0..10 { - unsafe { - let a = i << LOG_MMAP_SLAB_BYTES; - assert_eq!(FragmentedMapper::hash(Address::from_usize(a)), i); - - let b = a + ((i + 1) << (LOG_MMAP_SLAB_BYTES + LOG_SLAB_TABLE_SIZE + 1)); - assert_eq!( - FragmentedMapper::hash(Address::from_usize(b)), - i ^ ((i + 1) << 1) - ); - - let c = b + ((i + 2) << (LOG_MMAP_SLAB_BYTES + LOG_SLAB_TABLE_SIZE * 2 + 2)); - assert_eq!( - FragmentedMapper::hash(Address::from_usize(c)), - i ^ ((i + 1) << 1) ^ ((i + 2) << 2) - ); - } - } - } - - #[test] - fn ensure_mapped_1page() { - serial_test(|| { - let pages = 1; - with_cleanup( - || { - let mmapper = FragmentedMapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - let chunks = pages_to_chunks_up(pages); - for i in 0..chunks { - assert_eq!( - get_chunk_map_state( - &mmapper, - FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) - ), - Some(MapState::Mapped) - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - #[test] - fn ensure_mapped_1chunk() { - serial_test(|| { - let pages = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; - with_cleanup( - || { - let mmapper = FragmentedMapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - let chunks = pages_to_chunks_up(pages); - for i in 0..chunks { - assert_eq!( - get_chunk_map_state( - &mmapper, - FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) - ), - Some(MapState::Mapped) - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_more_than_1chunk() { - serial_test(|| { - let pages = (MMAP_CHUNK_BYTES + MMAP_CHUNK_BYTES / 2) >> LOG_BYTES_IN_PAGE as usize; - with_cleanup( - || { - let mmapper = FragmentedMapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - let chunks = pages_to_chunks_up(pages); - for i in 0..chunks { - assert_eq!( - get_chunk_map_state( - &mmapper, - FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) - ), - Some(MapState::Mapped) - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - - #[test] - fn protect() { - serial_test(|| { - with_cleanup( - || { - // map 2 chunks - let mmapper = FragmentedMapper::new(); - let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; - mmapper - .ensure_mapped( - FIXED_ADDRESS, - pages_per_chunk * 2, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - - // protect 1 chunk - mmapper.protect(FIXED_ADDRESS, pages_per_chunk); - - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS), - Some(MapState::Protected) - ); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), - Some(MapState::Mapped) - ); - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_on_protected_chunks() { - serial_test(|| { - with_cleanup( - || { - // map 2 chunks - let mmapper = FragmentedMapper::new(); - let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; - mmapper - .ensure_mapped( - FIXED_ADDRESS, - pages_per_chunk * 2, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - - // protect 1 chunk - mmapper.protect(FIXED_ADDRESS, pages_per_chunk); - - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS), - Some(MapState::Protected) - ); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), - Some(MapState::Mapped) - ); - - // ensure mapped - this will unprotect the previously protected chunk - mmapper - .ensure_mapped( - FIXED_ADDRESS, - pages_per_chunk * 2, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS), - Some(MapState::Mapped) - ); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), - Some(MapState::Mapped) - ); - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } -} diff --git a/src/util/heap/layout/mod.rs b/src/util/heap/layout/mod.rs index ea43254ef5..468d30ef68 100644 --- a/src/util/heap/layout/mod.rs +++ b/src/util/heap/layout/mod.rs @@ -4,8 +4,6 @@ pub mod vm_layout; mod mmapper; pub use self::mmapper::Mmapper; mod byte_map_mmapper; -#[cfg(target_pointer_width = "64")] -mod fragmented_mapper; mod multi_level_mmapper; mod map; @@ -38,7 +36,6 @@ pub fn create_mmapper() -> Box { #[cfg(target_pointer_width = "64")] pub fn create_mmapper() -> Box { // TODO: ByteMapMmapper for 39-bit or less virtual space - // Box::new(fragmented_mapper::FragmentedMapper::new()) Box::new(multi_level_mmapper::MultiLevelMmapper::new()) } From a212a6f6e021b67e553d00b86392f2b0e1889ef9 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 11 Jul 2025 14:34:07 +0800 Subject: [PATCH 4/9] Adjust comment x86_64 actualy has 47 bit address space for the user. --- src/util/heap/layout/multi_level_mmapper.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/heap/layout/multi_level_mmapper.rs b/src/util/heap/layout/multi_level_mmapper.rs index 84fbc687b4..5570d16faf 100644 --- a/src/util/heap/layout/multi_level_mmapper.rs +++ b/src/util/heap/layout/multi_level_mmapper.rs @@ -12,8 +12,8 @@ use std::io::Result; use std::sync::Mutex; /// Logarithm of the address space size a user-space program is allowed to use. -/// This is the address space size of x86_64 and some other architectures, -/// although we don't usually have this much physical memory. +/// This is enough for ARM64, x86_64 and some other architectures. +/// Feel free to increase it if we plan to support larger address spaces. const LOG_MAPPABLE_BYTES: usize = 48; /// Address space size a user-space program is allowed to use. const MAPPABLE_BYTES: usize = 1 << LOG_MAPPABLE_BYTES; From 461d5f485b96e0615e02492543ac2108b22ec51d Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 11 Jul 2025 14:35:56 +0800 Subject: [PATCH 5/9] Rename it again to TwoLevelMmapper The current implememntation only has two levels. We leave the name "MultiLevelMmapper" to a true multi-level implementation in the future. --- src/util/heap/layout/mod.rs | 4 +-- ..._level_mmapper.rs => two_level_mmapper.rs} | 36 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) rename src/util/heap/layout/{multi_level_mmapper.rs => two_level_mmapper.rs} (94%) diff --git a/src/util/heap/layout/mod.rs b/src/util/heap/layout/mod.rs index 468d30ef68..ab53492cc3 100644 --- a/src/util/heap/layout/mod.rs +++ b/src/util/heap/layout/mod.rs @@ -4,7 +4,7 @@ pub mod vm_layout; mod mmapper; pub use self::mmapper::Mmapper; mod byte_map_mmapper; -mod multi_level_mmapper; +mod two_level_mmapper; mod map; pub(crate) use self::map::CreateFreeListResult; @@ -36,7 +36,7 @@ pub fn create_mmapper() -> Box { #[cfg(target_pointer_width = "64")] pub fn create_mmapper() -> Box { // TODO: ByteMapMmapper for 39-bit or less virtual space - Box::new(multi_level_mmapper::MultiLevelMmapper::new()) + Box::new(two_level_mmapper::TwoLevelMmapper::new()) } use crate::util::Address; diff --git a/src/util/heap/layout/multi_level_mmapper.rs b/src/util/heap/layout/two_level_mmapper.rs similarity index 94% rename from src/util/heap/layout/multi_level_mmapper.rs rename to src/util/heap/layout/two_level_mmapper.rs index 5570d16faf..61ca683ad0 100644 --- a/src/util/heap/layout/multi_level_mmapper.rs +++ b/src/util/heap/layout/two_level_mmapper.rs @@ -22,7 +22,7 @@ const MAPPABLE_BYTES: usize = 1 << LOG_MAPPABLE_BYTES; /// For a two-level array, it is advisable to choose the arithmetic mean of [`LOG_MAPPABLE_BYTES`] /// and [`LOG_MMAP_CHUNK_BYTES`] in order to make [`MMAP_SLAB_BYTES`] the geometric mean of /// [`MAPPABLE_BYTES`] and [`MMAP_CHUNK_BYTES`]. This will balance the array size of -/// [`MultiLevelMapper::slabs`] and [`Slab`]. +/// [`TwoLevelMmapper::slabs`] and [`Slab`]. /// /// TODO: Use `usize::midpoint` after bumping MSRV to 1.85 const LOG_MMAP_SLAB_BYTES: usize = LOG_MMAP_CHUNK_BYTES + (LOG_MAPPABLE_BYTES - LOG_MMAP_CHUNK_BYTES) / 2; @@ -46,28 +46,28 @@ const MAX_SLABS: usize = 1 << LOG_MAX_SLABS; /// The slab type. Each slab holds the `MapState` of multiple chunks. type Slab = [Atomic; 1 << LOG_MMAP_CHUNKS_PER_SLAB]; -/// A multi-level implementation of `Mmapper`. -pub struct MultiLevelMmapper { +/// A two-level implementation of `Mmapper`. +pub struct TwoLevelMmapper { /// Lock for transitioning map states. /// /// FIXME: We only needs the lock when transitioning map states. - /// The `MultiLevelMapper` itself is completely lock-free even when allocating new slabs. + /// The `TwoLevelMmapper` itself is completely lock-free even when allocating new slabs. /// We should move the lock one leve above, to `MapState`. transition_lock: Mutex<()>, /// Slabs slabs: Vec>, } -unsafe impl Send for MultiLevelMmapper {} -unsafe impl Sync for MultiLevelMmapper {} +unsafe impl Send for TwoLevelMmapper {} +unsafe impl Sync for TwoLevelMmapper {} -impl fmt::Debug for MultiLevelMmapper { +impl fmt::Debug for TwoLevelMmapper { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "MultiLevelMapper({})", 1 << LOG_MAX_SLABS) + write!(f, "TwoLevelMapper({})", 1 << LOG_MAX_SLABS) } } -impl Mmapper for MultiLevelMmapper { +impl Mmapper for TwoLevelMmapper { fn eagerly_mmap_all_spaces(&self, _space_map: &[Address]) {} fn mark_as_mapped(&self, mut start: Address, bytes: usize) { @@ -232,7 +232,7 @@ impl Mmapper for MultiLevelMmapper { } } -impl MultiLevelMmapper { +impl TwoLevelMmapper { pub fn new() -> Self { Self { transition_lock: Default::default(), @@ -292,7 +292,7 @@ impl MultiLevelMmapper { } } -impl Default for MultiLevelMmapper { +impl Default for TwoLevelMmapper { fn default() -> Self { Self::new() } @@ -316,11 +316,11 @@ mod tests { conversions::raw_align_up(pages, MMAP_CHUNK_BYTES) / MMAP_CHUNK_BYTES } - fn get_chunk_map_state(mmapper: &MultiLevelMmapper, chunk: Address) -> Option { + fn get_chunk_map_state(mmapper: &TwoLevelMmapper, chunk: Address) -> Option { assert_eq!(conversions::mmap_chunk_align_up(chunk), chunk); let mapped = mmapper.slab_table(chunk); mapped.map(|m| { - m[MultiLevelMmapper::chunk_index(MultiLevelMmapper::slab_align_down(chunk), chunk)] + m[TwoLevelMmapper::chunk_index(TwoLevelMmapper::slab_align_down(chunk), chunk)] .load(Ordering::Relaxed) }) } @@ -331,7 +331,7 @@ mod tests { let pages = 1; with_cleanup( || { - let mmapper = MultiLevelMmapper::new(); + let mmapper = TwoLevelMmapper::new(); mmapper .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) .unwrap(); @@ -359,7 +359,7 @@ mod tests { let pages = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; with_cleanup( || { - let mmapper = MultiLevelMmapper::new(); + let mmapper = TwoLevelMmapper::new(); mmapper .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) .unwrap(); @@ -388,7 +388,7 @@ mod tests { let pages = (MMAP_CHUNK_BYTES + MMAP_CHUNK_BYTES / 2) >> LOG_BYTES_IN_PAGE as usize; with_cleanup( || { - let mmapper = MultiLevelMmapper::new(); + let mmapper = TwoLevelMmapper::new(); mmapper .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) .unwrap(); @@ -417,7 +417,7 @@ mod tests { with_cleanup( || { // map 2 chunks - let mmapper = MultiLevelMmapper::new(); + let mmapper = TwoLevelMmapper::new(); let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; mmapper .ensure_mapped( @@ -453,7 +453,7 @@ mod tests { with_cleanup( || { // map 2 chunks - let mmapper = MultiLevelMmapper::new(); + let mmapper = TwoLevelMmapper::new(); let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; mmapper .ensure_mapped( From 0b7330a695c982d0739e06e82594cd1e6edabcb9 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 11 Jul 2025 15:02:02 +0800 Subject: [PATCH 6/9] Minor comment change --- src/util/heap/layout/two_level_mmapper.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/heap/layout/two_level_mmapper.rs b/src/util/heap/layout/two_level_mmapper.rs index 61ca683ad0..cdc7c0c607 100644 --- a/src/util/heap/layout/two_level_mmapper.rs +++ b/src/util/heap/layout/two_level_mmapper.rs @@ -38,9 +38,9 @@ const MMAP_CHUNKS_PER_SLAB: usize = 1 << LOG_MMAP_CHUNKS_PER_SLAB; /// Invert this to get out-of-slab bits. const MMAP_SLAB_MASK: usize = (1 << LOG_MMAP_SLAB_BYTES) - 1; -/// Logarithm of maximum number of slabs, which determines the maximum mappable address space. +/// Logarithm of maximum number of slabs. const LOG_MAX_SLABS: usize = LOG_MAPPABLE_BYTES - LOG_MMAP_SLAB_BYTES; -/// maximum number of slabs, which determines the maximum mappable address space. +/// maximum number of slabs. const MAX_SLABS: usize = 1 << LOG_MAX_SLABS; /// The slab type. Each slab holds the `MapState` of multiple chunks. From 8142983058e490bdc6d0682058f78533863913a6 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 11 Jul 2025 16:03:50 +0800 Subject: [PATCH 7/9] Reimplement `slab_table` and `get_or_allocate_slab_table` --- src/util/heap/layout/two_level_mmapper.rs | 31 ++++++++--------------- src/util/rust_util/atomic_box.rs | 8 +++--- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/util/heap/layout/two_level_mmapper.rs b/src/util/heap/layout/two_level_mmapper.rs index cdc7c0c607..323edfcf25 100644 --- a/src/util/heap/layout/two_level_mmapper.rs +++ b/src/util/heap/layout/two_level_mmapper.rs @@ -25,7 +25,8 @@ const MAPPABLE_BYTES: usize = 1 << LOG_MAPPABLE_BYTES; /// [`TwoLevelMmapper::slabs`] and [`Slab`]. /// /// TODO: Use `usize::midpoint` after bumping MSRV to 1.85 -const LOG_MMAP_SLAB_BYTES: usize = LOG_MMAP_CHUNK_BYTES + (LOG_MAPPABLE_BYTES - LOG_MMAP_CHUNK_BYTES) / 2; +const LOG_MMAP_SLAB_BYTES: usize = + LOG_MMAP_CHUNK_BYTES + (LOG_MAPPABLE_BYTES - LOG_MMAP_CHUNK_BYTES) / 2; /// Number of bytes per slab. const MMAP_SLAB_BYTES: usize = 1 << LOG_MMAP_SLAB_BYTES; @@ -245,29 +246,17 @@ impl TwoLevelMmapper { } fn slab_table(&self, addr: Address) -> Option<&Slab> { - self.get_or_optionally_allocate_slab_table(addr, false) + let index: usize = addr >> LOG_MMAP_SLAB_BYTES; + let slot = self.slabs.get(index)?; + slot.get(Ordering::Relaxed) } fn get_or_allocate_slab_table(&self, addr: Address) -> &Slab { - self.get_or_optionally_allocate_slab_table(addr, true) - .unwrap() - } - - fn get_or_optionally_allocate_slab_table( - &self, - addr: Address, - allocate: bool, - ) -> Option<&Slab> { - let index = addr >> LOG_MMAP_SLAB_BYTES; - if index > self.slabs.len() { - panic!("addr: {addr}, index: {index}, slabs.len: {sl}", sl = self.slabs.len()); - } - let slot = &self.slabs[index]; - if allocate { - slot.get_or_init(Ordering::Acquire, Ordering::Release, Self::new_slab) - } else { - slot.get(Ordering::Acquire) - } + let index: usize = addr >> LOG_MMAP_SLAB_BYTES; + let Some(slot) = self.slabs.get(index) else { + panic!("Cannot allocate slab for address: {addr}"); + }; + slot.get_or_init(Ordering::Relaxed, Ordering::Relaxed, Self::new_slab) } fn chunk_index_to_address(base: Address, chunk: usize) -> Address { diff --git a/src/util/rust_util/atomic_box.rs b/src/util/rust_util/atomic_box.rs index 20de1f3aa1..498cc2906f 100644 --- a/src/util/rust_util/atomic_box.rs +++ b/src/util/rust_util/atomic_box.rs @@ -22,9 +22,9 @@ impl OnceOptionBox { order_load: Ordering, order_store: Ordering, init: impl FnOnce() -> T, - ) -> Option<&T> { + ) -> &T { if let Some(get_result) = self.get(order_load) { - return Some(get_result); + return get_result; } let new_inner = Box::into_raw(Box::new(init())); @@ -37,11 +37,11 @@ impl OnceOptionBox { match cas_result { Ok(old_inner) => { debug_assert_eq!(old_inner, std::ptr::null_mut()); - unsafe { new_inner.as_ref() } + unsafe { new_inner.as_ref().unwrap() } } Err(old_inner) => { drop(unsafe { Box::from_raw(new_inner) }); - unsafe { old_inner.as_ref() } + unsafe { old_inner.as_ref().unwrap() } } } } From 933285237d330b27e5b2cfd4eaa9881c13c53923 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 11 Jul 2025 16:07:39 +0800 Subject: [PATCH 8/9] Comments on Relaxed order --- src/util/heap/layout/two_level_mmapper.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/util/heap/layout/two_level_mmapper.rs b/src/util/heap/layout/two_level_mmapper.rs index 323edfcf25..a24105083f 100644 --- a/src/util/heap/layout/two_level_mmapper.rs +++ b/src/util/heap/layout/two_level_mmapper.rs @@ -248,6 +248,7 @@ impl TwoLevelMmapper { fn slab_table(&self, addr: Address) -> Option<&Slab> { let index: usize = addr >> LOG_MMAP_SLAB_BYTES; let slot = self.slabs.get(index)?; + // Note: We don't need acquire here. See `get_or_allocate_slab_table`. slot.get(Ordering::Relaxed) } @@ -256,6 +257,9 @@ impl TwoLevelMmapper { let Some(slot) = self.slabs.get(index) else { panic!("Cannot allocate slab for address: {addr}"); }; + // Note: We set both order_load and order_store to `Relaxed` because we never populate the + // content of the slab before making the `OnceOptionBox` point to the new slab. For this + // reason, the release-acquire relation is not needed here. slot.get_or_init(Ordering::Relaxed, Ordering::Relaxed, Self::new_slab) } From 98088c80ed76f97b711a658b42e43343c618cf1e Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 11 Jul 2025 16:44:05 +0800 Subject: [PATCH 9/9] Only make TwoLevelMmapper available on 64-bit --- src/util/heap/layout/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/util/heap/layout/mod.rs b/src/util/heap/layout/mod.rs index ab53492cc3..7f3366ac1f 100644 --- a/src/util/heap/layout/mod.rs +++ b/src/util/heap/layout/mod.rs @@ -4,6 +4,9 @@ pub mod vm_layout; mod mmapper; pub use self::mmapper::Mmapper; mod byte_map_mmapper; +// Only make `two_level_mapper` available on 64-bit architectures because it contains 64-bit `usize` +// constants that only work on 64-bit architectures. +#[cfg(target_pointer_width = "64")] mod two_level_mmapper; mod map;