Initial tests for the 'epsilon' collector

TODO: Consider moving the 'epsilon' collector to an independent crate?

As usual, debugging w/ malloc is much easier
than with arena allocation.

Author: Techcable

Cargo.toml

@@ -26,7 +26,7 @@ bumpalo = { version = "3", optional = true }
 members = ["libs/simple", "libs/derive", "libs/context"]
 
 [features]
-default = ["std", "epsilon"]
+default = ["std", "epsilon", "epsilon-arena-alloc"]
 # Depend on the standard library (optional)
 #
 # This implements tracing for most standard library types.
@@ -39,4 +39,7 @@ alloc = []
 # Serde support
 serde1 = ["serde", "zerogc-derive/__serde-internal", "indexmap/serde-1"]
 # Support the "epsilon" no-op collector
-epsilon = ["bumpalo"]
+epsilon = []
+# Configure the "epsilon" collector use arena allocation
+# (on by default)
+epsilon-arena-alloc = ["epsilon", "bumpalo"]

src/epsilon.rs

@@ -9,16 +9,18 @@
 #![cfg(feature = "epsilon")]
 
 mod layout;
+mod alloc;
 
 use crate::{CollectorId, GcContext, GcSafe, GcSimpleAlloc, GcSystem, GcVisitor, NullTrace, Trace, TraceImmutable, TrustedDrop};
 use std::ptr::NonNull;
 use std::alloc::Layout;
 use std::rc::Rc;
 use std::cell::Cell;
 use std::ffi::c_void;
-
 use std::lazy::OnceCell;
 
+use self::{alloc::{EpsilonAlloc}, layout::TypeInfo};
+
 /// Fake a [Gc] that points to the specified value
 ///
 /// This will never actually be collected
@@ -43,7 +45,7 @@ macro_rules! epsilon_static_array {
             array: [T; LEN]
         }
         static HEADERED: &'static ArrayWithHeader<$target, { $len }> = &ArrayWithHeader {
-            header: self::epsilon::ArrayHeaderHack::for_len($len),
+            header: self::epsilon::ArrayHeaderHack::for_len::<$target>($len),
             array: $values
         };
         std::mem::transmute::<
@@ -61,10 +63,10 @@ pub struct ArrayHeaderHack(layout::EpsilonArrayHeader);
 impl ArrayHeaderHack {
     #[inline]
     #[doc(hidden)]
-    pub const fn for_len(len: usize) -> ArrayHeaderHack {
+    pub const fn for_len<T>(len: usize) -> ArrayHeaderHack {
         ArrayHeaderHack(layout::EpsilonArrayHeader {
             len, common_header: layout::EpsilonHeader {
-                type_info: None,
+                type_info: TypeInfo::of_array::<T>(),
                 next: None
             }
         })
@@ -130,7 +132,9 @@ unsafe impl GcContext for EpsilonContext {
 
     #[inline]
     fn system(&self) -> &'_ Self::System {
-        unsafe { self.state.cast::<EpsilonSystem>().as_ref() }
+        // Pointer to a pointer
+        unsafe { NonNull::<NonNull<State>>::from(&self.state)
+            .cast::<EpsilonSystem>().as_ref() }
     }
 
 
@@ -151,7 +155,7 @@ impl Drop for EpsilonContext {
 }
 
 struct State {
-    alloc: bumpalo::Bump,
+    alloc: alloc::Default,
     /// The head of the linked-list of allocated objects.
     head: Cell<Option<NonNull<layout::EpsilonHeader>>>,
     empty_vec: OnceCell<NonNull<layout::EpsilonVecRepr>>
@@ -163,6 +167,39 @@ impl State {
         self.head.set(Some(header));
     }
 }
+impl Drop for State {
+    fn drop(&mut self) {
+        let mut ptr = self.head.get();
+        unsafe {
+            while let Some(header) = ptr {
+                let header_layout = layout::EpsilonHeader::LAYOUT;
+                let desired_align = header.as_ref().type_info.layout.align();
+                let padding = header_layout.padding_needed_for(desired_align);
+                let value_ptr = (header.as_ptr() as *const u8)
+                    .add(header_layout.size())
+                    .add(padding);
+                if let Some(drop_func) = header.as_ref().type_info.drop_func {
+                    (drop_func)(value_ptr as *const _ as *mut _);
+                }
+                let next = header.as_ref().next;
+                if self::alloc::Default::NEEDS_EXPLICIT_FREE {
+                    let value_layout = header.as_ref().determine_layout();
+                    let original_header = NonNull::new_unchecked(header.cast::<u8>()
+                        .as_ptr()
+                        .sub(header.as_ref().type_info.layout.common_header_offset()));
+                    let header_size = value_ptr.cast::<u8>()
+                        .offset_from(original_header.as_ptr()) as usize;
+                    let combined_layout = Layout::from_size_align_unchecked(
+                        value_layout.size() + header_size,
+                        value_layout.align().max(layout::EpsilonHeader::LAYOUT.align())
+                    );
+                    self.alloc.free_alloc(original_header, combined_layout);
+                }
+                ptr = next;
+            }
+        }
+    }
+}
 
 /// A dummy implementation of [GcSystem]
 /// which is useful for testing
@@ -191,7 +228,7 @@ impl EpsilonSystem {
     #[inline]
     pub fn leak() -> Self {
         EpsilonSystem::from_state(Rc::new(State {
-            alloc: bumpalo::Bump::new(),
+            alloc: self::alloc::Default::new(),
             head: Cell::new(None),
             empty_vec: OnceCell::new()
         }))
@@ -228,13 +265,16 @@ unsafe impl GcSimpleAlloc for EpsilonContext {
     #[inline]
     unsafe fn alloc_uninit<'gc, T>(&'gc self) -> (Self::Id, *mut T) where T: GcSafe<'gc, EpsilonCollectorId> + 'gc {
         let id = self.id();
-        let ptr = if let Some(tp) = self::layout::TypeInfo::of::<T>() {
+        let tp = self::layout::TypeInfo::of::<T>();
+        let needs_header = self::alloc::Default::NEEDS_EXPLICIT_FREE
+            || !tp.may_ignore();
+        let ptr = if needs_header {
             let (overall_layout, offset) = self::layout::EpsilonHeader::LAYOUT
                 .extend(Layout::new::<T>()).unwrap();
             let mem = self.system().state().alloc.alloc_layout(overall_layout);
             let header = mem.cast::<self::layout::EpsilonHeader>();
             header.as_ptr().write(self::layout::EpsilonHeader {
-                type_info: Some(tp),
+                type_info: tp,
                 next: None
             });
             self.system().state().push_state(header);
@@ -392,7 +432,7 @@ unsafe impl CollectorId for EpsilonCollectorId {
     unsafe fn assume_valid_system(&self) -> &Self::System {
         /*
          * NOTE: Supporting this would lose our ability to go from `&'static T` -> `Gc<'gc, T, EpsilonCollectorId>
-         * It would also nessesitate a header for `Copy` objects.
+         * It would also necessitate a header for `Copy` objects.
          */
         unimplemented!("Unable to convert EpsilonCollectorId -> EpsilonSystem")
     }

src/epsilon/alloc.rs

@@ -0,0 +1,47 @@
+use std::ptr::NonNull;
+use std::alloc::Layout;
+
+#[cfg(feature = "epsilon-arena-alloc")]
+mod arena;
+
+pub trait EpsilonAlloc {
+    fn new() -> Self;
+    fn alloc_layout(&self, layout: Layout) -> NonNull<u8>;
+    unsafe fn free_alloc(&self, target: NonNull<u8>, layout: Layout);
+    const NEEDS_EXPLICIT_FREE: bool;
+}
+
+#[cfg(feature = "epsilon-arena-alloc")]
+pub type Default = arena::BumpEpsilonAlloc;
+#[cfg(not(feature = "epsilon-arena-alloc"))]
+pub type Default = StdEpsilonAlloc;
+
+pub struct StdEpsilonAlloc;
+impl EpsilonAlloc for StdEpsilonAlloc {
+    #[inline]
+    fn new() -> Self {
+        StdEpsilonAlloc
+    }
+
+    #[inline]
+    fn alloc_layout(&self, layout: Layout) -> NonNull<u8> {
+        const EMPTY: &[u8] = b"";
+        if layout.size() == 0 {
+            return NonNull::from(EMPTY).cast();
+        }
+        // SAFETY: We checked for layout.size() == 0
+        NonNull::new(unsafe { std::alloc::alloc(layout) })
+            .unwrap_or_else(|| std::alloc::handle_alloc_error(layout))
+    }
+
+    #[inline]
+    unsafe fn free_alloc(&self, target: NonNull<u8>, layout: Layout) {
+        if layout.size() == 0 {
+            return; // We returned our dummy empty alloc
+        }
+        std::alloc::dealloc(target.as_ptr(), layout)
+    }
+
+    const NEEDS_EXPLICIT_FREE: bool = true;
+}
+

src/epsilon/alloc/arena.rs

@@ -0,0 +1,21 @@
+use std::ptr::NonNull;
+use std::alloc::Layout;
+
+use bumpalo::Bump;
+
+use super::EpsilonAlloc;
+
+pub struct BumpEpsilonAlloc(Bump);
+impl EpsilonAlloc for BumpEpsilonAlloc {
+    #[inline]
+    fn new() -> Self {
+        BumpEpsilonAlloc(Bump::new())
+    }
+    #[inline]
+    fn alloc_layout(&self, layout: Layout) -> NonNull<u8> {
+        self.0.alloc_layout(layout)
+    }
+    #[inline]
+    unsafe fn free_alloc(&self, _target: NonNull<u8>, _layout: Layout) {}
+    const NEEDS_EXPLICIT_FREE: bool = false;
+}

src/epsilon/layout.rs

@@ -12,7 +12,7 @@ use crate::vec::repr::GcVecRepr;
 /// They are also unnecessary for statically allocated objects.
 pub struct EpsilonHeader {
     /// This object's `TypeInfo`, or `None` if it doesn't need any.
-    pub type_info: Option<&'static TypeInfo>,
+    pub type_info: &'static TypeInfo,
     /// The next allocated object, or `None` if this is the final object.
     pub next: Option<NonNull<EpsilonHeader>>
 }
@@ -29,6 +29,20 @@ impl EpsilonHeader {
         let (_, offset) = Self::LAYOUT.extend(Layout::for_value(&*header)).unwrap_unchecked();
         (header as *const c_void).sub(offset).cast()
     }
+    #[inline]
+    #[track_caller]
+    pub unsafe fn determine_layout(&self) -> Layout {
+        let tp = self.type_info;
+        match tp.layout {
+            LayoutInfo::Fixed(fixed) => fixed,
+            LayoutInfo::Array { element_layout } |
+            LayoutInfo::Vec { element_layout } => {
+                let array_header = EpsilonArrayHeader::from_common_header(self);
+                let len = (*array_header).len;
+                element_layout.repeat(len).unwrap_unchecked().0
+            }
+        }
+    }
 }
 #[repr(C)]
 pub struct EpsilonArrayHeader {
@@ -49,37 +63,67 @@ pub struct EpsilonVecHeader {
     pub len: Cell<usize>,
     pub common_header: EpsilonHeader,
 }
+impl EpsilonVecHeader {
+    const COMMON_OFFSET: usize = std::mem::size_of::<Self>() - std::mem::size_of::<EpsilonHeader>();
+}
+pub enum LayoutInfo {
+    Fixed(Layout),
+    /// A variable sized array
+    Array {
+        element_layout: Layout
+    },
+    /// A variable sized vector
+    Vec {
+        element_layout: Layout
+    }
+}
+impl LayoutInfo {
+    #[inline]
+    pub const fn align(&self) -> usize {
+        match *self {
+            LayoutInfo::Fixed(layout) |
+            LayoutInfo::Array { element_layout: layout } |
+            LayoutInfo::Vec { element_layout: layout }  => layout.align()
+        }
+    }
+    #[inline]
+    pub fn common_header_offset(&self) -> usize {
+        match *self {
+            LayoutInfo::Fixed(_) => 0,
+            LayoutInfo::Array { .. } => EpsilonArrayHeader::COMMON_OFFSET,
+            LayoutInfo::Vec { .. } => EpsilonVecHeader::COMMON_OFFSET
+        }
+    }
+}
 pub struct TypeInfo {
     /// The function to drop this object, or `None` if the object doesn't need to be dropped
     pub drop_func: Option<unsafe fn(*mut c_void)>,
-    /// The size of the object, or `None` if this is an array or vector
-    pub size: Option<usize>
+    pub layout: LayoutInfo
 }
 impl TypeInfo {
     #[inline]
-    pub const fn of<T>() -> Option<&'static TypeInfo> {
-        if std::mem::needs_drop::<T>() {
-            Some(<T as StaticTypeInfo>::TYPE_INFO)
-        } else {
-            None
-        }
+    pub const fn may_ignore(&self) -> bool {
+        // NOTE: We don't care about `size`
+        self.drop_func.is_none() &&
+            self.layout.align() <= std::mem::align_of::<usize>()
     }
     #[inline]
-    pub const fn of_array<T>() -> Option<&'static TypeInfo> {
-        if std::mem::needs_drop::<T>() {
-            Some(<[T] as StaticTypeInfo>::TYPE_INFO)
-        } else {
-            None
-        }
+    pub const fn of<T>() -> &'static TypeInfo {
+        <T as StaticTypeInfo>::TYPE_INFO
+    }
+    #[inline]
+    pub const fn of_array<T>() -> &'static TypeInfo {
+        <[T] as StaticTypeInfo>::TYPE_INFO
     }
     #[inline]
-    pub const fn of_vec<T>() -> Option<&'static TypeInfo> {
+    pub const fn of_vec<T>() -> &'static TypeInfo {
         // For now, vectors and arrays share type info
-        Self::of_array::<T>()
+        <T as StaticTypeInfo>::VEC_INFO.as_ref().unwrap()
     }
 }
 trait StaticTypeInfo {
     const TYPE_INFO: &'static TypeInfo;
+    const VEC_INFO: &'static Option<TypeInfo>;
 }
 impl<T> StaticTypeInfo for T {
     const TYPE_INFO: &'static TypeInfo = &TypeInfo {
@@ -88,16 +132,29 @@ impl<T> StaticTypeInfo for T {
         } else {
             None
         },
-        size: Some(std::mem::size_of::<T>())
+        layout: LayoutInfo::Fixed(Layout::new::<T>()),
     };
+    const VEC_INFO: &'static Option<TypeInfo> = &Some(TypeInfo {
+        drop_func: if std::mem::needs_drop::<T>() {
+            Some(drop_array::<T>)
+        } else {
+            None
+        },
+        layout: LayoutInfo::Vec {
+            element_layout: Layout::new::<T>()
+        }
+    });
 }
 impl<T> StaticTypeInfo for [T] {
     const TYPE_INFO: &'static TypeInfo = &TypeInfo {
         drop_func: if std::mem::needs_drop::<T>() {
             Some(drop_array::<T>)
         } else { None },
-        size: None
+        layout: LayoutInfo::Array {
+            element_layout: Layout::new::<T>()
+        }
     };
+    const VEC_INFO: &'static Option<TypeInfo> = &None;
 }
 /// Drop an array or vector of the specified type
 unsafe fn drop_array<T>(ptr: *mut c_void) {
@@ -167,7 +224,6 @@ unsafe impl<'gc> GcVecRepr<'gc> for EpsilonVecRepr {
     fn capacity(&self) -> usize {
         unsafe { (*self.header()).capacity }
     }
-
     #[inline]
     unsafe fn ptr(&self) -> *const c_void {
         self as *const Self as *const c_void // We are actually just a GC pointer to the value ptr

src/lib.rs

@@ -9,6 +9,8 @@
     // Needed for epsilon collector:
     once_cell, // RFC 2788 (Probably will be accepted)
     alloc_layout_extra,
+    const_fn_fn_ptr_basics,
+    const_option,
     const_fn_trait_bound, // NOTE: Needed for the `epsilon_static_array` macro
 )]
 #![feature(maybe_uninit_slice)]