CoAlloc: Vec + related (Allocator, GlobalAlloc, proc_macro; vec/macros).

Author: peter-lyons-kehl

compiler/rustc_arena/src/lib.rs

@@ -11,10 +11,12 @@
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(no_crate_inject, attr(deny(warnings)))
 )]
+#![allow(incomplete_features)]
 #![feature(dropck_eyepatch)]
 #![feature(new_uninit)]
 #![feature(maybe_uninit_slice)]
-#![feature(min_specialization)]
+//#![feature(min_specialization)]
+#![feature(specialization)]
 #![feature(decl_macro)]
 #![feature(pointer_byte_offsets)]
 #![feature(rustc_attrs)]

compiler/rustc_ast/src/ast.rs

@@ -26,7 +26,6 @@ pub use UnsafeSource::*;
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Delimiter};
 use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, TokenStream};
-use core::alloc::GlobalCoAllocMeta;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::sync::Lrc;
@@ -35,6 +34,8 @@ use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 use rustc_span::source_map::{respan, Spanned};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
+use std::alloc::Allocator;
+use std::alloc::Global;
 use std::fmt;
 use std::mem;
 use thin_vec::{thin_vec, ThinVec};
@@ -3117,26 +3118,26 @@ mod size_asserts {
     static_assert_size!(AssocItem, 104);
     static_assert_size!(AssocItemKind, 32);
     static_assert_size!(Attribute, 32);
-    static_assert_size!(Block, 32 + mem::size_of::<GlobalCoAllocMeta>());
-    static_assert_size!(Expr, 72 + mem::size_of::<GlobalCoAllocMeta>());
-    static_assert_size!(ExprKind, 40 + mem::size_of::<GlobalCoAllocMeta>());
-    static_assert_size!(Fn, 152 + 2 * mem::size_of::<GlobalCoAllocMeta>());
+    static_assert_size!(Block, 32 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(Expr, 72 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(ExprKind, 40 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(Fn, 152 + 2 * mem::size_of::<<Global as Allocator>::CoAllocMeta>());
     static_assert_size!(ForeignItem, 96);
     static_assert_size!(ForeignItemKind, 24);
     static_assert_size!(GenericArg, 24);
-    static_assert_size!(GenericBound, 56 + mem::size_of::<GlobalCoAllocMeta>());
-    static_assert_size!(Generics, 40 + 2 * mem::size_of::<GlobalCoAllocMeta>());
-    static_assert_size!(Impl, 136 + 3 * mem::size_of::<GlobalCoAllocMeta>());
-    static_assert_size!(Item, 136 + 3 * mem::size_of::<GlobalCoAllocMeta>());
-    static_assert_size!(ItemKind, 64 + 3 * mem::size_of::<GlobalCoAllocMeta>());
+    static_assert_size!(GenericBound, 56 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(Generics, 40 + 2 * mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(Impl, 136 + 3 * mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(Item, 136 + 3 * mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(ItemKind, 64 + 3 * mem::size_of::<<Global as Allocator>::CoAllocMeta>());
     static_assert_size!(LitKind, 24);
     static_assert_size!(Local, 72);
     static_assert_size!(MetaItemLit, 40);
     static_assert_size!(Param, 40);
-    static_assert_size!(Pat, 72 + mem::size_of::<GlobalCoAllocMeta>());
+    static_assert_size!(Pat, 72 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
     static_assert_size!(Path, 24);
     static_assert_size!(PathSegment, 24);
-    static_assert_size!(PatKind, 48 + mem::size_of::<GlobalCoAllocMeta>());
+    static_assert_size!(PatKind, 48 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
     static_assert_size!(Stmt, 32);
     static_assert_size!(StmtKind, 16);
     static_assert_size!(Ty, 64);

compiler/rustc_ast/src/lib.rs

@@ -8,6 +8,7 @@
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(attr(deny(warnings)))
 )]
+#![feature(allocator_api)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(const_default_impls)]

compiler/rustc_parse/src/parser/mod.rs

@@ -166,7 +166,10 @@ pub struct Parser<'a> {
 // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure
 // it doesn't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(Parser<'_>, 288 + 4 * mem::size_of::<GlobalCoAllocMeta>());
+rustc_data_structures::static_assert_size!(
+    Parser<'_>,
+    288 + 4 * mem::size_of::<GlobalCoAllocMeta>()
+);
 
 /// Stores span information about a closure.
 #[derive(Clone)]

library/alloc/src/boxed.rs

@@ -146,6 +146,7 @@
 
 #![stable(feature = "rust1", since = "1.0.0")]
 
+use crate::co_alloc::CoAllocPref;
 use core::any::Any;
 use core::async_iter::AsyncIterator;
 use core::borrow;
@@ -641,7 +642,9 @@ impl<T> Box<[T]> {
     #[must_use]
     pub fn new_uninit_slice(len: usize) -> Box<[mem::MaybeUninit<T>]> {
         // false = no need for co-alloc metadata, since it would get lost once converted to Box.
-        unsafe { RawVec::<T, Global, false>::with_capacity(len).into_box(len) }
+        unsafe {
+            RawVec::<T, Global, { CO_ALLOC_PREF_META_NO!() }>::with_capacity(len).into_box(len)
+        }
     }
 
     /// Constructs a new boxed slice with uninitialized contents, with the memory
@@ -667,7 +670,10 @@ impl<T> Box<[T]> {
     #[must_use]
     pub fn new_zeroed_slice(len: usize) -> Box<[mem::MaybeUninit<T>]> {
         // false = no need for co-alloc metadata, since it would get lost once converted to Box.
-        unsafe { RawVec::<T, Global, false>::with_capacity_zeroed(len).into_box(len) }
+        unsafe {
+            RawVec::<T, Global, { CO_ALLOC_PREF_META_NO!() }>::with_capacity_zeroed(len)
+                .into_box(len)
+        }
     }
 
     /// Constructs a new boxed slice with uninitialized contents. Returns an error if
@@ -699,7 +705,7 @@ impl<T> Box<[T]> {
                 Err(_) => return Err(AllocError),
             };
             let ptr = Global.allocate(layout)?;
-            Ok(RawVec::<T, Global, false>::from_raw_parts_in(
+            Ok(RawVec::<T, Global, { CO_ALLOC_PREF_META_NO!() }>::from_raw_parts_in(
                 ptr.as_mut_ptr() as *mut _,
                 len,
                 Global,
@@ -736,7 +742,7 @@ impl<T> Box<[T]> {
                 Err(_) => return Err(AllocError),
             };
             let ptr = Global.allocate_zeroed(layout)?;
-            Ok(RawVec::<T, Global, false>::from_raw_parts_in(
+            Ok(RawVec::<T, Global, { CO_ALLOC_PREF_META_NO!() }>::from_raw_parts_in(
                 ptr.as_mut_ptr() as *mut _,
                 len,
                 Global,
@@ -746,9 +752,10 @@ impl<T> Box<[T]> {
     }
 }
 
+#[allow(unused_braces)]
 impl<T, A: Allocator> Box<[T], A>
 where
-    [(); core::alloc::co_alloc_metadata_num_slots::<A>()]:,
+    [(); { crate::meta_num_slots!(A, crate::CO_ALLOC_PREF_META_NO!()) }]:,
 {
     /// Constructs a new boxed slice with uninitialized contents in the provided allocator.
     ///
@@ -777,12 +784,10 @@ where
     // #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
     #[allow(unused_braces)]
-    pub fn new_uninit_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A>
-    where
-        // false = no need for co-alloc metadata, since it would get lost once converted to Box.
-        [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<A>(false)]:,
-    {
-        unsafe { RawVec::<T, A, false>::with_capacity_in(len, alloc).into_box(len) }
+    pub fn new_uninit_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> {
+        unsafe {
+            RawVec::<T, A, { CO_ALLOC_PREF_META_NO!() }>::with_capacity_in(len, alloc).into_box(len)
+        }
     }
 
     /// Constructs a new boxed slice with uninitialized contents in the provided allocator,
@@ -810,12 +815,11 @@ where
     // #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
     #[allow(unused_braces)]
-    pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A>
-    where
-        // false = no need for co-alloc metadata, since it would get lost once converted to Box.
-        [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<A>(false)]:,
-    {
-        unsafe { RawVec::<T, A, false>::with_capacity_zeroed_in(len, alloc).into_box(len) }
+    pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> {
+        unsafe {
+            RawVec::<T, A, { CO_ALLOC_PREF_META_NO!() }>::with_capacity_zeroed_in(len, alloc)
+                .into_box(len)
+        }
     }
 }
 
@@ -1523,7 +1527,7 @@ impl<T: Copy> From<&[T]> for Box<[T]> {
     fn from(slice: &[T]) -> Box<[T]> {
         let len = slice.len();
         // false = no need for co-alloc metadata, since it would get lost once converted to Box.
-        let buf = RawVec::<T, Global, false>::with_capacity(len);
+        let buf = RawVec::<T, Global, { CO_ALLOC_PREF_META_NO!() }>::with_capacity(len);
         unsafe {
             ptr::copy_nonoverlapping(slice.as_ptr(), buf.ptr(), len);
             buf.into_box(slice.len()).assume_init()
@@ -1687,12 +1691,13 @@ impl<T, const N: usize> TryFrom<Box<[T]>> for Box<[T; N]> {
 
 #[cfg(not(no_global_oom_handling))]
 #[stable(feature = "boxed_array_try_from_vec", since = "1.66.0")]
-impl<T, const N: usize, const COOP_PREFERRED: bool> TryFrom<Vec<T, Global, COOP_PREFERRED>>
+#[allow(unused_braces)]
+impl<T, const N: usize, const CO_ALLOC_PREF: CoAllocPref> TryFrom<Vec<T, Global, CO_ALLOC_PREF>>
     for Box<[T; N]>
 where
-    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:,
+    [(); { meta_num_slots_global!(CO_ALLOC_PREF) }]:,
 {
-    type Error = Vec<T, Global, COOP_PREFERRED>;
+    type Error = Vec<T, Global, CO_ALLOC_PREF>;
 
     /// Attempts to convert a `Vec<T>` into a `Box<[T; N]>`.
     ///
@@ -1712,7 +1717,7 @@ where
     /// let state: Box<[f32; 100]> = vec![1.0; 100].try_into().unwrap();
     /// assert_eq!(state.len(), 100);
     /// ```
-    fn try_from(vec: Vec<T, Global, COOP_PREFERRED>) -> Result<Self, Self::Error> {
+    fn try_from(vec: Vec<T, Global, CO_ALLOC_PREF>) -> Result<Self, Self::Error> {
         if vec.len() == N {
             let boxed_slice = vec.into_boxed_slice();
             Ok(unsafe { boxed_slice_as_array_unchecked(boxed_slice) })
@@ -2049,14 +2054,15 @@ impl<I> FromIterator<I> for Box<[I]> {
 
 #[cfg(not(no_global_oom_handling))]
 #[stable(feature = "box_slice_clone", since = "1.3.0")]
+#[allow(unused_braces)]
 impl<T: Clone, A: Allocator + Clone> Clone for Box<[T], A>
 where
-    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<A>(false)]:,
+    [(); { crate::meta_num_slots!(A, crate::CO_ALLOC_PREF_META_NO!()) }]:,
 {
     fn clone(&self) -> Self {
         let alloc = Box::allocator(self).clone();
         // false = no need for co-alloc metadata, since it would get lost once converted to the boxed slice.
-        self.to_vec_in::<A, false>(alloc).into_boxed_slice()
+        self.to_vec_in_co::<A, { CO_ALLOC_PREF_META_NO!() }>(alloc).into_boxed_slice()
     }
 
     fn clone_from(&mut self, other: &Self) {

library/alloc/src/co_alloc.rs

@@ -0,0 +1,47 @@
+//! CoAlloction-specific types that only apply in heap-based applications (hence not a part of
+//!  [::core]).
+//!
+//! Types here have names with `CoAlloc` prefix. Yes, when using a q ualified path (like
+//! ::alloc::co_alloc::CoAllocPref), that involves "stuttering", which is not recommended.
+//!
+//! However, as per Rust Book the common practice is to import type names fully and access them just
+//! with their name (except for cases of conflict). And we don't want the type names any shorter
+//! (such `Pref`), because thoe would be vague/confusing.
+
+/// `CoAllocPref` values indicate a type's preference for coallocation (in either user space, or
+/// `std` space). Used as a `const` generic parameter type (usually called `CO_ALLOC_PREF`).
+///
+/// The actual value may be overriden by the allocator. See also `CoAllocMetaNumSlotsPref` and
+/// `co_alloc_pref` macro .
+///
+/// This type  WILL CHANGE (once ``#![feature(generic_const_exprs)]` and
+/// `#![feature(adt_const_params)]` are stable) to a dedicated struct/enum. Hence:
+/// - DO NOT construct instances, but use `co_alloc_pref` macro together with constants
+/// `CO_ALLOC_PREF_META_YES` and `CO_ALLOC_PREF_META_NO`;
+/// - DO NOT hard code any values; and
+/// - DO NOT mix this/cast this with/to `u8`, `u16`, `usize` (nor any other integer).
+#[unstable(feature = "global_co_alloc_meta", issue = "none")]
+pub type CoAllocPref = usize; //u8;
+
+/// `CoAllocMetaNumSlotsPref` values indicate that a type (but not necessarily an allocator) prefers
+/// to coallocate by carrying metadata, or not. (In either user space, or `std` or `alloc` space).
+/// Used as an argument to macro call of `co_alloc_pref`, which generates a `CoAllocPref` value.
+///
+/// Currently this indicates only the (preferred) number of `CoAllocMetaBase` slots being used
+/// (either 1 = coallocation, or 0 = no coallocation). However, in the future this type may have
+/// other properties (serving as extra hints to the allocator).
+///
+/// The actual value may be overriden by the allocator. For example, if the allocator doesn't
+/// support coallocation, then whether this value prefers to coallocate or not makes no difference.
+///
+/// This type  WILL CHANGE (once ``#![feature(generic_const_exprs)]` and
+/// `#![feature(adt_const_params)]` are stable) to a dedicated struct/enum. Hence:
+/// - DO NOT mix this/cast this with/to `u8`, `u16`, (nor any other integer); and
+/// - DO NOT hard code any values, but use `CO_ALLOC_PREF_META_YES` and `CO_ALLOC_PREF_META_NO`.
+///
+/// This type is intentionally not `u16`, `u32`, nor `usize`. Why? This helps to prevent mistakes
+/// when one would use `CO_ALLOC_PREF_META_YES` or `CO_ALLOC_PREF_META_NO` in place of `CoAllocPref`
+/// vales, or in place of a result of `meta_num_slots` macro. That also prevents mixing up with
+/// [core::alloc::CoAllocatorMetaNumSlots].
+#[unstable(feature = "global_co_alloc_meta", issue = "none")]
+pub type CoAllocMetaNumSlotsPref = u16;

library/alloc/src/collections/binary_heap/mod.rs

@@ -143,6 +143,7 @@
 #![allow(missing_docs)]
 #![stable(feature = "rust1", since = "1.0.0")]
 
+use crate::co_alloc::CoAllocPref;
 use core::fmt;
 use core::iter::{FromIterator, FusedIterator, InPlaceIterable, SourceIter, TrustedLen};
 use core::mem::{self, swap, ManuallyDrop};
@@ -155,7 +156,7 @@ use crate::alloc::Global;
 use crate::collections::TryReserveError;
 use crate::slice;
 use crate::vec::{self, AsVecIntoIter, Vec};
-use crate::DEFAULT_COOP_PREFERRED;
+use crate::CO_ALLOC_PREF_DEFAULT;
 
 use super::SpecExtend;
 
@@ -1257,7 +1258,7 @@ impl<T> BinaryHeap<T> {
     #[inline]
     #[stable(feature = "drain", since = "1.6.0")]
     #[allow(unused_braces)]
-    pub fn drain(&mut self) -> Drain<'_, T, { SHORT_TERM_VEC_PREFERS_COOP!() }> {
+    pub fn drain(&mut self) -> Drain<'_, T, { SHORT_TERM_VEC_CO_ALLOC_PREF!() }> {
         Drain { iter: self.data.drain(..) }
     }
 
@@ -1537,17 +1538,19 @@ unsafe impl<T: Ord> TrustedLen for IntoIterSorted<T> {}
 /// [`drain`]: BinaryHeap::drain
 #[stable(feature = "drain", since = "1.6.0")]
 #[derive(Debug)]
-pub struct Drain<'a, T: 'a, const COOP_PREFERRED: bool>
+#[allow(unused_braces)]
+pub struct Drain<'a, T: 'a, const CO_ALLOC_PREF: CoAllocPref>
 where
-    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:,
+    [(); { meta_num_slots_global!(CO_ALLOC_PREF) }]:,
 {
-    iter: vec::Drain<'a, T, Global, COOP_PREFERRED>,
+    iter: vec::Drain<'a, T, Global, CO_ALLOC_PREF>,
 }
 
 #[stable(feature = "drain", since = "1.6.0")]
-impl<T, const COOP_PREFERRED: bool> Iterator for Drain<'_, T, COOP_PREFERRED>
+#[allow(unused_braces)]
+impl<T, const CO_ALLOC_PREF: CoAllocPref> Iterator for Drain<'_, T, CO_ALLOC_PREF>
 where
-    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:,
+    [(); { meta_num_slots_global!(CO_ALLOC_PREF) }]:,
 {
     type Item = T;
 
@@ -1563,9 +1566,10 @@ where
 }
 
 #[stable(feature = "drain", since = "1.6.0")]
-impl<T, const COOP_PREFERRED: bool> DoubleEndedIterator for Drain<'_, T, COOP_PREFERRED>
+#[allow(unused_braces)]
+impl<T, const CO_ALLOC_PREF: CoAllocPref> DoubleEndedIterator for Drain<'_, T, CO_ALLOC_PREF>
 where
-    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:,
+    [(); { meta_num_slots_global!(CO_ALLOC_PREF) }]:,
 {
     #[inline]
     fn next_back(&mut self) -> Option<T> {
@@ -1574,18 +1578,20 @@ where
 }
 
 #[stable(feature = "drain", since = "1.6.0")]
-impl<T, const COOP_PREFERRED: bool> ExactSizeIterator for Drain<'_, T, COOP_PREFERRED>
+#[allow(unused_braces)]
+impl<T, const CO_ALLOC_PREF: CoAllocPref> ExactSizeIterator for Drain<'_, T, CO_ALLOC_PREF>
 where
-    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:,
+    [(); { meta_num_slots_global!(CO_ALLOC_PREF) }]:,
 {
     fn is_empty(&self) -> bool {
         self.iter.is_empty()
     }
 }
 
 #[stable(feature = "fused", since = "1.26.0")]
-impl<T, const COOP_PREFERRED: bool> FusedIterator for Drain<'_, T, COOP_PREFERRED> where
-    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:
+#[allow(unused_braces)]
+impl<T, const CO_ALLOC_PREF: CoAllocPref> FusedIterator for Drain<'_, T, CO_ALLOC_PREF> where
+    [(); { meta_num_slots_global!(CO_ALLOC_PREF) }]:
 {
 }
 
@@ -1676,7 +1682,7 @@ impl<T: Ord, const N: usize> From<[T; N]> for BinaryHeap<T> {
 
 #[stable(feature = "binary_heap_extras_15", since = "1.5.0")]
 #[allow(unused_braces)]
-impl<T> From<BinaryHeap<T>> for Vec<T, Global, { DEFAULT_COOP_PREFERRED!() }> {
+impl<T> From<BinaryHeap<T>> for Vec<T, Global, { CO_ALLOC_PREF_DEFAULT!() }> {
     /// Converts a `BinaryHeap<T>` into a `Vec<T>`.
     ///
     /// This conversion requires no data movement or allocation, and has