From 9eec6a1ebff49b7a7cafb81c7d40f0d45548fb29 Mon Sep 17 00:00:00 2001 From: Techcable Date: Thu, 25 Apr 2024 13:55:20 -0700 Subject: [PATCH 01/35] Initial commit This is intended to be a replacement/redesign for zerogc --- .gitignore | 1 + Cargo.lock | 134 ++++++++++ Cargo.toml | 17 ++ libs/macros/Cargo.toml | 15 ++ libs/macros/src/collect_impl.rs | 425 ++++++++++++++++++++++++++++++++ libs/macros/src/helpers.rs | 156 ++++++++++++ libs/macros/src/lib.rs | 13 + src/collect.rs | 23 ++ src/collect/collections.rs | 20 ++ src/collect/macros.rs | 26 ++ src/collect/primitives.rs | 3 + src/context.rs | 192 +++++++++++++++ src/context/young.rs | 78 ++++++ src/gcptr.rs | 75 ++++++ src/lib.rs | 12 + src/utils.rs | 39 +++ src/utils/layout_helpers.rs | 75 ++++++ 17 files changed, 1304 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 libs/macros/Cargo.toml create mode 100644 libs/macros/src/collect_impl.rs create mode 100644 libs/macros/src/helpers.rs create mode 100644 libs/macros/src/lib.rs create mode 100644 src/collect.rs create mode 100644 src/collect/collections.rs create mode 100644 src/collect/macros.rs create mode 100644 src/collect/primitives.rs create mode 100644 src/context.rs create mode 100644 src/context/young.rs create mode 100644 src/gcptr.rs create mode 100644 src/lib.rs create mode 100644 src/utils.rs create mode 100644 src/utils/layout_helpers.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..2790a0d --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + +[[package]] +name = "arbitrary-int" +version = "1.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c84fc003e338a6f69fbd4f7fe9f92b535ff13e9af8997f3b14b6ddff8b1df46d" + +[[package]] +name = "bitbybit" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb157f9753a7cddfcf4a4f5fed928fbf4ce1b7b64b6bcc121d7a9f95d698997b" +dependencies = [ + "arbitrary-int", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "proc-macro-kwargs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb7ae601c212763e27833573c9ed3d6ce10734c09fffe359b83c8b0c49b7634" +dependencies = [ + "indexmap", + "proc-macro-kwargs-derive", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro-kwargs-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08fb124d7373d276571ef16e4e2e2181d68ac173b0f21a35a23c12c1e8c97b2d" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "2.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "zerog-next-macros" +version = "0.1.0-alpha.1" +dependencies = [ + "indexmap", + "proc-macro-kwargs", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerogc-next" +version = "0.1.0-alpha.1" +dependencies = [ + "allocator-api2", + "bitbybit", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..b2811f2 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "zerogc-next" +description = "Redesign of the zerogc API" +version.workspace = true +edition.workspace = true + +[dependencies] +allocator-api2 = "0.2.18" +bitbybit = "1.3.2" + +[workspace] +resolver = "2" +members = [".", "libs/*"] + +[workspace.package] +version = "0.1.0-alpha.1" +edition = "2021" diff --git a/libs/macros/Cargo.toml b/libs/macros/Cargo.toml new file mode 100644 index 0000000..bcb38e3 --- /dev/null +++ b/libs/macros/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "zerog-next-macros" +description = "Procedural macros for zerogc-next" +version.workspace = true +edition.workspace = true + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "2", features = ["full"] } +quote = "1" +proc-macro2 = "1" +proc-macro-kwargs = "0.2" +indexmap = "2" diff --git a/libs/macros/src/collect_impl.rs b/libs/macros/src/collect_impl.rs new file mode 100644 index 0000000..32949bf --- /dev/null +++ b/libs/macros/src/collect_impl.rs @@ -0,0 +1,425 @@ +//! The implementation of `unsafe_collect_impl!` +//! +//! Loosely based on `zerogc_derive::unsafe_gc_impl!` version `0.2.0-alpha.7`. +//! See here for the zerogc implementation: +//! +//! +//! A significant portion of code is copied from that file. +//! There is no copyright issue because I am the only author. +use indexmap::{indexmap, IndexMap}; +use proc_macro2::{Ident, Span, TokenStream}; +use proc_macro_kwargs::parse::{NestedDict, NestedList, Syn}; +use proc_macro_kwargs::{MacroArg, MacroKeywordArgs}; +use quote::quote; +use std::process::id; +use syn::parse::ParseStream; +use syn::{ + braced, parse_quote, Error, Expr, GenericParam, Generics, Lifetime, Path, Token, Type, + TypeParamBound, WhereClause, WherePredicate, +}; + +use crate::helpers::{self, zerogc_next_crate}; + +fn empty_clause() -> WhereClause { + WhereClause { + predicates: Default::default(), + where_token: Default::default(), + } +} + +#[derive(Debug, MacroKeywordArgs)] +pub struct MacroInput { + /// The target type we are implementing + /// + /// This has unconstrained use of the parameters defined in `params` + #[kwarg(rename = "target")] + pub target_type: Type, + /// The generic parameters (both types and lifetimes) that we want to + /// declare for each implementation + /// + /// This must not conflict with our internal generic names ;) + params: NestedList, + /// Custom bounds provided for each + /// + /// All of these bounds are optional. + /// This option can be omitted, + /// giving the equivalent of `bounds = {}` + #[kwarg(optional)] + bounds: CustomBounds, + /// Requirements for implementing `NullCollect` + /// + /// This is unsafe (obviously) and has no static checking. + null_collect: Option, + /// The associated type `Collect::Collected<'newgc>` + /// + /// Implicitly takes a `'newgc` parameter + collected_type: Type, + /// A (constant) expression determining whether the type needs to be collected + #[kwarg(rename = "NEEDS_COLLECT")] + needs_collect: Expr, + /// The fixed `CollectorId`s the type should implement `Collect` for, + /// or `*` if the type can work with any collector. + /// + /// If multiple collector ids are specified, + /// each one must have a specific lifetime. + #[kwarg(optional, rename = "collector_id")] + collector_id_info: CollectorIdInfo, + /// The actual implementation of the `Collect::copy_collect` method + /// + /// Because of the way the trait is designed, + /// this is a relatively safe method. + /// + /// This code is implicitly skipped if `Self::NEEDS_COLLECt` is `false` + #[kwarg(rename = "copy_collect")] + copy_collect_closure: CollectImplClosure, +} +fn generic_collector_id_param() -> Ident { + parse_quote!(AnyCollectorId) +} +impl MacroInput { + fn basic_generics(&self) -> Generics { + let mut generics = Generics::default(); + generics.params.extend(self.params.iter().cloned()); + generics + } + fn setup_collector_id_generics(&self, target: &mut Generics) -> syn::Result { + match self.collector_id_info { + CollectorIdInfo::Any => { + let zerogc_next_crate = zerogc_next_crate(); + let collector_id_param = generic_collector_id_param(); + target.params.push(GenericParam::Type(parse_quote!( + #collector_id_param: #zerogc_next_crate::CollectorId + ))); + Ok(syn::Path::from(collector_id_param)) + } + CollectorIdInfo::Specific { ref map } => { + let (collector_id, _lifetime) = match map.len() { + 0 => { + return Err(Error::new( + Span::call_site(), + "Using 'specific' CollectorId, but none specified", + )); + } + 1 => map.get_index(0).unwrap(), + _ => { + let mut errors = Vec::new(); + for (pth, _) in map.iter() { + errors.push(Error::new_spanned( + pth, + "Multiple `CollectorId`s is currently unsupported", + )); + } + return Err(helpers::combine_errors(errors).unwrap_err()); + } + }; + Ok(collector_id.clone()) + } + } + } + pub fn expand_output(&self) -> Result { + let zerogc_next_crate = zerogc_next_crate(); + let target_type = &self.target_type; + let collect_impl = self.expand_collect_impl()?; + let null_collect_clause = match self.null_collect { + TraitRequirements::Always { span: _ } => Some(empty_clause()), + TraitRequirements::Where(ref clause) => Some(clause.clone()), + TraitRequirements::Never { span: _ } => None, + }; + let null_collect_impl: TokenStream = if let Some(null_collect_clause) = null_collect_clause + { + let mut generics = self.basic_generics(); + let collector_id = self.setup_collector_id_generics(&mut generics)?; + generics + .make_where_clause() + .predicates + .extend(null_collect_clause.predicates); + let (impl_generics, _, where_clause) = generics.split_for_impl(); + quote! { + unsafe impl #impl_generics #zerogc_next_crate::NullCollect<#collector_id> for #target_type + #where_clause {} + } + } else { + quote!() + }; + Ok(quote! { + #collect_impl + #null_collect_impl + }) + } + fn expand_collect_impl(&self) -> Result { + let zerogc_next_crate = zerogc_next_crate(); + let target_type = &self.target_type; + let mut generics: syn::Generics = self.basic_generics(); + let collector_id = self.setup_collector_id_generics(&mut generics); + { + let clause = self.bounds.where_clause_collect(&self.params.elements)?; + generics + .make_where_clause() + .predicates + .extend(clause.predicates); + } + // NOTE: The `expand` method implicitly skips body if `!Self::NEEDS_COLLECT` + let collect_impl = self.copy_collect_closure.expand(); + let (impl_generics, _, where_clause) = generics.split_for_impl(); + let needs_collect_const = { + let expr = &self.needs_collect; + quote!(const NEEDS_TRACE: bool = { + // Import the trait so we can access `T::NEEDS_COLLECT` + use #zerogc_next_crate::Collect; + #expr + };) + }; + let collected_type = &self.collected_type; + Ok(quote! { + unsafe impl #impl_generics #zerogc_next_crate::Collect<#collector_id> for #target_type #where_clause { + type Collected<'newgc> = #collected_type; + #needs_collect_const + + #[inline] // TODO: Should this be unconditional? + #[allow(dead_code)] // possible if `Self::NEEDS_COLLECT` is always false + fn copy_collect<'newgc>(self, context: &mut #zerogc_next_crate::context::CollectContext<'newgc>) -> Self::Collected<'newgc> { + #collect_impl + } + } + }) + } +} + +/// Extra bounds +#[derive(Default, Debug, MacroKeywordArgs)] +pub struct CustomBounds { + /// Additional bounds on the `Collect` implementation + #[kwarg(optional, rename = "Collect")] + collect: Option, +} + +impl CustomBounds { + fn where_clause_collect( + &self, + generic_params: &[GenericParam], + ) -> Result { + match self.collect { + Some(TraitRequirements::Never { span }) => { + return Err(syn::Error::new(span, "Collect must always be implemented")) + } + Some(TraitRequirements::Always) => Ok(empty_clause()), // No requirements + Some(TraitRequirements::Where(ref explicit)) => Ok(explicit.clone()), + None => { + // generate the implicit requirements + let zerogc_next_crate = zerogc_next_crate(); + Ok(create_clause_with_default( + &self.collect, + generic_params, + vec![parse_quote!(#zerogc_next_crate::Collect)], + ) + .expect("Already checked for TraitRequirements::Never")) + } + } + } +} + +/// The requirements to implement a trait +/// +/// In addition to a where clause, you can specify `always` for unconditional +/// implementation and `never` to forbid generated implementations +#[derive(Clone, Debug)] +pub enum TraitRequirements { + /// The trait should never be implemented + Never { span: Span }, + /// The trait should only be implemented if + /// the specified where clause is satisfied + Where(WhereClause), + /// The trait should always be implemented + Always { span: Span }, +} + +impl MacroArg for TraitRequirements { + fn parse_macro_arg(input: ParseStream) -> syn::Result { + if input.peek(syn::Ident) { + let ident = input.parse::()?; + let span = ident.span(); + if ident == "always" { + Ok(TraitRequirements::Always { span }) + } else if ident == "never" { + Ok(TraitRequirements::Never { span }) + } else { + Err(Error::new( + ident.span(), + "Invalid identifier for `TraitRequirement`", + )) + } + } else if input.peek(syn::token::Brace) { + let inner; + syn::braced!(inner in input); + Ok(TraitRequirements::Where(inner.parse::()?)) + } else { + Err(input.error("Invalid `TraitRequirement`")) + } + } +} + +#[derive(Clone, Debug)] +pub enum CollectorIdInfo { + Any, + Specific { map: IndexMap }, +} +impl Default for CollectorIdInfo { + fn default() -> Self { + CollectorIdInfo::Any + } +} +impl CollectorIdInfo { + /// Create info from a single `CollectorId`, + /// implicitly assuming its lifetime is `'gc` + pub fn single(path: Path) -> Self { + CollectorIdInfo::Specific { + map: indexmap! { + path => parse_quote!('gc) + }, + } + } +} +impl MacroArg for CollectorIdInfo { + fn parse_macro_arg(stream: ParseStream) -> syn::Result { + if stream.peek(Token![*]) { + stream.parse::()?; + Ok(CollectorIdInfo::Any) + } else if stream.peek(syn::Ident) { + Ok(CollectorIdInfo::single(stream.parse()?)) + } else if stream.peek(syn::token::Brace) { + let inner = NestedDict::parse_macro_arg(stream)?; + Ok(CollectorIdInfo::Specific { + map: inner.elements, + }) + } else { + Err(stream.error("Expected either `*`, a path, or a map of Path => Lifetime")) + } + } +} + +#[derive(Debug, Clone)] +pub struct KnownArgClosure { + body: TokenStream, + brace: ::syn::token::Brace, +} +impl KnownArgClosure { + pub fn parse_with_fixed_args(input: ParseStream, fixed_args: &[&str]) -> syn::Result { + let arg_start = input.parse::()?.span; + let mut actual_args = Vec::new(); + while !input.peek(Token![|]) { + // Use 'parse_any' to accept keywords like 'self' + actual_args.push(Ident::parse_any(input)?); + if input.peek(Token![|]) { + break; // done + } else { + input.parse::()?; + } + } + let arg_end = input.parse::()?.span; + if actual_args.len() != fixed_args.len() { + return Err(Error::new( + arg_start.join(arg_end).unwrap(), + format!( + "Expected {} args but got {}", + fixed_args.len(), + actual_args.len() + ), + )); + } + for (index, (actual, &expected)) in actual_args.iter().zip(fixed_args).enumerate() { + if *actual != expected { + return Err(Error::new( + actual.span(), + format!("Expected arg #{} to be named {:?}", index, expected), + )); + } + } + if !input.peek(syn::token::Brace) { + return Err(input.error("Expected visitor closure to be braced")); + } + let body; + let brace = braced!(body in input); + let body = body.parse::()?; + Ok(KnownArgClosure { + body: quote!({ #body }), + brace, + }) + } +} +#[derive(Debug, Clone)] +pub struct CollectImplClosure(KnownArgClosure); +impl CollectImplClosure { + pub fn expand(&self) -> TokenStream { + let body = &self.0.body; + quote! { + if !Self::NEEDS_COLLECT { + return context.null_copy(self); + } + #body + } + } +} +impl MacroArg for CollectImplClosure { + fn parse_macro_arg(input: ParseStream) -> syn::Result { + Ok(CollectImplClosure(KnownArgClosure::parse_with_fixed_args( + input, + &["self", "context"], + )?)) + } +} + +fn create_clause_with_default( + target: &Option, + generic_params: &[GenericParam], + default_bounds: Vec, +) -> Result { + create_clause_with_default_and_ignored(target, generic_params, default_bounds, None) +} +fn create_clause_with_default_and_ignored( + target: &Option, + generic_params: &[GenericParam], + default_bounds: Vec, + mut should_ignore: Option<&mut dyn FnMut(&GenericParam) -> bool>, +) -> Result { + Ok(match *target { + Some(TraitRequirements::Never { span }) => { + // do not implement + return Err(NeverClauseError { span }); + } + Some(TraitRequirements::Where(ref explicit)) => explicit.clone(), + Some(TraitRequirements::Always { span: _ }) => { + // Absolutely no conditions on implementation + empty_clause() + } + None => { + let mut where_clause = empty_clause(); + // Infer bounds for all params + for param in generic_params { + if let Some(ref mut should_ignore) = should_ignore { + if should_ignore(param) { + continue; + } + } + if let GenericParam::Type(ref t) = param { + let ident = &t.ident; + where_clause + .predicates + .push(WherePredicate::Type(syn::PredicateType { + bounded_ty: parse_quote!(#ident), + colon_token: Default::default(), + bounds: default_bounds.iter().cloned().collect(), + lifetimes: None, + })) + } + } + where_clause + } + }) +} + +/// Indicates that [`TraitRequirements::Never`] was encountered, +/// explicitly disabling the clause +#[derive(Debug)] +struct NeverClauseError { + span: Span, +} diff --git a/libs/macros/src/helpers.rs b/libs/macros/src/helpers.rs new file mode 100644 index 0000000..10f3d0c --- /dev/null +++ b/libs/macros/src/helpers.rs @@ -0,0 +1,156 @@ +//! Helpers for macros, potentially shared across implementations. +use proc_macro2::TokenStream; +use quote::quote; +use syn::spanned::Spanned; +use syn::{Error, GenericArgument, Generics, Type}; + +pub fn combine_errors(errors: Vec) -> Result<(), syn::Error> { + let mut iter = errors.into_iter(); + if let Some(mut first) = iter.next() { + for other in iter { + first.combine(other); + } + Err(first) + } else { + Ok(()) + } +} + +pub fn rewrite_type( + target: &Type, + target_type_name: &str, + rewriter: &mut dyn FnMut(&Type) -> Option, +) -> Result { + if let Some(explicitly_rewritten) = rewriter(target) { + return Ok(explicitly_rewritten); + } + let mut target = target.clone(); + match target { + Type::Paren(ref mut inner) => { + *inner.elem = rewrite_type(&inner.elem, target_type_name, rewriter)? + } + Type::Group(ref mut inner) => { + *inner.elem = rewrite_type(&inner.elem, target_type_name, rewriter)? + } + Type::Reference(ref mut target) => { + // TODO: Lifetime safety? + // Rewrite reference target + *target.elem = rewrite_type(&target.elem, target_type_name, rewriter)? + } + Type::Path(syn::TypePath { + ref mut qself, + ref mut path, + }) => { + *qself = qself + .clone() + .map::, _>(|mut qself| { + qself.ty = + Box::new(rewrite_type(&*qself.ty, target_type_name, &mut *rewriter)?); + Ok(qself) + }) + .transpose()?; + path.segments = path + .segments + .iter() + .cloned() + .map(|mut segment| { + // old_segment.ident is ignored... + match segment.arguments { + syn::PathArguments::None => {} // Nothing to do here + syn::PathArguments::AngleBracketed(ref mut generic_args) => { + for arg in &mut generic_args.args { + match arg { + GenericArgument::Lifetime(_) | GenericArgument::Const(_) => {} + GenericArgument::Type(ref mut generic_type) => { + *generic_type = rewrite_type( + generic_type, + target_type_name, + &mut *rewriter, + )?; + } + // TODO: Handle other generic ? + _ => { + return Err(Error::new( + arg.span(), + format!( + "Unable to handle generic arg while rewriting as a {}", + target_type_name + ), + )) + } + } + } + } + syn::PathArguments::Parenthesized(ref mut paran_args) => { + return Err(Error::new( + paran_args.span(), + "TODO: Rewriting parenthesized (fn-style) args", + )); + } + } + Ok(segment) + }) + .collect::>()?; + } + _ => { + return Err(Error::new( + target.span(), + format!( + "Unable to rewrite type as a `{}`: {}", + target_type_name, + quote!(#target) + ), + )) + } + } + Ok(target) +} + +/// This refers to the zerogc_next crate, +/// equivalent to `$crate` for proc_macros +pub fn zerogc_next_crate() -> TokenStream { + /* + * TODO: There is no way to emulate for `$crate` for proc_macros + * + * Instead we re-export `extern crate self as zerogc_next` at the root of the zerogc_next crate. + */ + quote!(zerogc_next::) +} + +// Sort the parameters so that lifetime parameters come before +/// type parameters, and type parameters come before const paramaters +pub fn sort_params(generics: &mut Generics) { + #[derive(Eq, PartialEq, Ord, PartialOrd, Copy, Clone, Debug)] + enum ParamOrder { + Lifetime, + Type, + Const, + } + let mut pairs = std::mem::take(&mut generics.params) + .into_pairs() + .collect::>(); + use syn::punctuated::Pair; + pairs.sort_by_key(|pair| match pair.value() { + syn::GenericParam::Lifetime(_) => ParamOrder::Lifetime, + syn::GenericParam::Type(_) => ParamOrder::Type, + syn::GenericParam::Const(_) => ParamOrder::Const, + }); + /* + * NOTE: The `Pair::End` can only come at the end. + * Now that we've sorted, it's possible the old ending + * could be in the first position or some other position + * before the end. + * + * If that's the case, then add punctuation to the end. + */ + if let Some(old_ending_index) = pairs.iter().position(|p| p.punct().is_none()) { + if old_ending_index != pairs.len() - 1 { + let value = pairs.remove(old_ending_index).into_value(); + pairs.insert( + old_ending_index, + Pair::Punctuated(value, Default::default()), + ); + } + } + generics.params = pairs.into_iter().collect(); +} diff --git a/libs/macros/src/lib.rs b/libs/macros/src/lib.rs new file mode 100644 index 0000000..9adb2e8 --- /dev/null +++ b/libs/macros/src/lib.rs @@ -0,0 +1,13 @@ +use proc_macro2::TokenStream; + +mod collect_impl; +pub(crate) mod helpers; + +#[proc_macro] +pub fn unsafe_collect_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let parsed = syn::parse_macro_input!(input as collect_impl::MacroInput); + let res = parsed + .expand_output() + .unwrap_or_else(|e| e.to_compile_error()); + res.into() +} diff --git a/src/collect.rs b/src/collect.rs new file mode 100644 index 0000000..aafcd44 --- /dev/null +++ b/src/collect.rs @@ -0,0 +1,23 @@ +//! Defines the [`Collect`] trait and implements it for several types + +mod collections; +pub mod macros; +mod primitives; + +use crate::context::CollectContext; +use crate::CollectorId; +use std::mem::ManuallyDrop; +use std::ptr::NonNull; + +pub unsafe trait Collect { + type Collected<'newgc>: Collect; + const NEEDS_COLLECT: bool; + + unsafe fn collect_inplace(target: NonNull, context: &mut CollectContext<'_, Id>); +} + +pub unsafe trait NullCollect: Collect {} + +// +// macros +// diff --git a/src/collect/collections.rs b/src/collect/collections.rs new file mode 100644 index 0000000..ea14159 --- /dev/null +++ b/src/collect/collections.rs @@ -0,0 +1,20 @@ +use crate::collect::{Collect, NullCollect}; +use crate::context::CollectContext; +use crate::CollectorId; +use std::ptr::NonNull; + +unsafe impl> Collect for Vec { + type Collected<'newgc> = Vec>; + const NEEDS_COLLECT: bool = T::NEEDS_COLLECT; + + #[inline] + unsafe fn collect_inplace(target: NonNull, context: &mut CollectContext<'_, Id>) { + if Self::NEEDS_COLLECT { + for val in target.as_ref().iter() { + T::collect_inplace(NonNull::from(val), context); + } + } + } +} + +unsafe impl> NullCollect for Vec {} diff --git a/src/collect/macros.rs b/src/collect/macros.rs new file mode 100644 index 0000000..0d75fc4 --- /dev/null +++ b/src/collect/macros.rs @@ -0,0 +1,26 @@ +use crate::CollectContext; +use std::ptr::NonNull; +#[macro_export] +macro_rules! static_null_trace { + ($($target:ident),*) => { + $($crate::static_null_trace!(@single $target);)* + }; + (@single $target:ident) => { + unsafe impl $crate::Collect for $target { + type Collected<'newgc> = Self; + const NEEDS_COLLECT: bool = { + $crate::collect::macros::helpers::assert_static_lifetime::(); + false + }; + + #[inline(always)] // does nothing + unsafe fn collect_inplace(_target: NonNull, _context: &mut CollectContext<'_, Id>) {} + } + unsafe impl $crate::NullCollect for $target {} + }; +} + +#[doc(hidden)] +pub mod helpers { + pub const fn assert_static_lifetime() {} +} diff --git a/src/collect/primitives.rs b/src/collect/primitives.rs new file mode 100644 index 0000000..2c5c3cb --- /dev/null +++ b/src/collect/primitives.rs @@ -0,0 +1,3 @@ +use crate::static_null_trace; + +static_null_trace!(i8, i16, i32, i64, isize, u8, u16, u32, u64, usize, char, bool, String); diff --git a/src/context.rs b/src/context.rs new file mode 100644 index 0000000..a782583 --- /dev/null +++ b/src/context.rs @@ -0,0 +1,192 @@ +mod young; + +use crate::gcptr::Gc; +use crate::utils::LayoutExt; +use crate::{Collect, NullCollect}; +use allocator_api2::alloc::Allocator; +use bitbybit::bitfield; +use bumpalo::Bump; +use std::alloc::{Layout, LayoutError}; +use std::any::TypeId; +use std::cmp; +use std::fmt::Debug; +use std::mem::ManuallyDrop; +use std::ptr::NonNull; + +pub enum SingletonStatus { + /// The singleton is thread-local. + /// + /// This is slower to resolve, + /// but can be assumed to be unique + /// within the confines of an individual thread. + /// + /// This implies the [`CollectorId`] is `!Send` + ThreadLocal, + /// The singleton is global. + /// + /// This is faster to resolve, + /// and can further assume to be unique + /// across the entire program. + Global, +} + +pub unsafe trait CollectorId: Copy + Debug + Eq + 'static { + const SINGLETON: Option; + unsafe fn resolve_collector(&self) -> *mut GarbageCollector; + + unsafe fn summon_singleton() -> Option; +} + +pub struct GarbageCollector { + allocator: Bump, + id: Id, +} +impl GarbageCollector { + #[inline] + pub fn id(&self) -> Id { + self.id + } + + #[inline(always)] + pub fn alloc>(&self, value: T) -> Gc<'_, T, Id> { + self.alloc_with(|| value) + } + + #[inline(always)] + pub fn alloc_with>(&self, func: impl FnOnce() -> T) -> Gc<'_, T, Id> { + let allocatd = self.allocator.alloc_layout(); + self.gc + } + unsafe fn init_alloc(ptr: NonNull>) -> Gc<'_, T, Id> {} +} + +pub(crate) struct GcTypeInfo { + layout_info: GcLayout, + drop_func: Option, +} +trait TypeIdInit> { + const TYPE_INFO_VAL: GcTypeInfo = { + let layout = GcHeader::determine_layout(Layout::new::()); + let drop_func = if std::mem::needs_drop::() { + unsafe { + Some(std::mem::transmute::<_, unsafe fn(*mut ())>( + std::ptr::drop_in_place as unsafe fn(*mut T), + )) + } + } else { + None + }; + GcTypeInfo { + layout_info: layout, + drop_func, + } + }; + const TYPE_INFO_REF: &'static GcTypeInfo = &Self::TYPE_INFO_VAL; +} + +#[bitfield(u32)] +struct GcStateBits { + #[bit(0)] + forwarded: bool, + #[bit(1)] + yong_generation: bool, +} +union HeaderMetadata { + type_info: &'static GcTypeInfo, + forward_ptr: NonNull>, +} +union AllocInfo { + /// The number of bytes before the previous object header. + /// + /// This is used to iterate over objects in the young generation. + /// + /// If there is no previous object in this chunk, this is zero. + /// + /// This is guaranteed to be equal to `self.type_info().layout.overall_layout.size()` + /// of the previous object. + prev_object_offset: u32, + /// The index of the object within the vector of live objects. + /// + /// This is used in the old generation. + live_object_index: u32, +} +#[repr(C)] +pub(crate) struct GcHeader { + state_bits: GcStateBits, + alloc_info: AllocInfo, + metadata: HeaderMetadata, + collector_id: Id, +} +impl GcHeader { + const HEADER_LAYOUT: Layout = Layout::new::(); + + #[inline] + pub fn id(&self) -> Id { + self.collector_id + } + + #[inline] + unsafe fn resolve_type_info(&self) -> &'static GcTypeInfo { + if self.state_bits.forwarded() { + let forward_ptr = self.metadata.forward_ptr; + let forward_header = forward_ptr.as_ref(); + debug_assert!(!forward_header.state_bits.forwarded()); + forward_header.metadata.type_info + } else { + self.metadata.type_info + } + } + + #[inline] + pub const fn determine_layout(type_layout: Layout) -> GcLayout { + let Ok((overall_layout, value_offset)) = LayoutExt(Self::HEADER_LAYOUT).extend(type_layout) + else { + panic!("Layout error") + }; + let overall_layout = LayoutExt(overall_layout).pad_to_align(); + GcLayout { + value_offset, + overall_layout, + type_layout, + } + } +} + +/// TODO: We should cap the alignment of gc-allocated types +/// +/// This would be a win by avoiding runtime size calculations +/// and making the `value_offset` a compile-time constant +#[derive(Eq, PartialEq, Debug)] +pub(crate) struct GcLayout { + pub type_layout: Layout, + pub overall_layout: Layout, + /// The offset between the end of the header and the beginning of the value + pub value_offset: usize, +} + +pub struct CollectContext<'newgc, Id: CollectorId> { + id: Id, +} +impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { + #[inline] + pub fn id(&self) -> Id { + self.id + } + #[inline] + unsafe fn trace_gc_ptr(&mut self, target: NonNull>) { + let target = target.as_ptr(); + target.cast().write(self.collect_gc_ptr(target.read())); + } + unsafe fn collect_gc_ptr<'gc, T: Collect>( + &mut self, + target: Gc<'gc, T, Id>, + ) -> Gc<'newgc, T::Collected<'newgc>, Id> { + debug_assert_eq!(target.id(), self.id()); + let header = target.header(); + + if header.state_bits.forwarded() { + header.metadata.forward_ptr + } + crate::utils::transmute_arbitrary(target) + } +} diff --git a/src/context/young.rs b/src/context/young.rs new file mode 100644 index 0000000..725072e --- /dev/null +++ b/src/context/young.rs @@ -0,0 +1,78 @@ +use crate::context::{GcHeader, GcLayout, GcTypeInfo}; +use crate::utils::LayoutExt; +use crate::CollectorId; +use std::alloc::Layout; +use std::cell::Cell; +use std::ptr::NonNull; + +/// A yong-generation object-space +/// +/// If copying is in progress, +/// there may be two young generations for a single collector. +/// +/// The design of the allocator is heavily based on [`bumpalo`](https://crates.io/crates/bumpalo) +pub struct YoungGenerationSpace { + current_chunk: Cell>, + current_chunk_next_ptr: Cell>, + current_chunk_end_ptr: Cell>, + collector_id: Id, +} +impl YoungGenerationSpace { + #[inline(always)] + pub fn alloc_uninit( + &self, + layout_info: GcLayout, + type_info: &'static GcTypeInfo, + ) -> (*mut GcHeader, NonNull) { + debug_assert_eq!(layout_info, type_info.layout_info); + } + fn ensure_chunk(&self, layout: Layout) -> Layout { + x + } +} + +#[repr(C, align(16))] +pub struct ChunkHeader { + initial_object: Option>, + final_allocation_ptr: Option>, + prev_chunk: Option>, + data_size: usize, +} +impl ChunkHeader { + #[inline] + fn data_start_ptr(&self) -> NonNull { + unsafe { + NonNull::new_unchecked( + (self as *const Self as *const u8).add(Self::DATA_OFFSET) as *mut u8 + ) + } + } + #[inline] + fn data_end_ptr(&self) -> NonNull { + unsafe { NonNull::new_unchecked(self.start_ptr().as_ptr().add(self.data_size)) } + } + #[inline] + fn overall_layout(&self) -> Layout { + assert_eq!(Self::DATA_OFFSET, Self::HEADER_LAYOUT.size()); + unsafe { + Layout::from_size_align_unchecked( + Self::DATA_OFFSET.unchecked_add(self.data_size), + Self::DATA_ALIGNMENT, + ) + .unwrap() + } + } + #[inline] + fn data_layout(&self) -> Layout { + unsafe { Layout::from_size_align_unchecked(self.data_size, Self::DATA_ALIGNMENT) } + } + const HEADER_LAYOUT: Layout = Layout::new::(); + pub const DATA_OFFSET: usize = { + let this_layout = Self::HEADER_LAYOUT; + assert!(this_layout.align() <= Self::DATA_ALIGNMENT); + let padding = LayoutExt(this_layout).padding_needed_for(Self::DATA_ALIGNMENT); + this_layout.size() + padding + }; + /// This over-alignment should be good enough for everything... + pub const DATA_ALIGNMENT: usize = std::mem::align_of::(); +} diff --git a/src/gcptr.rs b/src/gcptr.rs new file mode 100644 index 0000000..7f46c08 --- /dev/null +++ b/src/gcptr.rs @@ -0,0 +1,75 @@ +use std::alloc::Layout; +use std::any::Any; +use std::marker::PhantomData; +use std::mem::ManuallyDrop; +use std::ops::Deref; +use std::ptr::NonNull; + +use crate::context::{GcHeader, GcTypeInfo}; +use crate::{Collect, CollectContext, CollectorId}; + +pub struct Gc<'gc, T, Id: CollectorId> { + ptr: NonNull, + marker: PhantomData<*const T>, + collect_marker: PhantomData<&'gc Id::Collector>, +} +impl<'gc, T, Id: CollectorId> Gc<'gc, T, Id> { + #[inline] + pub fn id(&self) -> Id { + match unsafe { Id::summon_singleton() } { + None => self.header().id(), + Some(id) => id, + } + } + + pub(crate) fn header(&self) -> &'_ GcHeader { + unsafe { + &*((self.ptr.as_ptr() as *mut u8) + .sub(const { GcHeader::determine_layout(Layout::new()).value_offset }) + as *mut GcHeader) + } + } + + pub(crate) fn type_info() -> &'static GcTypeInfo { + GcTypeInfo::of::() + } + + #[inline(always)] + pub unsafe fn from_raw_ptr(ptr: NonNull, id: Id) -> Self { + Gc { + ptr, + id, + marker: PhantomData, + collect_marker: PhantomData, + } + } +} +unsafe impl<'gc, Id: CollectorId, T: Collect> Collect for Gc<'gc, T, Id> { + type Collected<'newgc> = Gc<'newgc, T::Collected<'newgc>, Id>; + const NEEDS_COLLECT: bool = true; + + #[inline] + unsafe fn collect_inplace(target: NonNull, context: &mut CollectContext<'_, Id>) { + let resolved = target.as_ref(); + if matches!(Id::SINGLETON, None) && resolved.id() != context.id() { + return; + } + context.trace_gcptr(target) + } +} +impl<'gc, T, Id: CollectorId> Deref for Gc<'gc, T, Id> { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + unsafe { self.ptr.as_ref() } + } +} +impl<'gc, T, Id: CollectorId> Copy for Gc<'gc, T, Id> {} + +impl<'gc, T, Id: CollectorId> Clone for Gc<'gc, T, Id> { + #[inline] + fn clone(&self) -> Self { + *self + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..c8efd6a --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,12 @@ +#![feature( + inline_const, // will be stabilized in next release +)] +use std::fmt::Debug; + +pub mod collect; +pub mod context; +mod gcptr; +pub(crate) mod utils; + +pub use self::collect::{Collect, NullCollect}; +pub use self::context::{CollectContext, CollectorId, GarbageCollector}; diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..be6883a --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,39 @@ +pub(crate) mod layout_helpers; + +pub(crate) use self::layout_helpers::LayoutExt; +use crate::Collect; +use std::mem::ManuallyDrop; + +#[inline(always)] +pub unsafe fn transmute_arbitrary(val: T) -> U { + if std::mem::size_of::() != std::mem::size_of::() + || std::mem::align_of::() != std::mem::align_of::() + { + mismatch_transmute_size::() + } + ManuallyDrop::into_inner(std::ptr::read::>(&ManuallyDrop::new(val) + as *const ManuallyDrop + as *const ManuallyDrop)) +} + +#[cold] +fn mismatch_transmute_size() -> ! { + let original_type_name = std::any::type_name::(); + let collected_type_name = std::any::type_name::(); + let sizes = [ + ("sizes", std::mem::size_of::(), std::mem::size_of::()), + ( + "alignments", + std::mem::align_of::(), + std::mem::align_of::(), + ), + ]; + for (value_name, original_size, collected_size) in sizes { + assert_eq!( + original_size, collected_size, + "Mismatched {} between T `{}` and U `{}`", + value_name, original_type_name, collected_type_name + ); + } + unreachable!("sizes & alignments unexpectedly matched") +} diff --git a/src/utils/layout_helpers.rs b/src/utils/layout_helpers.rs new file mode 100644 index 0000000..6af2289 --- /dev/null +++ b/src/utils/layout_helpers.rs @@ -0,0 +1,75 @@ +//! Helpers for [`std::alloc::Layout`] +//! +//! Implementation mostly copied from the stdlib. + +use std::alloc::{Layout, LayoutError}; + +#[derive(Copy, Clone, Debug)] +pub struct LayoutExt(pub Layout); + +impl LayoutExt { + /// Copied from stdlib [`Layout::padding_needed_for`] + #[inline] + pub const fn padding_needed_for(&self, align: usize) -> usize { + let len = self.0.size(); + + let len_rounded_up = len.wrapping_add(align).wrapping_sub(1) & !align.wrapping_sub(1); + len_rounded_up.wrapping_sub(len) + } + + /// Copied from stdlib [`Layout::pad_to_align`] + /// + /// Adds trailing padding. + #[inline] + pub const fn pad_to_align(&self) -> Layout { + let pad = self.padding_needed_for(self.0.align()); + // This cannot overflow. See stdlib for details + let new_size = self.0.size() + pad; + + // SAFETY: padded size is guaranteed to not exceed `isize::MAX`. + unsafe { Layout::from_size_align_unchecked(new_size, self.align()) } + } + + /// Copied from stdlib [`Layout::extend`] + /// + /// Modified to be a `const fn` + /// + /// To correctly mimic a `repr(C)` struct, + /// you must call [`Self::pad_to_align`] to add trailing padding. + /// See stdlib docs for details. + #[inline] + pub const fn extend(&self, next: Layout) -> Result<(Layout, usize), LayoutError> { + let new_align = Self::const_max(self.0.align(), next.align()); + let pad = self.padding_needed_for(next.align()); + + let Some(offset) = self.size().checked_add(pad) else { + return LayoutError; + }; + let Some(new_size) = offset.checked_add(next.size()) else { + return LayoutError; + }; + + /* + * SAFETY: We checked size above, align already guaranteed to be power of two + * The advantage of a manual check over Layout::from_size_align + * is we skip the usize::is_power_of_two check. + */ + if new_size > Self::max_size_for_align(new_align) { + return Err(LayoutError); + } else { + Ok(( + unsafe { Layout::from_size_align_unchecked(new_size, new_align) }, + offset, + )) + } + } +} + +#[inline] +const fn const_max(first: usize, second: usize) -> usize { + if first > second { + first + } else { + second + } +} From b2b79ebf1b4b5097d3c6e43e87382b6f66e40191 Mon Sep 17 00:00:00 2001 From: Techcable Date: Thu, 25 Apr 2024 14:00:10 -0700 Subject: [PATCH 02/35] Add README --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..9ded601 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# zerogc-next (v0.3) +A safe garbage-collector API for rust. + +This is intended to be replacement/redesign for the [zerogc](https://github.com/DuckLogic/zerogc) API for the 0.3 release. The API used in v0.1/v0.2 is currently known to be unsound. + +It's as if types are being copied from ones with the old gc lifetime `'gc` into types with a new gc lifetime `'newgc`. + From 927e525bda9eb11b15eae18d88051437af8199ae Mon Sep 17 00:00:00 2001 From: Techcable Date: Mon, 29 Apr 2024 10:24:25 -0700 Subject: [PATCH 03/35] utils: Remove alignment check from transmute_arbitrary --- src/utils.rs | 82 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index be6883a..cebc573 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -2,38 +2,66 @@ pub(crate) mod layout_helpers; pub(crate) use self::layout_helpers::LayoutExt; use crate::Collect; +use crate::Collect; use std::mem::ManuallyDrop; +use std::mem::{self, ManuallyDrop}; +/// Transmute one type into another, +/// without doing compile-time checks for sizes. +/// +/// The difference between [`mem::transmute`] is that this function +/// does not attempt to check sizes at compile time. +/// In the regular [`mem::transmute`] function, an compile-time error occurs +/// if the sizes can't be proved equal at compile time. +/// This function does `assert_eq!` at runtime instead. +/// +/// In some contexts, the sizes of types are statically unknown, +/// so a runtime assertion is better than a static compile-time check. +/// +/// ## Safety +/// See [`mem::transmute`] for full details on safety. +/// +/// The sizes of the two types must exactly match, +/// but unlike [`mem::transmute`] this is not checked at compile time. +/// +/// Because transmute is a by-value operation, +/// the alignment of the transmuted values themselves is not a concern. #[inline(always)] -pub unsafe fn transmute_arbitrary(val: T) -> U { - if std::mem::size_of::() != std::mem::size_of::() - || std::mem::align_of::() != std::mem::align_of::() - { - mismatch_transmute_size::() +#[track_caller] // for the case where sizes don't match +pub unsafe fn transmute_arbitrary(val: Src) -> Dst { + const SIZE_MATCHES: bool = std::mem::size_of::() == std::mem::size_of::(); + if SIZE_MATCHES { + let src: ManuallyDrop = ManuallyDrop::new(val); + std::mem::transmute_copy(&src as &Src) + } else { + mismatch_transmute_sizes( + TransmuteTypeInfo::new::(), + TransmuteTypeInfo::new::(), + ) } - ManuallyDrop::into_inner(std::ptr::read::>(&ManuallyDrop::new(val) - as *const ManuallyDrop - as *const ManuallyDrop)) } -#[cold] -fn mismatch_transmute_size() -> ! { - let original_type_name = std::any::type_name::(); - let collected_type_name = std::any::type_name::(); - let sizes = [ - ("sizes", std::mem::size_of::(), std::mem::size_of::()), - ( - "alignments", - std::mem::align_of::(), - std::mem::align_of::(), - ), - ]; - for (value_name, original_size, collected_size) in sizes { - assert_eq!( - original_size, collected_size, - "Mismatched {} between T `{}` and U `{}`", - value_name, original_type_name, collected_type_name - ); +struct TransmuteTypeInfo { + size: usize, + type_name: &'static str, +} +impl TransmuteTypeInfo { + #[inline] + pub fn new() -> Self { + TransmuteTypeInfo { + size: std::mem::size_of::(), + type_name: std::any::type_name::(), + } } - unreachable!("sizes & alignments unexpectedly matched") +} + +#[cold] +#[track_caller] +fn mismatch_transmute_sizes(src: TransmuteTypeInfo, dst: TransmuteTypeInfo) -> ! { + assert_eq!( + src.size, dst.size, + "Mismatched size between Src `{}` and Dst `{}`", + src.type_name, dst.type_name + ); + unreachable!() // sizes actually match } From 283922309882db8f24d0d24398a40cfb0437cade Mon Sep 17 00:00:00 2001 From: Techcable Date: Mon, 29 Apr 2024 14:07:19 -0700 Subject: [PATCH 04/35] Use fixed alignment for GC types Simplifies size calculation. Use bumpalo for young-gen alloc --- Cargo.lock | 41 ++++- Cargo.toml | 4 + src/collect/macros.rs | 4 +- src/context.rs | 356 +++++++++++++++++++++++++++++------- src/context/young.rs | 168 +++++++++++------ src/gcptr.rs | 16 +- src/lib.rs | 2 + src/utils.rs | 15 +- src/utils/bumpalo_raw.rs | 29 +++ src/utils/layout_helpers.rs | 49 ++++- 10 files changed, 531 insertions(+), 153 deletions(-) create mode 100644 src/utils/bumpalo_raw.rs diff --git a/Cargo.lock b/Cargo.lock index 2790a0d..14a31f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,12 @@ dependencies = [ "syn", ] +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "equivalent" version = "1.0.1" @@ -34,9 +40,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "heck" @@ -81,9 +87,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" dependencies = [ "unicode-ident", ] @@ -99,15 +105,35 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.60" +version = "2.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "unicode-ident" version = "1.0.12" @@ -130,5 +156,8 @@ name = "zerogc-next" version = "0.1.0-alpha.1" dependencies = [ "allocator-api2", + "arbitrary-int", "bitbybit", + "bumpalo", + "thiserror", ] diff --git a/Cargo.toml b/Cargo.toml index b2811f2..89e98c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,8 +5,12 @@ version.workspace = true edition.workspace = true [dependencies] +bumpalo = "3.16" allocator-api2 = "0.2.18" bitbybit = "1.3.2" +arbitrary-int = "1.2.7" +thiserror = "1" + [workspace] resolver = "2" diff --git a/src/collect/macros.rs b/src/collect/macros.rs index 0d75fc4..926d50a 100644 --- a/src/collect/macros.rs +++ b/src/collect/macros.rs @@ -1,5 +1,3 @@ -use crate::CollectContext; -use std::ptr::NonNull; #[macro_export] macro_rules! static_null_trace { ($($target:ident),*) => { @@ -14,7 +12,7 @@ macro_rules! static_null_trace { }; #[inline(always)] // does nothing - unsafe fn collect_inplace(_target: NonNull, _context: &mut CollectContext<'_, Id>) {} + unsafe fn collect_inplace(_target: std::ptr::NonNull, _context: &mut crate::context::CollectContext<'_, Id>) {} } unsafe impl $crate::NullCollect for $target {} }; diff --git a/src/context.rs b/src/context.rs index a782583..0118dbe 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,18 +1,23 @@ -mod young; - -use crate::gcptr::Gc; -use crate::utils::LayoutExt; -use crate::{Collect, NullCollect}; -use allocator_api2::alloc::Allocator; -use bitbybit::bitfield; -use bumpalo::Bump; use std::alloc::{Layout, LayoutError}; use std::any::TypeId; +use std::cell::Cell; use std::cmp; use std::fmt::Debug; +use std::io::repeat; +use std::marker::PhantomData; use std::mem::ManuallyDrop; use std::ptr::NonNull; +use allocator_api2::alloc::Allocator; +use bitbybit::{bitenum, bitfield}; + +use crate::context::young::YoungGenerationSpace; +use crate::gcptr::Gc; +use crate::utils::LayoutExt; +use crate::{Collect, NullCollect}; + +mod young; + pub enum SingletonStatus { /// The singleton is thread-local. /// @@ -30,6 +35,13 @@ pub enum SingletonStatus { Global, } +/// An opaque identifier for a specific garbage collector. +/// +/// There is not necessarily a single global garbage collector. +/// There can be multiple ones as long as they have separate [`CollectorId`]s. +/// +/// ## Safety +/// This type must be `#[repr(C)`] and its alignment must be at most eight bytes. pub unsafe trait CollectorId: Copy + Debug + Eq + 'static { const SINGLETON: Option; unsafe fn resolve_collector(&self) -> *mut GarbageCollector; @@ -38,7 +50,6 @@ pub unsafe trait CollectorId: Copy + Debug + Eq + 'static { } pub struct GarbageCollector { - allocator: Bump, id: Id, } impl GarbageCollector { @@ -54,19 +65,116 @@ impl GarbageCollector { #[inline(always)] pub fn alloc_with>(&self, func: impl FnOnce() -> T) -> Gc<'_, T, Id> { - let allocatd = self.allocator.alloc_layout(); - self.gc + todo!() + } +} + +/// The layout of a "regular" (non-array) type +pub(crate) struct GcTypeLayout { + /// The layout of the underlying value + /// + /// INVARIANT: The maximum alignment is [`GcHeader::FIXED_ALIGNMENT`] + value_layout: Layout, + /// The overall size of the value including the header + /// and trailing padding. + overall_size: usize, + marker: PhantomData<&'static Id>, +} +impl GcTypeLayout { + #[inline] + pub const fn value_size(&self) -> usize { + self.value_layout.size() + } + + #[inline] + pub const fn value_align(&self) -> usize { + self.value_layout.align() + } + + #[inline] + pub const fn value_layout(&self) -> Layout { + self.value_layout + } + + #[inline] + pub const fn overall_layout(&self) -> Layout { + unsafe { + Layout::from_size_align_unchecked(self.overall_size, GcHeader::::FIXED_ALIGNMENT) + } + } + + //noinspection RsAssertEqual + const fn compute_overall_layout(value_layout: Layout) -> Layout { + let header_layout = GcHeader::::REGULAR_HEADER_LAYOUT; + let Ok((expected_overall_layout, value_offset)) = + LayoutExt(header_layout).extend(value_layout) + else { + unreachable!("layout overflow") + }; + assert!( + value_offset == GcHeader::::REGULAR_VALUE_OFFSET, + "Unexpected value offset" + ); + let res = LayoutExt(expected_overall_layout).pad_to_align(); + assert!( + res.align() == GcHeader::::FIXED_ALIGNMENT, + "Unexpected overall alignment" + ); + res + } + #[track_caller] + pub const fn from_value_layout(value_layout: Layout) -> Self { + assert!( + value_layout.align() <= GcHeader::::FIXED_ALIGNMENT, + "Alignment exceeds maximum", + ); + let overall_layout = Self::compute_overall_layout(value_layout); + GcTypeLayout { + value_layout, + overall_size: overall_layout.size(), + marker: PhantomData, + } + } +} + +#[repr(transparent)] +pub(crate) struct GcArrayTypeInfo { + /// The type info for the array's elements. + /// + /// This is stored as the first element to allow one-way + /// pointer casts from GcArrayTypeInfo -> GcTypeInfo. + /// This simulates OO-style inheritance. + element_type_info: GcTypeInfo, +} + +impl GcArrayTypeInfo { + //noinspection RsAssertEqual + #[inline] + pub const fn new>() -> &'static Self { + /* + * for the time being GcTypeInfo <--> GcArrayTypeInfo, + * so we just cast the pointers + */ + assert!(std::mem::size_of::() == std::mem::size_of::>()); + unsafe { + &*(GcTypeInfo::::new::() as *const GcTypeInfo as *const GcArrayTypeInfo) + } } - unsafe fn init_alloc(ptr: NonNull>) -> Gc<'_, T, Id> {} } -pub(crate) struct GcTypeInfo { - layout_info: GcLayout, +pub(crate) struct GcTypeInfo { + layout: GcTypeLayout, drop_func: Option, } +impl GcTypeInfo { + #[inline] + pub const fn new>() -> &'static Self { + >::TYPE_INFO_REF + } +} trait TypeIdInit> { - const TYPE_INFO_VAL: GcTypeInfo = { - let layout = GcHeader::determine_layout(Layout::new::()); + const TYPE_INFO_INIT_VAL: GcTypeInfo = { + let layout = GcTypeLayout::from_value_layout(Layout::new::()); let drop_func = if std::mem::needs_drop::() { unsafe { Some(std::mem::transmute::<_, unsafe fn(*mut ())>( @@ -76,49 +184,81 @@ trait TypeIdInit> { } else { None }; - GcTypeInfo { - layout_info: layout, - drop_func, - } + GcTypeInfo { layout, drop_func } }; - const TYPE_INFO_REF: &'static GcTypeInfo = &Self::TYPE_INFO_VAL; + const TYPE_INFO_REF: &'static GcTypeInfo = &Self::TYPE_INFO_INIT_VAL; } +struct GcTypeInitImpl; +impl> TypeIdInit for GcTypeInitImpl {} -#[bitfield(u32)] +#[derive(Debug, Eq, PartialEq)] +#[bitenum(u1, exhaustive = true)] +enum GenerationId { + Young = 0, + Old = 1, +} + +/// A bitfifeld +/// +/// ## Default +/// The `DEFAULT` value isn't valid here. +/// However, it currently needs to exist fo +/// the macro to generate the `builder` field +#[bitfield(u32, default = 0)] struct GcStateBits { - #[bit(0)] + #[bit(0, rw)] forwarded: bool, - #[bit(1)] - yong_generation: bool, + #[bit(1, rw)] + generation: GenerationId, + #[bit(2, rw)] + array: bool, } union HeaderMetadata { type_info: &'static GcTypeInfo, + array_type_info: &'static GcArrayTypeInfo, forward_ptr: NonNull>, } -union AllocInfo { - /// The number of bytes before the previous object header. +union AllocInfo { + /// The [overall size][`GcTypeLayout::overall_layout`] of this object. /// /// This is used to iterate over objects in the young generation. /// - /// If there is no previous object in this chunk, this is zero. + /// Objects whose size cannot fit into a `u32` + /// can never be allocated in the young generation. /// - /// This is guaranteed to be equal to `self.type_info().layout.overall_layout.size()` - /// of the previous object. - prev_object_offset: u32, + /// If this object is an array, + /// this is the overall size of + /// the header and all elements. + pub this_object_overall_size: u32, /// The index of the object within the vector of live objects. /// /// This is used in the old generation. - live_object_index: u32, + pub live_object_index: u32, } -#[repr(C)] +#[repr(C, align(8))] pub(crate) struct GcHeader { - state_bits: GcStateBits, - alloc_info: AllocInfo, - metadata: HeaderMetadata, - collector_id: Id, + pub state_bits: Cell, + pub alloc_info: AllocInfo, + pub metadata: HeaderMetadata, + /// The id for the collector where this object is allocated. + /// + /// If the collector is a singleton (either global or thread-local), + /// this will be a zero sized type. + /// + /// ## Safety + /// The alignment of this type must be smaller than [`GcHeader::FIXED_ALIGNMENT`]. + pub collector_id: Id, } impl GcHeader { - const HEADER_LAYOUT: Layout = Layout::new::(); + /// The fixed alignment for all GC types + /// + /// Allocating a type with an alignment greater than this is an error. + pub const FIXED_ALIGNMENT: usize = 8; + /// The fixed offset from the start of the GcHeader to a regular value + pub const REGULAR_VALUE_OFFSET: usize = std::mem::size_of::(); + pub const ARRAY_VALUE_OFFSET: usize = std::mem::size_of::>(); + const REGULAR_HEADER_LAYOUT: Layout = Layout::new::(); + const ARRAY_HEADER_LAYOUT: Layout = Layout::new::>(); #[inline] pub fn id(&self) -> Id { @@ -126,46 +266,119 @@ impl GcHeader { } #[inline] - unsafe fn resolve_type_info(&self) -> &'static GcTypeInfo { - if self.state_bits.forwarded() { - let forward_ptr = self.metadata.forward_ptr; - let forward_header = forward_ptr.as_ref(); - debug_assert!(!forward_header.state_bits.forwarded()); - forward_header.metadata.type_info - } else { - self.metadata.type_info + fn resolve_type_info(&self) -> &'static GcTypeInfo { + unsafe { + if self.state_bits.get().forwarded() { + let forward_ptr = self.metadata.forward_ptr; + let forward_header = forward_ptr.as_ref(); + debug_assert!(!forward_header.state_bits.get().forwarded()); + forward_header.metadata.type_info + } else { + self.metadata.type_info + } } } #[inline] - pub const fn determine_layout(type_layout: Layout) -> GcLayout { - let Ok((overall_layout, value_offset)) = LayoutExt(Self::HEADER_LAYOUT).extend(type_layout) - else { - panic!("Layout error") - }; - let overall_layout = LayoutExt(overall_layout).pad_to_align(); - GcLayout { - value_offset, - overall_layout, - type_layout, + pub fn regular_value_ptr(&self) -> NonNull { + unsafe { + NonNull::new_unchecked( + (self as *const Self as *mut Self as *mut u8).add(Self::REGULAR_VALUE_OFFSET), + ) } } + + #[inline] + pub unsafe fn assume_array_header(&self) -> &'_ GcArrayHeader { + &*(self as *const Self as *const GcArrayHeader) + } } -/// TODO: We should cap the alignment of gc-allocated types -/// -/// This would be a win by avoiding runtime size calculations -/// and making the `value_offset` a compile-time constant -#[derive(Eq, PartialEq, Debug)] -pub(crate) struct GcLayout { - pub type_layout: Layout, - pub overall_layout: Layout, - /// The offset between the end of the header and the beginning of the value - pub value_offset: usize, +#[repr(C, align(8))] +pub struct GcArrayHeader { + main_header: GcHeader, + /// The length of the array in elements + len_elements: usize, +} + +impl GcArrayHeader { + #[inline] + fn resolve_type_info(&self) -> &'static GcArrayTypeInfo { + unsafe { + &*(self.main_header.resolve_type_info() as *const GcTypeInfo + as *const GcArrayTypeInfo) + } + } + + #[inline] + fn element_layout(&self) -> Layout { + self.resolve_type_info() + .element_type_info + .layout + .value_layout + } + + #[cfg_attr(not(debug_assertions), inline)] + fn value_layout(&self) -> Layout { + let element_layout = self.element_layout(); + if cfg!(debug_assertions) { + debug_assert!(element_layout.align() <= GcHeader::::FIXED_ALIGNMENT); + debug_assert_eq!( + element_layout.pad_to_align(), + element_layout, + "padding should already be included" + ); + let Some(repeated_size) = element_layout.size().checked_mul(self.len_elements) else { + panic!( + "Invalid length {} triggers size overflow for {element_layout:?}", + self.len_elements + ) + }; + debug_assert!( + Layout::from_size_align(repeated_size, element_layout.align()).is_ok(), + "align overflow" + ); + } + unsafe { + Layout::from_size_align_unchecked( + element_layout.size().unchecked_mul(self.len_elements), + element_layout.align(), + ) + } + } + + #[cfg_attr(not(debug_assertions), inline)] + fn overall_layout(&self) -> Layout { + let value_layout = self.value_layout(); + if cfg!(debug_assertions) { + let Ok((overall_layout, actual_offset)) = + LayoutExt(GcHeader::::ARRAY_HEADER_LAYOUT).extend(value_layout) + else { + unreachable!("layout overflow") + }; + debug_assert_eq!(actual_offset, GcHeader::::ARRAY_VALUE_OFFSET); + debug_assert_eq!( + Some(overall_layout.size()), + value_layout + .size() + .checked_add(GcHeader::::ARRAY_VALUE_OFFSET) + ); + } + unsafe { + Layout::from_size_align_unchecked( + value_layout + .size() + .unchecked_add(GcHeader::::ARRAY_VALUE_OFFSET), + GcHeader::::FIXED_ALIGNMENT, + ) + .pad_to_align() + } + } } pub struct CollectContext<'newgc, Id: CollectorId> { id: Id, + new_yong_generation: &'newgc YoungGenerationSpace, } impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { #[inline] @@ -173,9 +386,11 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { self.id } #[inline] - unsafe fn trace_gc_ptr(&mut self, target: NonNull>) { + unsafe fn trace_gc_ptr>(&mut self, target: NonNull>) { let target = target.as_ptr(); - target.cast().write(self.collect_gc_ptr(target.read())); + target + .cast::, Id>>() + .write(self.collect_gc_ptr(target.read())); } unsafe fn collect_gc_ptr<'gc, T: Collect>( &mut self, @@ -184,8 +399,9 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { debug_assert_eq!(target.id(), self.id()); let header = target.header(); - if header.state_bits.forwarded() { - header.metadata.forward_ptr + if header.state_bits.get().forwarded() { + header.metadata.forward_ptr; + todo!() } crate::utils::transmute_arbitrary(target) } diff --git a/src/context/young.rs b/src/context/young.rs index 725072e..95778bc 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -1,78 +1,140 @@ -use crate::context::{GcHeader, GcLayout, GcTypeInfo}; -use crate::utils::LayoutExt; -use crate::CollectorId; use std::alloc::Layout; use std::cell::Cell; +use std::marker::PhantomData; use std::ptr::NonNull; -/// A yong-generation object-space +use bumpalo::ChunkRawIter; + +use crate::context::{AllocInfo, GcHeader, GcStateBits, GcTypeInfo, GenerationId, HeaderMetadata}; +use crate::utils::bumpalo_raw::{BumpAllocRaw, BumpAllocRawConfig}; +use crate::utils::{Alignment, LayoutExt}; +use crate::CollectorId; + +/// A young-generation object-space /// /// If copying is in progress, /// there may be two young generations for a single collector. /// /// The design of the allocator is heavily based on [`bumpalo`](https://crates.io/crates/bumpalo) -pub struct YoungGenerationSpace { - current_chunk: Cell>, - current_chunk_next_ptr: Cell>, - current_chunk_end_ptr: Cell>, +pub struct YoungGenerationSpace { + bump: BumpAllocRaw>, collector_id: Id, } impl YoungGenerationSpace { + /// The maximum size to allocate in the young generation. + /// + /// Anything larger than this is immediately sent to the old generation. + pub const SIZE_LIMIT: usize = 1024; + #[inline(always)] - pub fn alloc_uninit( + pub unsafe fn alloc_uninit( &self, - layout_info: GcLayout, type_info: &'static GcTypeInfo, - ) -> (*mut GcHeader, NonNull) { - debug_assert_eq!(layout_info, type_info.layout_info); - } - fn ensure_chunk(&self, layout: Layout) -> Layout { - x + ) -> Result>, YoungAllocError> { + let overall_size = type_info.layout.overall_size; + if overall_size > Self::SIZE_LIMIT { + return Err(YoungAllocError::SizeExceedsLimit); + } + let Ok(raw_ptr) = self + .bump + .try_alloc_layout(Layout::from_size_align_unchecked( + overall_size, + GcHeader::::FIXED_ALIGNMENT, + )) + else { + return Err(YoungAllocError::OutOfMemory); + }; + let header_ptr = raw_ptr.cast::>(); + header_ptr.as_ptr().write(GcHeader { + state_bits: Cell::new( + GcStateBits::builder() + .with_forwarded(false) + .with_generation(GenerationId::Young) + .with_array(false) + .build(), + ), + alloc_info: AllocInfo { + this_object_overall_size: overall_size as u32, + }, + metadata: HeaderMetadata { type_info }, + collector_id: self.collector_id, + }); + Ok(header_ptr) } -} -#[repr(C, align(16))] -pub struct ChunkHeader { - initial_object: Option>, - final_allocation_ptr: Option>, - prev_chunk: Option>, - data_size: usize, -} -impl ChunkHeader { #[inline] - fn data_start_ptr(&self) -> NonNull { - unsafe { - NonNull::new_unchecked( - (self as *const Self as *const u8).add(Self::DATA_OFFSET) as *mut u8 - ) + pub unsafe fn iter_raw_allocations(&self) -> IterRawAllocations<'_, Id> { + IterRawAllocations { + chunk_iter: self.bump.iter_allocated_chunks_raw(), + remaining_chunk_info: None, + marker: PhantomData, } } +} +#[derive(Debug, thiserror::Error)] +enum YoungAllocError { + #[error("Out of memory")] + OutOfMemory, + #[error("Size exceeds young-alloc limit")] + SizeExceedsLimit, +} + +struct IterRawAllocations<'bump, Id: CollectorId> { + chunk_iter: ChunkRawIter<'bump>, + remaining_chunk_info: Option<(NonNull, usize)>, + marker: PhantomData, +} +impl Iterator for IterRawAllocations<'_, Id> { + type Item = NonNull>; + #[inline] - fn data_end_ptr(&self) -> NonNull { - unsafe { NonNull::new_unchecked(self.start_ptr().as_ptr().add(self.data_size)) } - } - #[inline] - fn overall_layout(&self) -> Layout { - assert_eq!(Self::DATA_OFFSET, Self::HEADER_LAYOUT.size()); - unsafe { - Layout::from_size_align_unchecked( - Self::DATA_OFFSET.unchecked_add(self.data_size), - Self::DATA_ALIGNMENT, - ) - .unwrap() + fn next(&mut self) -> Option { + loop { + if let Some((ref mut remaining_chunk_ptr, ref mut remaining_chunk_size)) = + self.remaining_chunk_info + { + if *remaining_chunk_size == 0 { + continue; + } + debug_assert!( + *remaining_chunk_size >= GcHeader::::REGULAR_HEADER_LAYOUT.size() + ); + debug_assert_eq!( + // TODO: Use `is_aligned_to` once stabilized + remaining_chunk_ptr + .as_ptr() + .align_offset(GcHeader::::FIXED_ALIGNMENT), + 0 + ); + unsafe { + let header = &*remaining_chunk_ptr.as_ptr().cast::>(); + debug_assert_eq!(header.state_bits.get().generation(), GenerationId::Young); + let overall_object_size = header.alloc_info.this_object_overall_size as usize; + *remaining_chunk_ptr = NonNull::new_unchecked( + remaining_chunk_ptr.as_ptr().add(overall_object_size), + ); + *remaining_chunk_size = remaining_chunk_size.unchecked_sub(overall_object_size); + return Some(NonNull::from(header)); + } + } else { + match self.chunk_iter.next() { + None => return None, + Some((remaining_chunk_ptr, remaining_chunk_size)) => { + self.remaining_chunk_info = Some(( + unsafe { NonNull::new_unchecked(remaining_chunk_ptr) }, + remaining_chunk_size, + )); + } + } + } } } - #[inline] - fn data_layout(&self) -> Layout { - unsafe { Layout::from_size_align_unchecked(self.data_size, Self::DATA_ALIGNMENT) } - } - const HEADER_LAYOUT: Layout = Layout::new::(); - pub const DATA_OFFSET: usize = { - let this_layout = Self::HEADER_LAYOUT; - assert!(this_layout.align() <= Self::DATA_ALIGNMENT); - let padding = LayoutExt(this_layout).padding_needed_for(Self::DATA_ALIGNMENT); - this_layout.size() + padding +} + +struct BumpConfig(PhantomData<&'static Id>); +impl BumpAllocRawConfig for BumpConfig { + const FIXED_ALIGNMENT: Alignment = match Alignment::new(GcHeader::::FIXED_ALIGNMENT) { + Ok(alignment) => alignment, + Err(_) => unreachable!("GcHeader alignment must be valid"), }; - /// This over-alignment should be good enough for everything... - pub const DATA_ALIGNMENT: usize = std::mem::align_of::(); } diff --git a/src/gcptr.rs b/src/gcptr.rs index 7f46c08..fc7f9df 100644 --- a/src/gcptr.rs +++ b/src/gcptr.rs @@ -6,14 +6,14 @@ use std::ops::Deref; use std::ptr::NonNull; use crate::context::{GcHeader, GcTypeInfo}; -use crate::{Collect, CollectContext, CollectorId}; +use crate::{Collect, CollectContext, CollectorId, GarbageCollector}; pub struct Gc<'gc, T, Id: CollectorId> { ptr: NonNull, marker: PhantomData<*const T>, - collect_marker: PhantomData<&'gc Id::Collector>, + collect_marker: PhantomData<&'gc GarbageCollector>, } -impl<'gc, T, Id: CollectorId> Gc<'gc, T, Id> { +impl<'gc, T: Collect, Id: CollectorId> Gc<'gc, T, Id> { #[inline] pub fn id(&self) -> Id { match unsafe { Id::summon_singleton() } { @@ -22,23 +22,23 @@ impl<'gc, T, Id: CollectorId> Gc<'gc, T, Id> { } } + #[inline] pub(crate) fn header(&self) -> &'_ GcHeader { unsafe { - &*((self.ptr.as_ptr() as *mut u8) - .sub(const { GcHeader::determine_layout(Layout::new()).value_offset }) + &*((self.ptr.as_ptr() as *mut u8).sub(GcHeader::::FIXED_ALIGNMENT) as *mut GcHeader) } } + #[inline] pub(crate) fn type_info() -> &'static GcTypeInfo { - GcTypeInfo::of::() + GcTypeInfo::new::() } #[inline(always)] - pub unsafe fn from_raw_ptr(ptr: NonNull, id: Id) -> Self { + pub unsafe fn from_raw_ptr(ptr: NonNull) -> Self { Gc { ptr, - id, marker: PhantomData, collect_marker: PhantomData, } diff --git a/src/lib.rs b/src/lib.rs index c8efd6a..7339a89 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,8 @@ #![feature( inline_const, // will be stabilized in next release )] +extern crate core; + use std::fmt::Debug; pub mod collect; diff --git a/src/utils.rs b/src/utils.rs index cebc573..b62de40 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,11 +1,10 @@ -pub(crate) mod layout_helpers; - -pub(crate) use self::layout_helpers::LayoutExt; -use crate::Collect; -use crate::Collect; -use std::mem::ManuallyDrop; use std::mem::{self, ManuallyDrop}; +pub(crate) mod bumpalo_raw; +mod layout_helpers; + +pub use self::layout_helpers::{Alignment, LayoutExt}; + /// Transmute one type into another, /// without doing compile-time checks for sizes. /// @@ -29,8 +28,8 @@ use std::mem::{self, ManuallyDrop}; #[inline(always)] #[track_caller] // for the case where sizes don't match pub unsafe fn transmute_arbitrary(val: Src) -> Dst { - const SIZE_MATCHES: bool = std::mem::size_of::() == std::mem::size_of::(); - if SIZE_MATCHES { + let size_matches = const { std::mem::size_of::() == std::mem::size_of::() }; + if size_matches { let src: ManuallyDrop = ManuallyDrop::new(val); std::mem::transmute_copy(&src as &Src) } else { diff --git a/src/utils/bumpalo_raw.rs b/src/utils/bumpalo_raw.rs new file mode 100644 index 0000000..d66ece1 --- /dev/null +++ b/src/utils/bumpalo_raw.rs @@ -0,0 +1,29 @@ +use bumpalo::{AllocErr, Bump}; +use std::alloc::Layout; +use std::cell::Cell; +use std::marker::PhantomData; +use std::mem::MaybeUninit; +use std::ptr::NonNull; + +use crate::utils::Alignment; + +pub struct BumpAllocRaw { + inner: Bump, + marker: PhantomData, +} +impl BumpAllocRaw { + #[inline(always)] + pub fn try_alloc_layout(&self, layout: Layout) -> Result, AllocErr> { + assert_eq!(layout.align(), Config::FIXED_ALIGNMENT); + self.inner.try_alloc_layout(layout) + } + + #[inline] + pub unsafe fn iter_allocated_chunks_raw(&self) -> bumpalo::ChunkRawIter<'_> { + self.inner.iter_allocated_chunks_raw() + } +} + +pub trait BumpAllocRawConfig { + const FIXED_ALIGNMENT: Alignment; +} diff --git a/src/utils/layout_helpers.rs b/src/utils/layout_helpers.rs index 6af2289..199dee5 100644 --- a/src/utils/layout_helpers.rs +++ b/src/utils/layout_helpers.rs @@ -3,6 +3,41 @@ //! Implementation mostly copied from the stdlib. use std::alloc::{Layout, LayoutError}; +use std::fmt::{Debug, Formatter}; +use std::num::{NonZero, NonZeroUsize}; + +/// Represents a valid alignment for a type. +/// +/// This emulates the unstable [`std::ptr::Alignment`] API. +/// +/// ## Safety +/// The alignment must be a nonzero power of two +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct Alignment(NonZeroUsize); + +impl Alignment { + #[inline] + pub const fn new(value: usize) -> Result { + if value.is_power_of_two() { + Ok(Alignment(unsafe { NonZero::new_unchecked(value) })) + } else { + Err(InvalidAlignmentError) + } + } + + #[inline] + pub fn value(&self) -> usize { + self.0.get() + } +} +impl Debug for Alignment { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("Alignment").field(&self.value()).finish() + } +} +#[derive(Debug, thiserror::Error)] +#[error("Invalid alignment")] +struct InvalidAlignmentError; #[derive(Copy, Clone, Debug)] pub struct LayoutExt(pub Layout); @@ -27,7 +62,7 @@ impl LayoutExt { let new_size = self.0.size() + pad; // SAFETY: padded size is guaranteed to not exceed `isize::MAX`. - unsafe { Layout::from_size_align_unchecked(new_size, self.align()) } + unsafe { Layout::from_size_align_unchecked(new_size, self.0.align()) } } /// Copied from stdlib [`Layout::extend`] @@ -38,15 +73,15 @@ impl LayoutExt { /// you must call [`Self::pad_to_align`] to add trailing padding. /// See stdlib docs for details. #[inline] - pub const fn extend(&self, next: Layout) -> Result<(Layout, usize), LayoutError> { + pub const fn extend(&self, next: Layout) -> Result<(Layout, usize), LayoutExtError> { let new_align = Self::const_max(self.0.align(), next.align()); let pad = self.padding_needed_for(next.align()); let Some(offset) = self.size().checked_add(pad) else { - return LayoutError; + return LayoutExtError; }; let Some(new_size) = offset.checked_add(next.size()) else { - return LayoutError; + return LayoutExtError; }; /* @@ -55,7 +90,7 @@ impl LayoutExt { * is we skip the usize::is_power_of_two check. */ if new_size > Self::max_size_for_align(new_align) { - return Err(LayoutError); + return Err(LayoutExtError); } else { Ok(( unsafe { Layout::from_size_align_unchecked(new_size, new_align) }, @@ -65,6 +100,10 @@ impl LayoutExt { } } +#[derive(Debug, thiserror::Error)] +#[error("Layout error")] +struct LayoutExtError; + #[inline] const fn const_max(first: usize, second: usize) -> usize { if first > second { From 6b9397fae3a433e3c167af8b18795495682e9999 Mon Sep 17 00:00:00 2001 From: Techcable Date: Wed, 1 May 2024 03:08:28 -0700 Subject: [PATCH 05/35] Add bindings for mimalloc The oldgen collector is going to use a mimalloc heap for its internal allocations. --- Cargo.lock | 38 +++++++ Cargo.toml | 2 + libs/mimalloc-semisafe/Cargo.toml | 13 +++ libs/mimalloc-semisafe/src/heap.rs | 161 +++++++++++++++++++++++++++++ libs/mimalloc-semisafe/src/lib.rs | 1 + src/collect.rs | 11 +- src/collect/macros.rs | 2 +- src/context.rs | 66 +++++++++--- src/context/old.rs | 14 +++ src/context/young.rs | 22 +++- src/gcptr.rs | 8 +- src/lib.rs | 5 - src/utils.rs | 2 +- src/utils/bumpalo_raw.rs | 4 +- src/utils/layout_helpers.rs | 31 ++++-- 15 files changed, 335 insertions(+), 45 deletions(-) create mode 100644 libs/mimalloc-semisafe/Cargo.toml create mode 100644 libs/mimalloc-semisafe/src/heap.rs create mode 100644 libs/mimalloc-semisafe/src/lib.rs create mode 100644 src/context/old.rs diff --git a/Cargo.lock b/Cargo.lock index 14a31f9..bf33fb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,6 +32,18 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "cc" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" + +[[package]] +name = "cty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" + [[package]] name = "equivalent" version = "1.0.1" @@ -60,6 +72,23 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "libc" +version = "0.2.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" + +[[package]] +name = "libmimalloc-sys" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81eb4061c0582dedea1cbc7aff2240300dd6982e0239d1c99e65c1dbf4a30ba7" +dependencies = [ + "cc", + "cty", + "libc", +] + [[package]] name = "proc-macro-kwargs" version = "0.2.0" @@ -160,4 +189,13 @@ dependencies = [ "bitbybit", "bumpalo", "thiserror", + "zerogc-next-mimalloc-semisafe", +] + +[[package]] +name = "zerogc-next-mimalloc-semisafe" +version = "0.1.0-alpha.1" +dependencies = [ + "allocator-api2", + "libmimalloc-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 89e98c6..f49b114 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,8 @@ bitbybit = "1.3.2" arbitrary-int = "1.2.7" thiserror = "1" +# Internal bindings to mimalloc +zerogc-next-mimalloc-semisafe = { version = "0.1.0-alpha.1", path = "libs/mimalloc-semisafe" } [workspace] resolver = "2" diff --git a/libs/mimalloc-semisafe/Cargo.toml b/libs/mimalloc-semisafe/Cargo.toml new file mode 100644 index 0000000..d0de6eb --- /dev/null +++ b/libs/mimalloc-semisafe/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "zerogc-next-mimalloc-semisafe" +description = "Somewhat safe bindings for mimalloc, used by zerogc-next" +version.workspace = true +edition.workspace = true + +[dependencies] +libmimalloc-sys = { version = "0.1.37", features = ["extended"] } +allocator-api2 = "0.2.18" + +[features] +# Use the "real" nightly allocator api instead of allocator-api2 +nightly-allocapi = ["allocator-api2/nightly"] diff --git a/libs/mimalloc-semisafe/src/heap.rs b/libs/mimalloc-semisafe/src/heap.rs new file mode 100644 index 0000000..eac3a4b --- /dev/null +++ b/libs/mimalloc-semisafe/src/heap.rs @@ -0,0 +1,161 @@ +//! First-class mimalloc heaps +//! +//! These heaps are not thread-safe. + +use std::alloc::Layout; +use std::ffi::c_void; +use std::fmt::{Debug, Formatter}; +use std::marker::PhantomData; +use std::ptr::NonNull; + +use allocator_api2::alloc::{AllocError, Allocator}; + +use libmimalloc_sys as sys; + +pub enum HeapState { + Owned, + Destroyed, +} + +/// A heap used for mimalloc allocations. +/// +/// This is always an explicitly created heap, +/// never the default one. +/// +/// It is implicitly destroyed on drop. +pub struct MimallocHeap { + raw: NonNull, + /// The `mimalloc` can only be safely used in the thread that created it. + /// + /// This marker prevents both `!Send` and `!Sync` although it is + /// technically redundant with the `raw` pointer. + nosend_marker: PhantomData>, +} +impl MimallocHeap { + /// Create a new heap. + #[inline] + pub fn new() -> Self { + MimallocHeap { + raw: unsafe { NonNull::new(sys::mi_heap_new()).unwrap() }, + nosend_marker: PhantomData, + } + } + + /// A raw pointer to the underlying heap + #[inline] + pub unsafe fn as_raw(&self) -> *mut sys::mi_heap_t { + self.raw.as_ptr() + } + + #[inline] + unsafe fn alloc_from_raw_ptr(ptr: *mut u8, size: usize) -> Result, AllocError> { + if ptr.is_null() { + Err(AllocError) + } else { + Ok(NonNull::from(std::slice::from_raw_parts_mut( + ptr as *mut u8, + size, + ))) + } + } + + /// Shared function used for all realloc functions + #[inline] + unsafe fn realloc( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + let _ = old_layout; // mimalloc doesn't use this + Self::alloc_from_raw_ptr( + sys::mi_heap_realloc_aligned( + self.as_raw(), + ptr.as_ptr() as *mut c_void, + new_layout.size(), + new_layout.align(), + ) as *mut u8, + new_layout.size(), + ) + } +} +impl Debug for MimallocHeap { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MimallocHeap").finish_non_exhaustive() + } +} +unsafe impl Allocator for MimallocHeap { + #[inline] + fn allocate(&self, layout: Layout) -> Result, AllocError> { + unsafe { + Self::alloc_from_raw_ptr( + sys::mi_heap_malloc_aligned(self.as_raw(), layout.size(), layout.align()) + as *mut u8, + // do not request actual size to avoid out-of-line call + layout.size(), + ) + } + } + + #[inline] + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + unsafe { + Self::alloc_from_raw_ptr( + sys::mi_heap_zalloc_aligned(self.as_raw(), layout.size(), layout.align()) + as *mut u8, + // don't request actual size to avoid out-of-line call + layout.size(), + ) + } + } + + #[inline] + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + sys::mi_free_size_aligned(ptr.as_ptr() as *mut c_void, layout.size(), layout.align()) + } + + #[inline] + unsafe fn grow( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!(new_layout.size() >= old_layout.size()); + self.realloc(ptr, old_layout, new_layout) + } + + #[inline] + unsafe fn grow_zeroed( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + /* + * Using rezalloc is only valid if existing memory is zeroed, + * which we cannot guarentee her + */ + self.grow(ptr, old_layout, new_layout) + } + + #[inline] + unsafe fn shrink( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!(new_layout.size() <= old_layout.size()); + self.realloc(ptr, old_layout, new_layout) + } +} + +impl Drop for MimallocHeap { + #[inline] + fn drop(&mut self) { + unsafe { + sys::mi_heap_destroy(self.as_raw()); + } + } +} diff --git a/libs/mimalloc-semisafe/src/lib.rs b/libs/mimalloc-semisafe/src/lib.rs new file mode 100644 index 0000000..0e7aad9 --- /dev/null +++ b/libs/mimalloc-semisafe/src/lib.rs @@ -0,0 +1 @@ +pub mod heap; diff --git a/src/collect.rs b/src/collect.rs index aafcd44..5e94d68 100644 --- a/src/collect.rs +++ b/src/collect.rs @@ -1,13 +1,14 @@ //! Defines the [`Collect`] trait and implements it for several types -mod collections; -pub mod macros; -mod primitives; +use std::ptr::NonNull; use crate::context::CollectContext; use crate::CollectorId; -use std::mem::ManuallyDrop; -use std::ptr::NonNull; + +mod collections; +#[doc(hidden)] // has an internal helper module +pub mod macros; +mod primitives; pub unsafe trait Collect { type Collected<'newgc>: Collect; diff --git a/src/collect/macros.rs b/src/collect/macros.rs index 926d50a..0d751dc 100644 --- a/src/collect/macros.rs +++ b/src/collect/macros.rs @@ -12,7 +12,7 @@ macro_rules! static_null_trace { }; #[inline(always)] // does nothing - unsafe fn collect_inplace(_target: std::ptr::NonNull, _context: &mut crate::context::CollectContext<'_, Id>) {} + unsafe fn collect_inplace(_target: std::ptr::NonNull, _context: &mut $crate::context::CollectContext<'_, Id>) {} } unsafe impl $crate::NullCollect for $target {} }; diff --git a/src/context.rs b/src/context.rs index 0118dbe..a30d4ca 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,21 +1,18 @@ -use std::alloc::{Layout, LayoutError}; -use std::any::TypeId; +use std::alloc::Layout; use std::cell::Cell; -use std::cmp; use std::fmt::Debug; -use std::io::repeat; use std::marker::PhantomData; -use std::mem::ManuallyDrop; use std::ptr::NonNull; -use allocator_api2::alloc::Allocator; use bitbybit::{bitenum, bitfield}; +use crate::context::old::OldGenerationSpace; use crate::context::young::YoungGenerationSpace; use crate::gcptr::Gc; use crate::utils::LayoutExt; -use crate::{Collect, NullCollect}; +use crate::Collect; +mod old; mod young; pub enum SingletonStatus { @@ -109,7 +106,7 @@ impl GcTypeLayout { let Ok((expected_overall_layout, value_offset)) = LayoutExt(header_layout).extend(value_layout) else { - unreachable!("layout overflow") + panic!("layout overflow") }; assert!( value_offset == GcHeader::::REGULAR_VALUE_OFFSET, @@ -191,6 +188,12 @@ trait TypeIdInit> { struct GcTypeInitImpl; impl> TypeIdInit for GcTypeInitImpl {} +trait Generation { + const ID: GenerationId; + fn cast_young(&self) -> Option<&'_ YoungGenerationSpace>; + fn cast_old(&self) -> Option<&'_ OldGenerationSpace>; +} + #[derive(Debug, Eq, PartialEq)] #[bitenum(u1, exhaustive = true)] enum GenerationId { @@ -198,6 +201,35 @@ enum GenerationId { Old = 1, } +#[bitenum(u1, exhaustive = true)] +enum GcMarkBits { + White = 0, + Black = 1, +} + +#[bitenum(u1, exhaustive = true)] +enum GcRawMarkBits { + Red = 0, + Green = 1, +} +impl GcRawMarkBits { + #[inline] + pub fn resolve>(&self, gen: &T) -> GcMarkBits { + let inverted = match T::Id { + GenerationId::Young => false, + GenerationId::Old => { + self.to_regular_markbits(gen.cast_old().unwrap().mark_bits_inverted()) + } + }; + let bits = self.raw_value(); + GcMarkBits::new_with_raw_value(if inverted { + as arbitrary_int::Number>::MAX - bits + } else { + bits + }) + } +} + /// A bitfifeld /// /// ## Default @@ -212,6 +244,8 @@ struct GcStateBits { generation: GenerationId, #[bit(2, rw)] array: bool, + #[bit(3, rw)] + raw_mark_bits: GcRawMarkBits, } union HeaderMetadata { type_info: &'static GcTypeInfo, @@ -235,6 +269,7 @@ union AllocInfo { /// This is used in the old generation. pub live_object_index: u32, } + #[repr(C, align(8))] pub(crate) struct GcHeader { pub state_bits: Cell, @@ -386,7 +421,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { self.id } #[inline] - unsafe fn trace_gc_ptr>(&mut self, target: NonNull>) { + pub unsafe fn trace_gc_ptr_mut>(&mut self, target: NonNull>) { let target = target.as_ptr(); target .cast::, Id>>() @@ -398,11 +433,16 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { ) -> Gc<'newgc, T::Collected<'newgc>, Id> { debug_assert_eq!(target.id(), self.id()); let header = target.header(); - if header.state_bits.get().forwarded() { - header.metadata.forward_ptr; - todo!() + return Gc::from_raw_ptr( + header + .metadata + .forward_ptr + .as_ref() + .regular_value_ptr() + .cast(), + ); } - crate::utils::transmute_arbitrary(target) + todo!() } } diff --git a/src/context/old.rs b/src/context/old.rs new file mode 100644 index 0000000..ea7bc4f --- /dev/null +++ b/src/context/old.rs @@ -0,0 +1,14 @@ +use crate::CollectorId; +use zerogc_next_mimalloc_semisafe::heap::MimallocHeap; + +pub struct OldGenerationSpace { + heap: MimallocHeap, + collector_id: Id, + mark_bits_inverted: bool, +} +impl OldGenerationSpace { + #[inline] + pub fn mark_bits_inverted(&self) -> bool { + self.mark_bits_inverted + } +} diff --git a/src/context/young.rs b/src/context/young.rs index 95778bc..b3a2855 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -5,9 +5,12 @@ use std::ptr::NonNull; use bumpalo::ChunkRawIter; -use crate::context::{AllocInfo, GcHeader, GcStateBits, GcTypeInfo, GenerationId, HeaderMetadata}; +use crate::context::old::OldGenerationSpace; +use crate::context::{ + AllocInfo, GcHeader, GcStateBits, GcTypeInfo, GenerationId, GenerationKind, HeaderMetadata, +}; use crate::utils::bumpalo_raw::{BumpAllocRaw, BumpAllocRawConfig}; -use crate::utils::{Alignment, LayoutExt}; +use crate::utils::Alignment; use crate::CollectorId; /// A young-generation object-space @@ -71,6 +74,19 @@ impl YoungGenerationSpace { } } } +impl super::Generation for YoungGenerationSpace { + const ID: GenerationId = GenerationId::Young; + + #[inline] + fn cast_young(&self) -> Option<&'_ YoungGenerationSpace> { + Some(self) + } + + #[inline] + fn cast_old(&self) -> Option<&'_ OldGenerationSpace> { + None + } +} #[derive(Debug, thiserror::Error)] enum YoungAllocError { #[error("Out of memory")] @@ -135,6 +151,6 @@ struct BumpConfig(PhantomData<&'static Id>); impl BumpAllocRawConfig for BumpConfig { const FIXED_ALIGNMENT: Alignment = match Alignment::new(GcHeader::::FIXED_ALIGNMENT) { Ok(alignment) => alignment, - Err(_) => unreachable!("GcHeader alignment must be valid"), + Err(_) => panic!("GcHeader alignment must be valid"), }; } diff --git a/src/gcptr.rs b/src/gcptr.rs index fc7f9df..c8a6f4a 100644 --- a/src/gcptr.rs +++ b/src/gcptr.rs @@ -1,7 +1,4 @@ -use std::alloc::Layout; -use std::any::Any; use std::marker::PhantomData; -use std::mem::ManuallyDrop; use std::ops::Deref; use std::ptr::NonNull; @@ -50,11 +47,10 @@ unsafe impl<'gc, Id: CollectorId, T: Collect> Collect for Gc<'gc, T, Id> #[inline] unsafe fn collect_inplace(target: NonNull, context: &mut CollectContext<'_, Id>) { - let resolved = target.as_ref(); - if matches!(Id::SINGLETON, None) && resolved.id() != context.id() { + if matches!(Id::SINGLETON, None) && target.as_ref().id() != context.id() { return; } - context.trace_gcptr(target) + context.trace_gc_ptr_mut(target) } } impl<'gc, T, Id: CollectorId> Deref for Gc<'gc, T, Id> { diff --git a/src/lib.rs b/src/lib.rs index 7339a89..609eec2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,5 @@ -#![feature( - inline_const, // will be stabilized in next release -)] extern crate core; -use std::fmt::Debug; - pub mod collect; pub mod context; mod gcptr; diff --git a/src/utils.rs b/src/utils.rs index b62de40..7de3e7f 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,4 @@ -use std::mem::{self, ManuallyDrop}; +use std::mem::ManuallyDrop; pub(crate) mod bumpalo_raw; mod layout_helpers; diff --git a/src/utils/bumpalo_raw.rs b/src/utils/bumpalo_raw.rs index d66ece1..b3514fb 100644 --- a/src/utils/bumpalo_raw.rs +++ b/src/utils/bumpalo_raw.rs @@ -1,8 +1,6 @@ use bumpalo::{AllocErr, Bump}; use std::alloc::Layout; -use std::cell::Cell; use std::marker::PhantomData; -use std::mem::MaybeUninit; use std::ptr::NonNull; use crate::utils::Alignment; @@ -14,7 +12,7 @@ pub struct BumpAllocRaw { impl BumpAllocRaw { #[inline(always)] pub fn try_alloc_layout(&self, layout: Layout) -> Result, AllocErr> { - assert_eq!(layout.align(), Config::FIXED_ALIGNMENT); + assert_eq!(layout.align(), Config::FIXED_ALIGNMENT.value()); self.inner.try_alloc_layout(layout) } diff --git a/src/utils/layout_helpers.rs b/src/utils/layout_helpers.rs index 199dee5..f1d5fc0 100644 --- a/src/utils/layout_helpers.rs +++ b/src/utils/layout_helpers.rs @@ -2,7 +2,7 @@ //! //! Implementation mostly copied from the stdlib. -use std::alloc::{Layout, LayoutError}; +use std::alloc::Layout; use std::fmt::{Debug, Formatter}; use std::num::{NonZero, NonZeroUsize}; @@ -16,10 +16,16 @@ use std::num::{NonZero, NonZeroUsize}; pub struct Alignment(NonZeroUsize); impl Alignment { + #[inline] + pub const unsafe fn new_unchecked(value: usize) -> Self { + debug_assert!(value.is_power_of_two()); + Alignment(unsafe { NonZero::new_unchecked(value) }) + } + #[inline] pub const fn new(value: usize) -> Result { if value.is_power_of_two() { - Ok(Alignment(unsafe { NonZero::new_unchecked(value) })) + Ok(unsafe { Self::new_unchecked(value) }) } else { Err(InvalidAlignmentError) } @@ -35,9 +41,11 @@ impl Debug for Alignment { f.debug_tuple("Alignment").field(&self.value()).finish() } } + #[derive(Debug, thiserror::Error)] #[error("Invalid alignment")] -struct InvalidAlignmentError; +#[non_exhaustive] +pub struct InvalidAlignmentError; #[derive(Copy, Clone, Debug)] pub struct LayoutExt(pub Layout); @@ -74,14 +82,14 @@ impl LayoutExt { /// See stdlib docs for details. #[inline] pub const fn extend(&self, next: Layout) -> Result<(Layout, usize), LayoutExtError> { - let new_align = Self::const_max(self.0.align(), next.align()); + let new_align = const_max(self.0.align(), next.align()); let pad = self.padding_needed_for(next.align()); - let Some(offset) = self.size().checked_add(pad) else { - return LayoutExtError; + let Some(offset) = self.0.size().checked_add(pad) else { + return Err(LayoutExtError); }; let Some(new_size) = offset.checked_add(next.size()) else { - return LayoutExtError; + return Err(LayoutExtError); }; /* @@ -89,7 +97,7 @@ impl LayoutExt { * The advantage of a manual check over Layout::from_size_align * is we skip the usize::is_power_of_two check. */ - if new_size > Self::max_size_for_align(new_align) { + if new_size > Self::max_size_for_align(unsafe { Alignment::new_unchecked(new_align) }) { return Err(LayoutExtError); } else { Ok(( @@ -98,10 +106,17 @@ impl LayoutExt { )) } } + + /// Copied from stdlib [`Layout::max_size_for_align`] + #[inline] + const fn max_size_for_align(align: Alignment) -> usize { + isize::MAX as usize - (align.value() - 1) + } } #[derive(Debug, thiserror::Error)] #[error("Layout error")] +#[non_exhaustive] struct LayoutExtError; #[inline] From ea40db0ada2c7b6093a9c23b5eb5b8d40c90952e Mon Sep 17 00:00:00 2001 From: Techcable Date: Wed, 1 May 2024 20:38:48 -0700 Subject: [PATCH 06/35] Work on mark bits state (can be inverted in oldgen) --- Cargo.lock | 7 +++++ Cargo.toml | 1 + src/context.rs | 47 +++++++++++++++++++++++--------- src/context/old.rs | 64 +++++++++++++++++++++++++++++++++++++++++++- src/context/young.rs | 2 +- src/gcptr.rs | 1 + 6 files changed, 108 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf33fb6..acda4c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,6 +89,12 @@ dependencies = [ "libc", ] +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + [[package]] name = "proc-macro-kwargs" version = "0.2.0" @@ -188,6 +194,7 @@ dependencies = [ "arbitrary-int", "bitbybit", "bumpalo", + "log", "thiserror", "zerogc-next-mimalloc-semisafe", ] diff --git a/Cargo.toml b/Cargo.toml index f49b114..fc14cf4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ thiserror = "1" # Internal bindings to mimalloc zerogc-next-mimalloc-semisafe = { version = "0.1.0-alpha.1", path = "libs/mimalloc-semisafe" } +log = "0.4.21" [workspace] resolver = "2" diff --git a/src/context.rs b/src/context.rs index a30d4ca..59f66aa 100644 --- a/src/context.rs +++ b/src/context.rs @@ -188,7 +188,7 @@ trait TypeIdInit> { struct GcTypeInitImpl; impl> TypeIdInit for GcTypeInitImpl {} -trait Generation { +unsafe trait Generation { const ID: GenerationId; fn cast_young(&self) -> Option<&'_ YoungGenerationSpace>; fn cast_old(&self) -> Option<&'_ OldGenerationSpace>; @@ -201,12 +201,27 @@ enum GenerationId { Old = 1, } +/// The raw bit representation of [GcMarkBits] +type GcMarkBitsRepr = arbitrary_int::UInt; + #[bitenum(u1, exhaustive = true)] enum GcMarkBits { White = 0, Black = 1, } +impl GcMarkBits { + #[inline] + pub fn to_raw>(&self, gen: &T) -> GcRawMarkBits { + let bits: GcMarkBitsRepr = self.raw_value(); + GcRawMarkBits::new_with_raw_value(if GcRawMarkBits::is_inverted(gen) { + GcRawMarkBits::invert_bits(bits) + } else { + bits + }) + } +} + #[bitenum(u1, exhaustive = true)] enum GcRawMarkBits { Red = 0, @@ -215,22 +230,29 @@ enum GcRawMarkBits { impl GcRawMarkBits { #[inline] pub fn resolve>(&self, gen: &T) -> GcMarkBits { - let inverted = match T::Id { - GenerationId::Young => false, - GenerationId::Old => { - self.to_regular_markbits(gen.cast_old().unwrap().mark_bits_inverted()) - } - }; - let bits = self.raw_value(); - GcMarkBits::new_with_raw_value(if inverted { - as arbitrary_int::Number>::MAX - bits + let bits: GcMarkBitsRepr = self.raw_value(); + GcMarkBits::new_with_raw_value(if Self::is_inverted(gen) { + Self::invert_bits(bits) } else { bits }) } + + #[inline] + fn invert_bits(bits: GcMarkBitsRepr) -> GcMarkBitsRepr { + ::MAX - bits + } + + #[inline] + fn is_inverted>(gen: &T) -> bool { + match T::Id { + GenerationId::Young => false, + GenerationId::Old => gen.cast_old().unwrap().mark_bits_inverted(), + } + } } -/// A bitfifeld +/// A bitfield for the garbage collector's state. /// /// ## Default /// The `DEFAULT` value isn't valid here. @@ -413,13 +435,13 @@ impl GcArrayHeader { pub struct CollectContext<'newgc, Id: CollectorId> { id: Id, - new_yong_generation: &'newgc YoungGenerationSpace, } impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { #[inline] pub fn id(&self) -> Id { self.id } + #[inline] pub unsafe fn trace_gc_ptr_mut>(&mut self, target: NonNull>) { let target = target.as_ptr(); @@ -427,6 +449,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { .cast::, Id>>() .write(self.collect_gc_ptr(target.read())); } + unsafe fn collect_gc_ptr<'gc, T: Collect>( &mut self, target: Gc<'gc, T, Id>, diff --git a/src/context/old.rs b/src/context/old.rs index ea7bc4f..1ddc956 100644 --- a/src/context/old.rs +++ b/src/context/old.rs @@ -1,8 +1,18 @@ -use crate::CollectorId; +use allocator_api2::alloc::Allocator; +use std::alloc::Layout; +use std::cell::Cell; +use std::ptr::NonNull; use zerogc_next_mimalloc_semisafe::heap::MimallocHeap; +use crate::context::young::YoungGenerationSpace; +use crate::context::{ + AllocInfo, GcHeader, GcMarkBits, GcStateBits, GcTypeInfo, GenerationId, HeaderMetadata, +}; +use crate::CollectorId; + pub struct OldGenerationSpace { heap: MimallocHeap, + live_objects: Vec>>, collector_id: Id, mark_bits_inverted: bool, } @@ -11,4 +21,56 @@ impl OldGenerationSpace { pub fn mark_bits_inverted(&self) -> bool { self.mark_bits_inverted } + + #[inline(always)] + pub unsafe fn alloc_uninit( + &self, + type_info: &'static GcTypeInfo, + ) -> Result>, OldAllocError> { + let overall_size = type_info.layout.overall_size; + let raw_ptr = match self.heap.allocate(Layout::from_size_align_unchecked( + overall_size, + GcHeader::::FIXED_ALIGNMENT, + )) { + Ok(raw_ptr) => raw_ptr, + Err(allocator_api2::alloc::AllocError) => return Err(OldAllocError::OutOfMemory), + }; + let header_ptr = raw_ptr.cast::>(); + header_ptr.as_ptr().write(GcHeader { + state_bits: Cell::new( + GcStateBits::builder() + .with_forwarded(false) + .with_generation(GenerationId::Old) + .with_array(false) + .with_raw_mark_bits(GcMarkBits::White.to_raw(self)) + .build(), + ), + alloc_info: AllocInfo { + this_object_overall_size: overall_size as u32, + }, + metadata: HeaderMetadata { type_info }, + collector_id: self.collector_id, + }); + Ok(header_ptr) + } +} + +#[derive(Debug, thiserror::Error)] +enum OldAllocError { + #[error("Out of memory (oldgen)")] + OutOfMemory, +} + +unsafe impl super::Generation for OldGenerationSpace { + const ID: GenerationId = GenerationId::Old; + + #[inline] + fn cast_young(&self) -> Option<&'_ YoungGenerationSpace> { + None + } + + #[inline] + fn cast_old(&self) -> Option<&'_ OldGenerationSpace> { + Some(self) + } } diff --git a/src/context/young.rs b/src/context/young.rs index b3a2855..0d13a21 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -74,7 +74,7 @@ impl YoungGenerationSpace { } } } -impl super::Generation for YoungGenerationSpace { +unsafe impl super::Generation for YoungGenerationSpace { const ID: GenerationId = GenerationId::Young; #[inline] diff --git a/src/gcptr.rs b/src/gcptr.rs index c8a6f4a..f914581 100644 --- a/src/gcptr.rs +++ b/src/gcptr.rs @@ -1,3 +1,4 @@ +use log::debug; use std::marker::PhantomData; use std::ops::Deref; use std::ptr::NonNull; From 4e445d3397d3b8b3cb60a7d2135ea9d73608a618 Mon Sep 17 00:00:00 2001 From: Techcable Date: Thu, 2 May 2024 06:55:58 -0700 Subject: [PATCH 07/35] Write pointer trace & copy function --- Cargo.lock | 51 ++++++++ Cargo.toml | 3 +- src/context.rs | 262 +++++++++++++++++++++++++++++++++++---- src/context/old.rs | 55 ++++++-- src/context/young.rs | 11 +- src/gcptr.rs | 5 + src/utils/bumpalo_raw.rs | 5 + 7 files changed, 356 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acda4c6..e5a475d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,12 @@ version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "cty" version = "0.2.2" @@ -129,6 +135,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +dependencies = [ + "cc", +] + [[package]] name = "quote" version = "1.0.36" @@ -138,6 +153,19 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "stacker" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "winapi", +] + [[package]] name = "syn" version = "2.0.61" @@ -175,6 +203,28 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "zerog-next-macros" version = "0.1.0-alpha.1" @@ -195,6 +245,7 @@ dependencies = [ "bitbybit", "bumpalo", "log", + "stacker", "thiserror", "zerogc-next-mimalloc-semisafe", ] diff --git a/Cargo.toml b/Cargo.toml index fc14cf4..509daa6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,8 @@ allocator-api2 = "0.2.18" bitbybit = "1.3.2" arbitrary-int = "1.2.7" thiserror = "1" - +# Easier to debug recusion than an explicit queue +stacker = "0.1" # Internal bindings to mimalloc zerogc-next-mimalloc-semisafe = { version = "0.1.0-alpha.1", path = "libs/mimalloc-semisafe" } log = "0.4.21" diff --git a/src/context.rs b/src/context.rs index 59f66aa..10b3845 100644 --- a/src/context.rs +++ b/src/context.rs @@ -2,6 +2,7 @@ use std::alloc::Layout; use std::cell::Cell; use std::fmt::Debug; use std::marker::PhantomData; +use std::os::macos::raw::stat; use std::ptr::NonNull; use bitbybit::{bitenum, bitfield}; @@ -46,8 +47,44 @@ pub unsafe trait CollectorId: Copy + Debug + Eq + 'static { unsafe fn summon_singleton() -> Option; } +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +enum CollectStageTracker { + NotCollecting, + Stage { current: CollectStage }, + FinishedStage { last_stage: CollectStage }, +} + +impl CollectStageTracker { + #[inline] + fn begin_stage(&mut self, expected_stage: Option, new_stage: CollectStage) { + assert_eq!( + match expected_stage { + Some(last_stage) => CollectStageTracker::FinishedStage { last_stage }, + None => CollectStageTracker::NotCollecting, + }, + self + ); + *self = CollectStageTracker::Stage { current: new_stage }; + } + + #[inline] + fn finish_stage(&mut self, stage: CollectStage) { + assert_eq!(CollectStageTracker::Stage { current: stage }, self); + *self = CollectStageTracker::FinishedStage { last_stage: stage }; + } +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +enum CollectStage { + Mark, + Sweep, +} + pub struct GarbageCollector { id: Id, + young_generation: YoungGenerationSpace, + old_generation: OldGenerationSpace, + mark_bits_inverted: bool, } impl GarbageCollector { #[inline] @@ -159,9 +196,11 @@ impl GcArrayTypeInfo { } } +type TraceFuncPtr = unsafe fn(NonNull<()>, &mut CollectContext); pub(crate) struct GcTypeInfo { layout: GcTypeLayout, drop_func: Option, + trace_func: Option>, } impl GcTypeInfo { #[inline] @@ -181,19 +220,29 @@ trait TypeIdInit> { } else { None }; - GcTypeInfo { layout, drop_func } + let trace_func = if T::NEEDS_COLLECT { + unsafe { + Some(std::mem::transmute::< + _, + unsafe fn(NonNull<()>, &mut CollectContext), + >( + T::collect_inplace as unsafe fn(NonNull, &mut CollectContext), + )) + } + } else { + None + }; + GcTypeInfo { + layout, + drop_func, + trace_func, + } }; const TYPE_INFO_REF: &'static GcTypeInfo = &Self::TYPE_INFO_INIT_VAL; } struct GcTypeInitImpl; impl> TypeIdInit for GcTypeInitImpl {} -unsafe trait Generation { - const ID: GenerationId; - fn cast_young(&self) -> Option<&'_ YoungGenerationSpace>; - fn cast_old(&self) -> Option<&'_ OldGenerationSpace>; -} - #[derive(Debug, Eq, PartialEq)] #[bitenum(u1, exhaustive = true)] enum GenerationId { @@ -206,15 +255,21 @@ type GcMarkBitsRepr = arbitrary_int::UInt; #[bitenum(u1, exhaustive = true)] enum GcMarkBits { + /// Indicates that tracing has not yet marked the object. + /// + /// Once tracing completes, this means the object is dead. White = 0, + /// Indicates that tracing has marked the object. + /// + /// This means the object is live. Black = 1, } impl GcMarkBits { #[inline] - pub fn to_raw>(&self, gen: &T) -> GcRawMarkBits { + pub fn to_raw(&self, collector: &GarbageCollector) -> GcRawMarkBits { let bits: GcMarkBitsRepr = self.raw_value(); - GcRawMarkBits::new_with_raw_value(if GcRawMarkBits::is_inverted(gen) { + GcRawMarkBits::new_with_raw_value(if collector.mark_bits_inverted { GcRawMarkBits::invert_bits(bits) } else { bits @@ -229,9 +284,9 @@ enum GcRawMarkBits { } impl GcRawMarkBits { #[inline] - pub fn resolve>(&self, gen: &T) -> GcMarkBits { + pub fn resolve(&self, collector: &GarbageCollector) -> GcMarkBits { let bits: GcMarkBitsRepr = self.raw_value(); - GcMarkBits::new_with_raw_value(if Self::is_inverted(gen) { + GcMarkBits::new_with_raw_value(if collector.mark_bits_inverted { Self::invert_bits(bits) } else { bits @@ -242,14 +297,6 @@ impl GcRawMarkBits { fn invert_bits(bits: GcMarkBitsRepr) -> GcMarkBitsRepr { ::MAX - bits } - - #[inline] - fn is_inverted>(gen: &T) -> bool { - match T::Id { - GenerationId::Young => false, - GenerationId::Old => gen.cast_old().unwrap().mark_bits_inverted(), - } - } } /// A bitfield for the garbage collector's state. @@ -294,9 +341,9 @@ union AllocInfo { #[repr(C, align(8))] pub(crate) struct GcHeader { - pub state_bits: Cell, - pub alloc_info: AllocInfo, - pub metadata: HeaderMetadata, + pub(crate) state_bits: Cell, + pub(crate) alloc_info: AllocInfo, + pub(crate) metadata: HeaderMetadata, /// The id for the collector where this object is allocated. /// /// If the collector is a singleton (either global or thread-local), @@ -307,6 +354,13 @@ pub(crate) struct GcHeader { pub collector_id: Id, } impl GcHeader { + #[inline] + pub(crate) unsafe fn update_state_bits(&self, func: impl FnOnce(&mut GcStateBits)) { + let mut bits = self.state_bits.get(); + func(&mut bits); + self.state_bits.set(bits); + } + /// The fixed alignment for all GC types /// /// Allocating a type with an alignment greater than this is an error. @@ -367,6 +421,16 @@ impl GcArrayHeader { } } + #[inline] + pub fn array_value_ptr(&self) -> NonNull { + unsafe { + NonNull::new_unchecked( + (self as *const Self as *mut Self as *mut u8) + .add(GcHeader::::ARRAY_VALUE_OFFSET), + ) + } + } + #[inline] fn element_layout(&self) -> Layout { self.resolve_type_info() @@ -435,6 +499,7 @@ impl GcArrayHeader { pub struct CollectContext<'newgc, Id: CollectorId> { id: Id, + garbage_collector: &'newgc mut GarbageCollector, } impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { #[inline] @@ -450,13 +515,27 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { .write(self.collect_gc_ptr(target.read())); } + #[cfg_attr(not(debug_assertions), inline)] unsafe fn collect_gc_ptr<'gc, T: Collect>( &mut self, target: Gc<'gc, T, Id>, ) -> Gc<'newgc, T::Collected<'newgc>, Id> { - debug_assert_eq!(target.id(), self.id()); let header = target.header(); + assert_eq!(header.collector_id, self.id, "Mismatched collector ids"); + debug_assert!( + !header.state_bits.get().array(), + "Incorrectly marked as an array" + ); if header.state_bits.get().forwarded() { + debug_assert_eq!(header.state_bits.get().generation(), GenerationId::Young); + debug_assert_eq!( + header + .state_bits + .get() + .raw_mark_bits() + .resolve(&self.garbage_collector.young_generation), + GcMarkBits::Black + ); return Gc::from_raw_ptr( header .metadata @@ -466,6 +545,141 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { .cast(), ); } - todo!() + match header + .state_bits + .get() + .raw_mark_bits() + .resolve(self.garbage_collector) + { + GcMarkBits::White => { + let new_header = self.fallback_collect_gcptr(header); + Gc::from_raw_ptr(new_header.as_ref().regular_value_ptr().cast()) + } + GcMarkBits::Black => { + // already traced, can skip it + Gc::from_raw_ptr(target.as_raw_ptr()) + } + } + } + + #[cold] + unsafe fn fallback_collect_gcheaer<'gc>( + &mut self, + header_ptr: NonNull>, + ) -> NonNull> { + let type_info: &'static GcTypeInfo; + let prev_generation: GenerationId; + { + let header = header_ptr.as_ref(); + debug_assert_eq!( + header + .state_bits + .get() + .raw_mark_bits() + .resolve(self.garbage_collector), + GcMarkBits::White + ); + // mark as black + header.update_state_bits(|state_bits| { + state_bits.with_raw_mark_bits(GcMarkBits::Black.to_raw(self.garbage_collector)); + }); + prev_generation = header.state_bits.get().generation(); + type_info = header.metadata.type_info; + } + let forwarded_ptr = match prev_generation { + GenerationId::Young => { + assert!( + !header_ptr.as_ref().state_bits.get().array(), + "TODO: Support arrays in youngen copy" + ); + // reallocate in oldgen + // TODO: This panic is fatal, will cause an abort + let forwarded_ptr = self + .garbage_collector + .old_generation + .alloc_uninit(type_info) + .expect("Oldgen alloc failure"); + forwarded_ptr + .as_ref() + .state_bits + .set(header_ptr.as_ref().state_bits.get()); + forwarded_ptr.as_ref().update_state_bits(|bits| { + debug_assert!(!bits.forwarded()); + bits.with_generation(GenerationId::Old); + }); + header_ptr.as_ref().update_state_bits(|bits| { + bits.with_forwarded(true); + }); + (&mut *header_ptr.as_ptr()).metadata.forward_ptr = forwarded_ptr.cast(); + // NOTE: Copy uninitialized bytes is safe here, as long as they are not read in dest + forwarded_ptr + .as_ref() + .regular_value_ptr() + .cast::() + .as_ptr() + .copy_from_nonoverlapping( + header_ptr + .as_ref() + .regular_value_ptr() + .cast::() + .as_ptr(), + type_info.layout.value_layout().size(), + ); + forwarded_ptr + } + GenerationId::Old => header_ptr, // no copying needed for oldgen + }; + /* + * finally, trace the value + * this needs to come after forwarding and switching the mark bit + * so we can properly update self-referential pointers + */ + if let Some(trace_func) = type_info.trace_func { + /* + * NOTE: Cannot have aliasing &mut header references during this recursion + * The parameters to maybe_grow are completely arbitrary right now. + */ + stacker::maybe_grow( + 4096, // 4KB + 128 * 1024, // 128KB + || self.trace_children(forwarded_ptr, trace_func), + ); + } + forwarded_ptr + } + + #[inline] + unsafe fn trace_children( + &mut self, + header: NonNull>, + trace_func: TraceFuncPtr, + ) { + debug_assert!( + !header.as_ref().state_bits.get().forwarded(), + "Cannot be forwarded" + ); + if header.as_ref().state_bits.get().array() { + self.trace_children_array(header.cast(), trace_func); + } else { + trace_func(header.as_ref().regular_value_ptr().cast(), self); + } + } + unsafe fn trace_children_array( + &mut self, + header: NonNull>, + trace_func: TraceFuncPtr, + ) { + let type_info = header.as_ref().main_header.metadata.type_info; + debug_assert_eq!(type_info.trace_func, Some(trace_func)); + let array_header = header.cast::>(); + let element_layout = type_info.layout.value_layout; + let len = array_header.as_ref().len_elements; + let element_start_ptr = array_header.as_ref().array_value_ptr(); + for i in 0..len { + let element = element_start_ptr + .as_ptr() + .add(i.unchecked_mul(element_layout.size())); + trace_func(NonNull::new_unchecked(element as *mut ()), self); + } } } diff --git a/src/context/old.rs b/src/context/old.rs index 1ddc956..ebf706e 100644 --- a/src/context/old.rs +++ b/src/context/old.rs @@ -1,25 +1,56 @@ use allocator_api2::alloc::Allocator; use std::alloc::Layout; -use std::cell::Cell; +use std::cell::{Cell, UnsafeCell}; use std::ptr::NonNull; use zerogc_next_mimalloc_semisafe::heap::MimallocHeap; use crate::context::young::YoungGenerationSpace; use crate::context::{ - AllocInfo, GcHeader, GcMarkBits, GcStateBits, GcTypeInfo, GenerationId, HeaderMetadata, + AllocInfo, CollectStage, CollectStageTracker, GcHeader, GcMarkBits, GcStateBits, GcTypeInfo, + GenerationId, HeaderMetadata, }; use crate::CollectorId; pub struct OldGenerationSpace { heap: MimallocHeap, - live_objects: Vec>>, + live_objects: UnsafeCell>>>, collector_id: Id, - mark_bits_inverted: bool, + stage: CollectStageTracker, } impl OldGenerationSpace { - #[inline] - pub fn mark_bits_inverted(&self) -> bool { - self.mark_bits_inverted + pub unsafe fn sweep(&mut self) { + self.stage + .begin_stage(Some(CollectStage::Mark), CollectStage::Sweep); + let mut next_index: u32 = 0; + self.live_objects.get_mut().retain(|func| { + let header = &mut *func.as_ptr(); + debug_assert_eq!(header.collector_id, self.collector_id); + debug_assert_eq!(header.state_bits.get().generation(), GenerationId::Old); + let mark_bits = header.state_bits.get().raw_mark_bits().resolve(self); + match mark_bits { + GcMarkBits::White => { + // unmarked + if cfg!(debug_assertions) { + header.alloc_info.live_object_index = u32::MAX; + } + false + } + GcMarkBits::Black => { + // marked + header.alloc_info.live_object_index = next_index; + next_index += 1; + true + } + } + }); + assert_eq!(next_index as usize, self.live_objects.get_mut().len()); + if cfg!(debug_assertions) { + // second pass to check indexes + for (index, live) in self.live_objects.get_mut().iter().enumerate() { + assert_eq!(live.as_ref().alloc_info.live_object_index as usize, index); + } + } + self.stage.finish_stage(CollectStage::Sweep); } #[inline(always)] @@ -36,6 +67,12 @@ impl OldGenerationSpace { Err(allocator_api2::alloc::AllocError) => return Err(OldAllocError::OutOfMemory), }; let header_ptr = raw_ptr.cast::>(); + let live_object_index: u32; + { + let live_objects = &mut *self.live_objects.get(); + live_object_index = u32::try_from(live_objects.len()).unwrap(); + live_objects.push(header_ptr); + } header_ptr.as_ptr().write(GcHeader { state_bits: Cell::new( GcStateBits::builder() @@ -45,9 +82,7 @@ impl OldGenerationSpace { .with_raw_mark_bits(GcMarkBits::White.to_raw(self)) .build(), ), - alloc_info: AllocInfo { - this_object_overall_size: overall_size as u32, - }, + alloc_info: AllocInfo { live_object_index }, metadata: HeaderMetadata { type_info }, collector_id: self.collector_id, }); diff --git a/src/context/young.rs b/src/context/young.rs index 0d13a21..da0bc17 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -7,7 +7,8 @@ use bumpalo::ChunkRawIter; use crate::context::old::OldGenerationSpace; use crate::context::{ - AllocInfo, GcHeader, GcStateBits, GcTypeInfo, GenerationId, GenerationKind, HeaderMetadata, + AllocInfo, CollectStage, CollectStageTracker, GcHeader, GcStateBits, GcTypeInfo, GenerationId, + GenerationKind, HeaderMetadata, }; use crate::utils::bumpalo_raw::{BumpAllocRaw, BumpAllocRawConfig}; use crate::utils::Alignment; @@ -22,6 +23,7 @@ use crate::CollectorId; pub struct YoungGenerationSpace { bump: BumpAllocRaw>, collector_id: Id, + stage: CollectStageTracker, } impl YoungGenerationSpace { /// The maximum size to allocate in the young generation. @@ -29,6 +31,13 @@ impl YoungGenerationSpace { /// Anything larger than this is immediately sent to the old generation. pub const SIZE_LIMIT: usize = 1024; + pub unsafe fn sweep(&mut self) { + self.stage + .begin_stage(Some(CollectStage::Mark), CollectStage::Sweep); + self.bump.reset(); + self.stage.finish_stage(CollectStage::Sweep); + } + #[inline(always)] pub unsafe fn alloc_uninit( &self, diff --git a/src/gcptr.rs b/src/gcptr.rs index f914581..2e497e4 100644 --- a/src/gcptr.rs +++ b/src/gcptr.rs @@ -33,6 +33,11 @@ impl<'gc, T: Collect, Id: CollectorId> Gc<'gc, T, Id> { GcTypeInfo::new::() } + #[inline(always)] + pub unsafe fn as_raw_ptr(&self) -> NonNull { + self.ptr + } + #[inline(always)] pub unsafe fn from_raw_ptr(ptr: NonNull) -> Self { Gc { diff --git a/src/utils/bumpalo_raw.rs b/src/utils/bumpalo_raw.rs index b3514fb..8f1a71e 100644 --- a/src/utils/bumpalo_raw.rs +++ b/src/utils/bumpalo_raw.rs @@ -16,6 +16,11 @@ impl BumpAllocRaw { self.inner.try_alloc_layout(layout) } + #[inline] + pub fn reset(&mut self) { + self.inner.reset(); + } + #[inline] pub unsafe fn iter_allocated_chunks_raw(&self) -> bumpalo::ChunkRawIter<'_> { self.inner.iter_allocated_chunks_raw() From b15b5d4b2e055f85c4eb35d354382a8f17ee4dcc Mon Sep 17 00:00:00 2001 From: Techcable Date: Thu, 2 May 2024 09:42:05 -0700 Subject: [PATCH 08/35] Cleanup alloc using RawAllocTarget trait --- src/context.rs | 206 +++++++++++++++++++++++++++++++++++++------ src/context/old.rs | 51 +++++------ src/context/young.rs | 68 ++++++-------- 3 files changed, 229 insertions(+), 96 deletions(-) diff --git a/src/context.rs b/src/context.rs index 10b3845..f0d371f 100644 --- a/src/context.rs +++ b/src/context.rs @@ -103,6 +103,75 @@ impl GarbageCollector { } } +unsafe trait RawAllocTarget { + const ARRAY: bool; + type Header: Sized; + fn header_metadata(&self) -> HeaderMetadata; + unsafe fn init_header(&self, header_ptr: NonNull, base_header: GcHeader); + fn overall_layout(&self) -> Layout; +} +struct RegularAlloc { + type_info: &'static GcTypeInfo, +} +unsafe impl RawAllocTarget for RegularAlloc { + const ARRAY: bool = false; + type Header = GcHeader; + + #[inline] + fn header_metadata(&self) -> HeaderMetadata { + HeaderMetadata { + type_info: self.type_info, + } + } + + #[inline] + unsafe fn init_header(&self, header_ptr: NonNull>, base_header: GcHeader) { + header_ptr.as_ptr().write(base_header) + } + + #[inline] + fn overall_layout(&self) -> Layout { + unsafe { + Layout::from_size_align_unchecked( + self.type_info.layout.overall_size, + GcHeader::::FIXED_ALIGNMENT, + ) + } + } +} +struct ArrayAlloc { + type_info: &'static GcArrayTypeInfo, + layout_info: GcArrayLayoutInfo, +} +unsafe impl RawAllocTarget for ArrayAlloc { + const ARRAY: bool = true; + type Header = GcArrayHeader; + + #[inline] + fn header_metadata(&self) -> HeaderMetadata { + HeaderMetadata { + array_type_info: self.type_info, + } + } + + #[inline] + unsafe fn init_header( + &self, + header_ptr: NonNull>, + base_header: GcHeader, + ) { + header_ptr.as_ptr().write(GcArrayHeader { + main_header: base_header, + len_elements: self.layout_info.len_elements, + }) + } + + #[inline] + fn overall_layout(&self) -> Layout { + self.layout_info.overall_layout() + } +} + /// The layout of a "regular" (non-array) type pub(crate) struct GcTypeLayout { /// The layout of the underlying value @@ -197,12 +266,23 @@ impl GcArrayTypeInfo { } type TraceFuncPtr = unsafe fn(NonNull<()>, &mut CollectContext); +#[repr(C)] pub(crate) struct GcTypeInfo { layout: GcTypeLayout, drop_func: Option, trace_func: Option>, } impl GcTypeInfo { + #[inline] + pub unsafe fn assume_array_info(&self) -> &'_ GcArrayTypeInfo { + // Takes advantage of fact repr is identical + assert_eq!( + std::mem::size_of::(), + std::mem::size_of::>() + ); + &*(self as *const Self as *const GcArrayTypeInfo) + } + #[inline] pub const fn new>() -> &'static Self { >::TYPE_INFO_REF @@ -431,6 +511,15 @@ impl GcArrayHeader { } } + #[inline] + pub fn layout_info(&self) -> GcArrayLayoutInfo { + GcArrayLayoutInfo { + element_layout: self.element_layout(), + len_elements: self.len_elements, + marker: PhantomData, + } + } + #[inline] fn element_layout(&self) -> Layout { self.resolve_type_info() @@ -439,6 +528,27 @@ impl GcArrayHeader { .value_layout } + #[inline] + fn value_layout(&self) -> Layout { + self.layout_info().value_layout() + } + + #[inline] + fn overall_layout(&self) -> Layout { + self.layout_info().overall_layout() + } +} +struct GcArrayLayoutInfo { + element_layout: Layout, + len_elements: usize, + marker: PhantomData<&'static Id>, +} +impl GcArrayLayoutInfo { + #[inline] + pub const fn element_layout(&self) -> Layout { + self.element_layout + } + #[cfg_attr(not(debug_assertions), inline)] fn value_layout(&self) -> Layout { let element_layout = self.element_layout(); @@ -552,7 +662,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { .resolve(self.garbage_collector) { GcMarkBits::White => { - let new_header = self.fallback_collect_gcptr(header); + let new_header = self.fallback_collect_gc_header(NonNull::from(header)); Gc::from_raw_ptr(new_header.as_ref().regular_value_ptr().cast()) } GcMarkBits::Black => { @@ -563,11 +673,12 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { } #[cold] - unsafe fn fallback_collect_gcheaer<'gc>( + unsafe fn fallback_collect_gc_header<'gc>( &mut self, header_ptr: NonNull>, ) -> NonNull> { let type_info: &'static GcTypeInfo; + let array = header_ptr.as_ref().state_bits.get().array(); let prev_generation: GenerationId; { let header = header_ptr.as_ref(); @@ -588,44 +699,81 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { } let forwarded_ptr = match prev_generation { GenerationId::Young => { - assert!( - !header_ptr.as_ref().state_bits.get().array(), - "TODO: Support arrays in youngen copy" - ); + let array_value_size: Option; // reallocate in oldgen - // TODO: This panic is fatal, will cause an abort - let forwarded_ptr = self - .garbage_collector - .old_generation - .alloc_uninit(type_info) - .expect("Oldgen alloc failure"); - forwarded_ptr + let copied_ptr = if array { + let array_type_info = type_info.assume_array_info(); + debug_assert!(std::ptr::eq( + array_type_info, + header_ptr.as_ref().metadata.array_type_info + )); + let array_layout = GcArrayLayoutInfo { + element_layout: array_type_info.element_type_info.layout.value_layout, + len_elements: header_ptr.cast::>().as_ref().len_elements, + marker: PhantomData, + }; + array_value_size = Some(array_layout.value_layout().size()); + self.garbage_collector + .old_generation + .alloc_raw(ArrayAlloc { + layout_info: array_layout, + type_info: array_type_info, + }) + .map(NonNull::cast::>) + } else { + array_value_size = None; + self.garbage_collector + .old_generation + .alloc_raw(RegularAlloc { type_info }) + } + .unwrap_or_else(|| { + // TODO: This panic is fatal, will cause an abort + panic!("Oldgen alloc failure") + }); + copied_ptr .as_ref() .state_bits .set(header_ptr.as_ref().state_bits.get()); - forwarded_ptr.as_ref().update_state_bits(|bits| { + copied_ptr.as_ref().update_state_bits(|bits| { debug_assert!(!bits.forwarded()); bits.with_generation(GenerationId::Old); }); header_ptr.as_ref().update_state_bits(|bits| { bits.with_forwarded(true); }); - (&mut *header_ptr.as_ptr()).metadata.forward_ptr = forwarded_ptr.cast(); + (&mut *header_ptr.as_ptr()).metadata.forward_ptr = copied_ptr.cast(); // NOTE: Copy uninitialized bytes is safe here, as long as they are not read in dest - forwarded_ptr - .as_ref() - .regular_value_ptr() - .cast::() - .as_ptr() - .copy_from_nonoverlapping( - header_ptr - .as_ref() - .regular_value_ptr() - .cast::() - .as_ptr(), - type_info.layout.value_layout().size(), - ); - forwarded_ptr + if array { + copied_ptr + .cast::>() + .as_ref() + .array_value_ptr() + .cast::() + .as_ptr() + .copy_from_nonoverlapping( + header_ptr + .cast::>() + .as_ref() + .array_value_ptr() + .as_ptr(), + array_value_size.unwrap(), + ) + } else { + copied_ptr + .as_ref() + .regular_value_ptr() + .cast::() + .as_ptr() + .copy_from_nonoverlapping( + header_ptr + .as_ref() + .regular_value_ptr() + .cast::() + .as_ptr(), + type_info.layout.value_layout().size(), + ); + } + copied_ptr } GenerationId::Old => header_ptr, // no copying needed for oldgen }; diff --git a/src/context/old.rs b/src/context/old.rs index ebf706e..69b6083 100644 --- a/src/context/old.rs +++ b/src/context/old.rs @@ -6,9 +6,10 @@ use zerogc_next_mimalloc_semisafe::heap::MimallocHeap; use crate::context::young::YoungGenerationSpace; use crate::context::{ - AllocInfo, CollectStage, CollectStageTracker, GcHeader, GcMarkBits, GcStateBits, GcTypeInfo, - GenerationId, HeaderMetadata, + AllocInfo, CollectStage, CollectStageTracker, GcArrayTypeInfo, GcHeader, GcMarkBits, + GcStateBits, GcTypeInfo, GenerationId, HeaderMetadata, }; +use crate::gcptr::Gc; use crate::CollectorId; pub struct OldGenerationSpace { @@ -53,39 +54,39 @@ impl OldGenerationSpace { self.stage.finish_stage(CollectStage::Sweep); } - #[inline(always)] - pub unsafe fn alloc_uninit( + #[inline] + pub unsafe fn alloc_raw>( &self, - type_info: &'static GcTypeInfo, - ) -> Result>, OldAllocError> { - let overall_size = type_info.layout.overall_size; - let raw_ptr = match self.heap.allocate(Layout::from_size_align_unchecked( - overall_size, - GcHeader::::FIXED_ALIGNMENT, - )) { + target: T, + ) -> Result, OldAllocError> { + let overall_layout = target.overall_layout(); + let raw_ptr = match self.heap.allocate(overall_layout) { Ok(raw_ptr) => raw_ptr, Err(allocator_api2::alloc::AllocError) => return Err(OldAllocError::OutOfMemory), }; - let header_ptr = raw_ptr.cast::>(); + let header_ptr = raw_ptr.cast::(); let live_object_index: u32; { let live_objects = &mut *self.live_objects.get(); live_object_index = u32::try_from(live_objects.len()).unwrap(); live_objects.push(header_ptr); } - header_ptr.as_ptr().write(GcHeader { - state_bits: Cell::new( - GcStateBits::builder() - .with_forwarded(false) - .with_generation(GenerationId::Old) - .with_array(false) - .with_raw_mark_bits(GcMarkBits::White.to_raw(self)) - .build(), - ), - alloc_info: AllocInfo { live_object_index }, - metadata: HeaderMetadata { type_info }, - collector_id: self.collector_id, - }); + target.init_header( + header_ptr, + GcHeader { + state_bits: Cell::new( + GcStateBits::builder() + .with_forwarded(false) + .with_generation(GenerationId::Old) + .with_array(T::ARRAY) + .with_raw_mark_bits(GcMarkBits::White.to_raw(self)) + .build(), + ), + alloc_info: AllocInfo { live_object_index }, + metadata: HeaderMetadata { type_info }, + collector_id: self.collector_id, + }, + ); Ok(header_ptr) } } diff --git a/src/context/young.rs b/src/context/young.rs index da0bc17..d4029b9 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -7,8 +7,8 @@ use bumpalo::ChunkRawIter; use crate::context::old::OldGenerationSpace; use crate::context::{ - AllocInfo, CollectStage, CollectStageTracker, GcHeader, GcStateBits, GcTypeInfo, GenerationId, - GenerationKind, HeaderMetadata, + AllocInfo, CollectStage, CollectStageTracker, GcHeader, GcStateBits, GenerationId, + HeaderMetadata, }; use crate::utils::bumpalo_raw::{BumpAllocRaw, BumpAllocRawConfig}; use crate::utils::Alignment; @@ -38,39 +38,36 @@ impl YoungGenerationSpace { self.stage.finish_stage(CollectStage::Sweep); } - #[inline(always)] - pub unsafe fn alloc_uninit( + #[inline] + pub unsafe fn alloc_raw>( &self, - type_info: &'static GcTypeInfo, - ) -> Result>, YoungAllocError> { - let overall_size = type_info.layout.overall_size; - if overall_size > Self::SIZE_LIMIT { + target: T, + ) -> Result, YoungAllocError> { + let overall_layout = target.overall_layout(); + if overall_layout.size() > Self::SIZE_LIMIT { return Err(YoungAllocError::SizeExceedsLimit); } - let Ok(raw_ptr) = self - .bump - .try_alloc_layout(Layout::from_size_align_unchecked( - overall_size, - GcHeader::::FIXED_ALIGNMENT, - )) - else { + let Ok(raw_ptr) = self.bump.try_alloc_layout(overall_layout) else { return Err(YoungAllocError::OutOfMemory); }; - let header_ptr = raw_ptr.cast::>(); - header_ptr.as_ptr().write(GcHeader { - state_bits: Cell::new( - GcStateBits::builder() - .with_forwarded(false) - .with_generation(GenerationId::Young) - .with_array(false) - .build(), - ), - alloc_info: AllocInfo { - this_object_overall_size: overall_size as u32, + let header_ptr = raw_ptr.cast::(); + target.init_header( + header_ptr, + GcHeader { + state_bits: Cell::new( + GcStateBits::builder() + .with_forwarded(false) + .with_generation(GenerationId::Young) + .with_array(T::ARRAY) + .build(), + ), + alloc_info: AllocInfo { + this_object_overall_size: overall_layout.size() as u32, + }, + metadata: HeaderMetadata { type_info }, + collector_id: self.collector_id, }, - metadata: HeaderMetadata { type_info }, - collector_id: self.collector_id, - }); + ); Ok(header_ptr) } @@ -83,19 +80,6 @@ impl YoungGenerationSpace { } } } -unsafe impl super::Generation for YoungGenerationSpace { - const ID: GenerationId = GenerationId::Young; - - #[inline] - fn cast_young(&self) -> Option<&'_ YoungGenerationSpace> { - Some(self) - } - - #[inline] - fn cast_old(&self) -> Option<&'_ OldGenerationSpace> { - None - } -} #[derive(Debug, thiserror::Error)] enum YoungAllocError { #[error("Out of memory")] From e3bf38501a16ca400ca15355af84295d15d67387 Mon Sep 17 00:00:00 2001 From: Techcable Date: Sat, 4 May 2024 14:06:25 -0700 Subject: [PATCH 09/35] Split layout code into seperate module --- libs/macros/src/collect_impl.rs | 24 +- libs/macros/src/helpers.rs | 3 +- libs/macros/src/lib.rs | 2 - libs/mimalloc-semisafe/src/heap.rs | 19 +- src/context.rs | 514 ++++------------------------- src/context/layout.rs | 492 +++++++++++++++++++++++++++ src/context/old.rs | 42 +-- src/context/young.rs | 18 +- src/gcptr.rs | 3 +- src/utils/layout_helpers.rs | 4 +- 10 files changed, 596 insertions(+), 525 deletions(-) create mode 100644 src/context/layout.rs diff --git a/libs/macros/src/collect_impl.rs b/libs/macros/src/collect_impl.rs index 32949bf..e275f60 100644 --- a/libs/macros/src/collect_impl.rs +++ b/libs/macros/src/collect_impl.rs @@ -8,10 +8,10 @@ //! There is no copyright issue because I am the only author. use indexmap::{indexmap, IndexMap}; use proc_macro2::{Ident, Span, TokenStream}; -use proc_macro_kwargs::parse::{NestedDict, NestedList, Syn}; +use proc_macro_kwargs::parse::{NestedDict, NestedList}; use proc_macro_kwargs::{MacroArg, MacroKeywordArgs}; use quote::quote; -use std::process::id; +use syn::ext::IdentExt; use syn::parse::ParseStream; use syn::{ braced, parse_quote, Error, Expr, GenericParam, Generics, Lifetime, Path, Token, Type, @@ -49,7 +49,7 @@ pub struct MacroInput { /// Requirements for implementing `NullCollect` /// /// This is unsafe (obviously) and has no static checking. - null_collect: Option, + null_collect: TraitRequirements, /// The associated type `Collect::Collected<'newgc>` /// /// Implicitly takes a `'newgc` parameter @@ -150,7 +150,7 @@ impl MacroInput { let zerogc_next_crate = zerogc_next_crate(); let target_type = &self.target_type; let mut generics: syn::Generics = self.basic_generics(); - let collector_id = self.setup_collector_id_generics(&mut generics); + let collector_id = self.setup_collector_id_generics(&mut generics)?; { let clause = self.bounds.where_clause_collect(&self.params.elements)?; generics @@ -200,9 +200,9 @@ impl CustomBounds { ) -> Result { match self.collect { Some(TraitRequirements::Never { span }) => { - return Err(syn::Error::new(span, "Collect must always be implemented")) + Err(syn::Error::new(span, "Collect must always be implemented")) } - Some(TraitRequirements::Always) => Ok(empty_clause()), // No requirements + Some(TraitRequirements::Always { span: _ }) => Ok(empty_clause()), // No requirements Some(TraitRequirements::Where(ref explicit)) => Ok(explicit.clone()), None => { // generate the implicit requirements @@ -258,15 +258,13 @@ impl MacroArg for TraitRequirements { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub enum CollectorIdInfo { + #[default] Any, - Specific { map: IndexMap }, -} -impl Default for CollectorIdInfo { - fn default() -> Self { - CollectorIdInfo::Any - } + Specific { + map: IndexMap, + }, } impl CollectorIdInfo { /// Create info from a single `CollectorId`, diff --git a/libs/macros/src/helpers.rs b/libs/macros/src/helpers.rs index 10f3d0c..d6ba354 100644 --- a/libs/macros/src/helpers.rs +++ b/libs/macros/src/helpers.rs @@ -44,8 +44,7 @@ pub fn rewrite_type( *qself = qself .clone() .map::, _>(|mut qself| { - qself.ty = - Box::new(rewrite_type(&*qself.ty, target_type_name, &mut *rewriter)?); + qself.ty = Box::new(rewrite_type(&qself.ty, target_type_name, &mut *rewriter)?); Ok(qself) }) .transpose()?; diff --git a/libs/macros/src/lib.rs b/libs/macros/src/lib.rs index 9adb2e8..a8c75f2 100644 --- a/libs/macros/src/lib.rs +++ b/libs/macros/src/lib.rs @@ -1,5 +1,3 @@ -use proc_macro2::TokenStream; - mod collect_impl; pub(crate) mod helpers; diff --git a/libs/mimalloc-semisafe/src/heap.rs b/libs/mimalloc-semisafe/src/heap.rs index eac3a4b..44e5fff 100644 --- a/libs/mimalloc-semisafe/src/heap.rs +++ b/libs/mimalloc-semisafe/src/heap.rs @@ -42,6 +42,13 @@ impl MimallocHeap { } /// A raw pointer to the underlying heap + /// + /// ## Safety + /// Should not unexpectedly free any allocations or destroy the arena, + /// as that would violate the [`Allocator`] api. + /// + /// Technically, those operations would themselves be `unsafe`, + /// so this is slightly redundant. #[inline] pub unsafe fn as_raw(&self) -> *mut sys::mi_heap_t { self.raw.as_ptr() @@ -52,10 +59,7 @@ impl MimallocHeap { if ptr.is_null() { Err(AllocError) } else { - Ok(NonNull::from(std::slice::from_raw_parts_mut( - ptr as *mut u8, - size, - ))) + Ok(NonNull::from(std::slice::from_raw_parts_mut(ptr, size))) } } @@ -159,3 +163,10 @@ impl Drop for MimallocHeap { } } } + +impl Default for MimallocHeap { + #[inline] + fn default() -> Self { + Self::new() + } +} diff --git a/src/context.rs b/src/context.rs index f0d371f..b01dfaf 100644 --- a/src/context.rs +++ b/src/context.rs @@ -2,17 +2,20 @@ use std::alloc::Layout; use std::cell::Cell; use std::fmt::Debug; use std::marker::PhantomData; -use std::os::macos::raw::stat; use std::ptr::NonNull; -use bitbybit::{bitenum, bitfield}; +use bitbybit::bitenum; +use crate::context::layout::{ + GcArrayHeader, GcArrayLayoutInfo, GcArrayTypeInfo, GcHeader, GcMarkBits, GcStateBits, + GcTypeInfo, HeaderMetadata, TraceFuncPtr, +}; use crate::context::old::OldGenerationSpace; use crate::context::young::YoungGenerationSpace; use crate::gcptr::Gc; -use crate::utils::LayoutExt; use crate::Collect; +pub(crate) mod layout; mod old; mod young; @@ -62,14 +65,14 @@ impl CollectStageTracker { Some(last_stage) => CollectStageTracker::FinishedStage { last_stage }, None => CollectStageTracker::NotCollecting, }, - self + *self ); *self = CollectStageTracker::Stage { current: new_stage }; } #[inline] fn finish_stage(&mut self, stage: CollectStage) { - assert_eq!(CollectStageTracker::Stage { current: stage }, self); + assert_eq!(CollectStageTracker::Stage { current: stage }, *self); *self = CollectStageTracker::FinishedStage { last_stage: stage }; } } @@ -80,16 +83,24 @@ enum CollectStage { Sweep, } +/// The state of a [GarbageCollector] +/// +/// Seperated out to pass around as a separate reference. +/// This is important to avoid `&mut` from different sub-structures. +pub(crate) struct CollectorState { + collector_id: Id, + mark_bits_inverted: Cell, +} + pub struct GarbageCollector { - id: Id, + state: CollectorState, young_generation: YoungGenerationSpace, old_generation: OldGenerationSpace, - mark_bits_inverted: bool, } impl GarbageCollector { #[inline] pub fn id(&self) -> Id { - self.id + self.state.collector_id } #[inline(always)] @@ -109,11 +120,23 @@ unsafe trait RawAllocTarget { fn header_metadata(&self) -> HeaderMetadata; unsafe fn init_header(&self, header_ptr: NonNull, base_header: GcHeader); fn overall_layout(&self) -> Layout; + #[inline] + fn init_state_bits(&self, gen: GenerationId) -> GcStateBits { + GcStateBits::builder() + .with_forwarded(false) + .with_generation(gen) + .with_array(Self::ARRAY) + .with_raw_mark_bits(GcMarkBits::White.to_raw(self.collector_state())) + .build() + } + + fn collector_state(&self) -> &'_ CollectorState; } -struct RegularAlloc { +struct RegularAlloc<'a, Id: CollectorId> { + state: &'a CollectorState, type_info: &'static GcTypeInfo, } -unsafe impl RawAllocTarget for RegularAlloc { +unsafe impl RawAllocTarget for RegularAlloc<'_, Id> { const ARRAY: bool = false; type Header = GcHeader; @@ -133,17 +156,23 @@ unsafe impl RawAllocTarget for RegularAlloc { fn overall_layout(&self) -> Layout { unsafe { Layout::from_size_align_unchecked( - self.type_info.layout.overall_size, + self.type_info.layout.overall_layout().size(), GcHeader::::FIXED_ALIGNMENT, ) } } + + #[inline] + fn collector_state(&self) -> &'_ CollectorState { + self.state + } } -struct ArrayAlloc { +struct ArrayAlloc<'a, Id: CollectorId> { type_info: &'static GcArrayTypeInfo, layout_info: GcArrayLayoutInfo, + state: &'a CollectorState, } -unsafe impl RawAllocTarget for ArrayAlloc { +unsafe impl RawAllocTarget for ArrayAlloc<'_, Id> { const ARRAY: bool = true; type Header = GcArrayHeader; @@ -162,7 +191,7 @@ unsafe impl RawAllocTarget for ArrayAlloc { ) { header_ptr.as_ptr().write(GcArrayHeader { main_header: base_header, - len_elements: self.layout_info.len_elements, + len_elements: self.layout_info.len_elements(), }) } @@ -170,158 +199,12 @@ unsafe impl RawAllocTarget for ArrayAlloc { fn overall_layout(&self) -> Layout { self.layout_info.overall_layout() } -} - -/// The layout of a "regular" (non-array) type -pub(crate) struct GcTypeLayout { - /// The layout of the underlying value - /// - /// INVARIANT: The maximum alignment is [`GcHeader::FIXED_ALIGNMENT`] - value_layout: Layout, - /// The overall size of the value including the header - /// and trailing padding. - overall_size: usize, - marker: PhantomData<&'static Id>, -} -impl GcTypeLayout { - #[inline] - pub const fn value_size(&self) -> usize { - self.value_layout.size() - } - - #[inline] - pub const fn value_align(&self) -> usize { - self.value_layout.align() - } - - #[inline] - pub const fn value_layout(&self) -> Layout { - self.value_layout - } - - #[inline] - pub const fn overall_layout(&self) -> Layout { - unsafe { - Layout::from_size_align_unchecked(self.overall_size, GcHeader::::FIXED_ALIGNMENT) - } - } - - //noinspection RsAssertEqual - const fn compute_overall_layout(value_layout: Layout) -> Layout { - let header_layout = GcHeader::::REGULAR_HEADER_LAYOUT; - let Ok((expected_overall_layout, value_offset)) = - LayoutExt(header_layout).extend(value_layout) - else { - panic!("layout overflow") - }; - assert!( - value_offset == GcHeader::::REGULAR_VALUE_OFFSET, - "Unexpected value offset" - ); - let res = LayoutExt(expected_overall_layout).pad_to_align(); - assert!( - res.align() == GcHeader::::FIXED_ALIGNMENT, - "Unexpected overall alignment" - ); - res - } - #[track_caller] - pub const fn from_value_layout(value_layout: Layout) -> Self { - assert!( - value_layout.align() <= GcHeader::::FIXED_ALIGNMENT, - "Alignment exceeds maximum", - ); - let overall_layout = Self::compute_overall_layout(value_layout); - GcTypeLayout { - value_layout, - overall_size: overall_layout.size(), - marker: PhantomData, - } - } -} - -#[repr(transparent)] -pub(crate) struct GcArrayTypeInfo { - /// The type info for the array's elements. - /// - /// This is stored as the first element to allow one-way - /// pointer casts from GcArrayTypeInfo -> GcTypeInfo. - /// This simulates OO-style inheritance. - element_type_info: GcTypeInfo, -} - -impl GcArrayTypeInfo { - //noinspection RsAssertEqual - #[inline] - pub const fn new>() -> &'static Self { - /* - * for the time being GcTypeInfo <--> GcArrayTypeInfo, - * so we just cast the pointers - */ - assert!(std::mem::size_of::() == std::mem::size_of::>()); - unsafe { - &*(GcTypeInfo::::new::() as *const GcTypeInfo as *const GcArrayTypeInfo) - } - } -} - -type TraceFuncPtr = unsafe fn(NonNull<()>, &mut CollectContext); -#[repr(C)] -pub(crate) struct GcTypeInfo { - layout: GcTypeLayout, - drop_func: Option, - trace_func: Option>, -} -impl GcTypeInfo { - #[inline] - pub unsafe fn assume_array_info(&self) -> &'_ GcArrayTypeInfo { - // Takes advantage of fact repr is identical - assert_eq!( - std::mem::size_of::(), - std::mem::size_of::>() - ); - &*(self as *const Self as *const GcArrayTypeInfo) - } #[inline] - pub const fn new>() -> &'static Self { - >::TYPE_INFO_REF + fn collector_state(&self) -> &'_ CollectorState { + self.state } } -trait TypeIdInit> { - const TYPE_INFO_INIT_VAL: GcTypeInfo = { - let layout = GcTypeLayout::from_value_layout(Layout::new::()); - let drop_func = if std::mem::needs_drop::() { - unsafe { - Some(std::mem::transmute::<_, unsafe fn(*mut ())>( - std::ptr::drop_in_place as unsafe fn(*mut T), - )) - } - } else { - None - }; - let trace_func = if T::NEEDS_COLLECT { - unsafe { - Some(std::mem::transmute::< - _, - unsafe fn(NonNull<()>, &mut CollectContext), - >( - T::collect_inplace as unsafe fn(NonNull, &mut CollectContext), - )) - } - } else { - None - }; - GcTypeInfo { - layout, - drop_func, - trace_func, - } - }; - const TYPE_INFO_REF: &'static GcTypeInfo = &Self::TYPE_INFO_INIT_VAL; -} -struct GcTypeInitImpl; -impl> TypeIdInit for GcTypeInitImpl {} #[derive(Debug, Eq, PartialEq)] #[bitenum(u1, exhaustive = true)] @@ -330,283 +213,6 @@ enum GenerationId { Old = 1, } -/// The raw bit representation of [GcMarkBits] -type GcMarkBitsRepr = arbitrary_int::UInt; - -#[bitenum(u1, exhaustive = true)] -enum GcMarkBits { - /// Indicates that tracing has not yet marked the object. - /// - /// Once tracing completes, this means the object is dead. - White = 0, - /// Indicates that tracing has marked the object. - /// - /// This means the object is live. - Black = 1, -} - -impl GcMarkBits { - #[inline] - pub fn to_raw(&self, collector: &GarbageCollector) -> GcRawMarkBits { - let bits: GcMarkBitsRepr = self.raw_value(); - GcRawMarkBits::new_with_raw_value(if collector.mark_bits_inverted { - GcRawMarkBits::invert_bits(bits) - } else { - bits - }) - } -} - -#[bitenum(u1, exhaustive = true)] -enum GcRawMarkBits { - Red = 0, - Green = 1, -} -impl GcRawMarkBits { - #[inline] - pub fn resolve(&self, collector: &GarbageCollector) -> GcMarkBits { - let bits: GcMarkBitsRepr = self.raw_value(); - GcMarkBits::new_with_raw_value(if collector.mark_bits_inverted { - Self::invert_bits(bits) - } else { - bits - }) - } - - #[inline] - fn invert_bits(bits: GcMarkBitsRepr) -> GcMarkBitsRepr { - ::MAX - bits - } -} - -/// A bitfield for the garbage collector's state. -/// -/// ## Default -/// The `DEFAULT` value isn't valid here. -/// However, it currently needs to exist fo -/// the macro to generate the `builder` field -#[bitfield(u32, default = 0)] -struct GcStateBits { - #[bit(0, rw)] - forwarded: bool, - #[bit(1, rw)] - generation: GenerationId, - #[bit(2, rw)] - array: bool, - #[bit(3, rw)] - raw_mark_bits: GcRawMarkBits, -} -union HeaderMetadata { - type_info: &'static GcTypeInfo, - array_type_info: &'static GcArrayTypeInfo, - forward_ptr: NonNull>, -} -union AllocInfo { - /// The [overall size][`GcTypeLayout::overall_layout`] of this object. - /// - /// This is used to iterate over objects in the young generation. - /// - /// Objects whose size cannot fit into a `u32` - /// can never be allocated in the young generation. - /// - /// If this object is an array, - /// this is the overall size of - /// the header and all elements. - pub this_object_overall_size: u32, - /// The index of the object within the vector of live objects. - /// - /// This is used in the old generation. - pub live_object_index: u32, -} - -#[repr(C, align(8))] -pub(crate) struct GcHeader { - pub(crate) state_bits: Cell, - pub(crate) alloc_info: AllocInfo, - pub(crate) metadata: HeaderMetadata, - /// The id for the collector where this object is allocated. - /// - /// If the collector is a singleton (either global or thread-local), - /// this will be a zero sized type. - /// - /// ## Safety - /// The alignment of this type must be smaller than [`GcHeader::FIXED_ALIGNMENT`]. - pub collector_id: Id, -} -impl GcHeader { - #[inline] - pub(crate) unsafe fn update_state_bits(&self, func: impl FnOnce(&mut GcStateBits)) { - let mut bits = self.state_bits.get(); - func(&mut bits); - self.state_bits.set(bits); - } - - /// The fixed alignment for all GC types - /// - /// Allocating a type with an alignment greater than this is an error. - pub const FIXED_ALIGNMENT: usize = 8; - /// The fixed offset from the start of the GcHeader to a regular value - pub const REGULAR_VALUE_OFFSET: usize = std::mem::size_of::(); - pub const ARRAY_VALUE_OFFSET: usize = std::mem::size_of::>(); - const REGULAR_HEADER_LAYOUT: Layout = Layout::new::(); - const ARRAY_HEADER_LAYOUT: Layout = Layout::new::>(); - - #[inline] - pub fn id(&self) -> Id { - self.collector_id - } - - #[inline] - fn resolve_type_info(&self) -> &'static GcTypeInfo { - unsafe { - if self.state_bits.get().forwarded() { - let forward_ptr = self.metadata.forward_ptr; - let forward_header = forward_ptr.as_ref(); - debug_assert!(!forward_header.state_bits.get().forwarded()); - forward_header.metadata.type_info - } else { - self.metadata.type_info - } - } - } - - #[inline] - pub fn regular_value_ptr(&self) -> NonNull { - unsafe { - NonNull::new_unchecked( - (self as *const Self as *mut Self as *mut u8).add(Self::REGULAR_VALUE_OFFSET), - ) - } - } - - #[inline] - pub unsafe fn assume_array_header(&self) -> &'_ GcArrayHeader { - &*(self as *const Self as *const GcArrayHeader) - } -} - -#[repr(C, align(8))] -pub struct GcArrayHeader { - main_header: GcHeader, - /// The length of the array in elements - len_elements: usize, -} - -impl GcArrayHeader { - #[inline] - fn resolve_type_info(&self) -> &'static GcArrayTypeInfo { - unsafe { - &*(self.main_header.resolve_type_info() as *const GcTypeInfo - as *const GcArrayTypeInfo) - } - } - - #[inline] - pub fn array_value_ptr(&self) -> NonNull { - unsafe { - NonNull::new_unchecked( - (self as *const Self as *mut Self as *mut u8) - .add(GcHeader::::ARRAY_VALUE_OFFSET), - ) - } - } - - #[inline] - pub fn layout_info(&self) -> GcArrayLayoutInfo { - GcArrayLayoutInfo { - element_layout: self.element_layout(), - len_elements: self.len_elements, - marker: PhantomData, - } - } - - #[inline] - fn element_layout(&self) -> Layout { - self.resolve_type_info() - .element_type_info - .layout - .value_layout - } - - #[inline] - fn value_layout(&self) -> Layout { - self.layout_info().value_layout() - } - - #[inline] - fn overall_layout(&self) -> Layout { - self.layout_info().overall_layout() - } -} -struct GcArrayLayoutInfo { - element_layout: Layout, - len_elements: usize, - marker: PhantomData<&'static Id>, -} -impl GcArrayLayoutInfo { - #[inline] - pub const fn element_layout(&self) -> Layout { - self.element_layout - } - - #[cfg_attr(not(debug_assertions), inline)] - fn value_layout(&self) -> Layout { - let element_layout = self.element_layout(); - if cfg!(debug_assertions) { - debug_assert!(element_layout.align() <= GcHeader::::FIXED_ALIGNMENT); - debug_assert_eq!( - element_layout.pad_to_align(), - element_layout, - "padding should already be included" - ); - let Some(repeated_size) = element_layout.size().checked_mul(self.len_elements) else { - panic!( - "Invalid length {} triggers size overflow for {element_layout:?}", - self.len_elements - ) - }; - debug_assert!( - Layout::from_size_align(repeated_size, element_layout.align()).is_ok(), - "align overflow" - ); - } - unsafe { - Layout::from_size_align_unchecked( - element_layout.size().unchecked_mul(self.len_elements), - element_layout.align(), - ) - } - } - - #[cfg_attr(not(debug_assertions), inline)] - fn overall_layout(&self) -> Layout { - let value_layout = self.value_layout(); - if cfg!(debug_assertions) { - let Ok((overall_layout, actual_offset)) = - LayoutExt(GcHeader::::ARRAY_HEADER_LAYOUT).extend(value_layout) - else { - unreachable!("layout overflow") - }; - debug_assert_eq!(actual_offset, GcHeader::::ARRAY_VALUE_OFFSET); - debug_assert_eq!( - Some(overall_layout.size()), - value_layout - .size() - .checked_add(GcHeader::::ARRAY_VALUE_OFFSET) - ); - } - unsafe { - Layout::from_size_align_unchecked( - value_layout - .size() - .unchecked_add(GcHeader::::ARRAY_VALUE_OFFSET), - GcHeader::::FIXED_ALIGNMENT, - ) - .pad_to_align() - } - } -} - pub struct CollectContext<'newgc, Id: CollectorId> { id: Id, garbage_collector: &'newgc mut GarbageCollector, @@ -643,7 +249,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { .state_bits .get() .raw_mark_bits() - .resolve(&self.garbage_collector.young_generation), + .resolve(&self.garbage_collector.state), GcMarkBits::Black ); return Gc::from_raw_ptr( @@ -659,7 +265,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { .state_bits .get() .raw_mark_bits() - .resolve(self.garbage_collector) + .resolve(&self.garbage_collector.state) { GcMarkBits::White => { let new_header = self.fallback_collect_gc_header(NonNull::from(header)); @@ -667,7 +273,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { } GcMarkBits::Black => { // already traced, can skip it - Gc::from_raw_ptr(target.as_raw_ptr()) + Gc::from_raw_ptr(target.as_raw_ptr().cast()) } } } @@ -687,12 +293,13 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { .state_bits .get() .raw_mark_bits() - .resolve(self.garbage_collector), + .resolve(&self.garbage_collector.state), GcMarkBits::White ); // mark as black header.update_state_bits(|state_bits| { - state_bits.with_raw_mark_bits(GcMarkBits::Black.to_raw(self.garbage_collector)); + state_bits + .with_raw_mark_bits(GcMarkBits::Black.to_raw(&self.garbage_collector.state)); }); prev_generation = header.state_bits.get().generation(); type_info = header.metadata.type_info; @@ -707,26 +314,29 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { array_type_info, header_ptr.as_ref().metadata.array_type_info )); - let array_layout = GcArrayLayoutInfo { - element_layout: array_type_info.element_type_info.layout.value_layout, - len_elements: header_ptr.cast::>().as_ref().len_elements, - marker: PhantomData, - }; + let array_layout = GcArrayLayoutInfo::new_unchecked( + array_type_info.element_type_info.layout.value_layout(), + header_ptr.cast::>().as_ref().len_elements, + ); array_value_size = Some(array_layout.value_layout().size()); self.garbage_collector .old_generation .alloc_raw(ArrayAlloc { layout_info: array_layout, type_info: array_type_info, + state: &self.garbage_collector.state, }) .map(NonNull::cast::>) } else { array_value_size = None; self.garbage_collector .old_generation - .alloc_raw(RegularAlloc { type_info }) + .alloc_raw(RegularAlloc { + type_info, + state: &self.garbage_collector.state, + }) } - .unwrap_or_else(|| { + .unwrap_or_else(|_| { // TODO: This panic is fatal, will cause an abort panic!("Oldgen alloc failure") }); @@ -820,7 +430,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { let type_info = header.as_ref().main_header.metadata.type_info; debug_assert_eq!(type_info.trace_func, Some(trace_func)); let array_header = header.cast::>(); - let element_layout = type_info.layout.value_layout; + let element_layout = type_info.layout.value_layout(); let len = array_header.as_ref().len_elements; let element_start_ptr = array_header.as_ref().array_value_ptr(); for i in 0..len { diff --git a/src/context/layout.rs b/src/context/layout.rs new file mode 100644 index 0000000..ac97d56 --- /dev/null +++ b/src/context/layout.rs @@ -0,0 +1,492 @@ +use crate::context::{CollectorState, GenerationId}; +use crate::utils::LayoutExt; +use crate::{Collect, CollectContext, CollectorId}; +use bitbybit::{bitenum, bitfield}; +use std::alloc::Layout; +use std::cell::Cell; +use std::marker::PhantomData; +use std::ptr::NonNull; + +/// The layout of a "regular" (non-array) type +pub(crate) struct GcTypeLayout { + /// The layout of the underlying value + /// + /// INVARIANT: The maximum alignment is [`GcHeader::FIXED_ALIGNMENT`] + value_layout: Layout, + /// The overall size of the value including the header + /// and trailing padding. + overall_size: usize, + marker: PhantomData<&'static Id>, +} +impl GcTypeLayout { + #[inline] + pub const fn value_size(&self) -> usize { + self.value_layout.size() + } + + #[inline] + pub const fn value_align(&self) -> usize { + self.value_layout.align() + } + + #[inline] + pub const fn value_layout(&self) -> Layout { + self.value_layout + } + + #[inline] + pub const fn overall_layout(&self) -> Layout { + unsafe { + Layout::from_size_align_unchecked(self.overall_size, GcHeader::::FIXED_ALIGNMENT) + } + } + + //noinspection RsAssertEqual + const fn compute_overall_layout(value_layout: Layout) -> Layout { + let header_layout = GcHeader::::REGULAR_HEADER_LAYOUT; + let Ok((expected_overall_layout, value_offset)) = + LayoutExt(header_layout).extend(value_layout) + else { + panic!("layout overflow") + }; + assert!( + value_offset == GcHeader::::REGULAR_VALUE_OFFSET, + "Unexpected value offset" + ); + let res = LayoutExt(expected_overall_layout).pad_to_align(); + assert!( + res.align() == GcHeader::::FIXED_ALIGNMENT, + "Unexpected overall alignment" + ); + res + } + + #[track_caller] + pub const fn from_value_layout(value_layout: Layout) -> Self { + assert!( + value_layout.align() <= GcHeader::::FIXED_ALIGNMENT, + "Alignment exceeds maximum", + ); + let overall_layout = Self::compute_overall_layout(value_layout); + GcTypeLayout { + value_layout, + overall_size: overall_layout.size(), + marker: PhantomData, + } + } +} + +#[repr(transparent)] +pub(crate) struct GcArrayTypeInfo { + /// The type info for the array's elements. + /// + /// This is stored as the first element to allow one-way + /// pointer casts from GcArrayTypeInfo -> GcTypeInfo. + /// This simulates OO-style inheritance. + pub(super) element_type_info: GcTypeInfo, +} + +impl GcArrayTypeInfo { + //noinspection RsAssertEqual + #[inline] + pub const fn new>() -> &'static Self { + /* + * for the time being GcTypeInfo <--> GcArrayTypeInfo, + * so we just cast the pointers + */ + assert!(std::mem::size_of::() == std::mem::size_of::>()); + unsafe { + &*(GcTypeInfo::::new::() as *const GcTypeInfo as *const GcArrayTypeInfo) + } + } +} + +pub type TraceFuncPtr = unsafe fn(NonNull<()>, &mut CollectContext); + +#[repr(C)] +pub(crate) struct GcTypeInfo { + pub(super) layout: GcTypeLayout, + pub(super) drop_func: Option, + pub(super) trace_func: Option>, +} +impl GcTypeInfo { + #[inline] + pub unsafe fn assume_array_info(&self) -> &'_ GcArrayTypeInfo { + // Takes advantage of fact repr is identical + assert_eq!( + std::mem::size_of::(), + std::mem::size_of::>() + ); + &*(self as *const Self as *const GcArrayTypeInfo) + } + + #[inline] + pub const fn new>() -> &'static Self { + >::TYPE_INFO_REF + } +} +trait TypeIdInit> { + const TYPE_INFO_INIT_VAL: GcTypeInfo = { + let layout = GcTypeLayout::from_value_layout(Layout::new::()); + let drop_func = if std::mem::needs_drop::() { + unsafe { + Some(std::mem::transmute::<_, unsafe fn(*mut ())>( + std::ptr::drop_in_place as unsafe fn(*mut T), + )) + } + } else { + None + }; + let trace_func = if T::NEEDS_COLLECT { + unsafe { + Some(std::mem::transmute::< + _, + unsafe fn(NonNull<()>, &mut CollectContext), + >( + T::collect_inplace as unsafe fn(NonNull, &mut CollectContext), + )) + } + } else { + None + }; + GcTypeInfo { + layout, + drop_func, + trace_func, + } + }; + const TYPE_INFO_REF: &'static GcTypeInfo = &Self::TYPE_INFO_INIT_VAL; +} +struct GcTypeInitImpl; +impl> TypeIdInit for GcTypeInitImpl {} + +/// The raw bit representation of [crate::context::GcMarkBits] +type GcMarkBitsRepr = arbitrary_int::UInt; + +#[derive(Debug, Eq, PartialEq)] +#[bitenum(u1, exhaustive = true)] +pub enum GcMarkBits { + /// Indicates that tracing has not yet marked the object. + /// + /// Once tracing completes, this means the object is dead. + White = 0, + /// Indicates that tracing has marked the object. + /// + /// This means the object is live. + Black = 1, +} + +impl GcMarkBits { + #[inline] + pub fn to_raw(&self, state: &CollectorState) -> GcRawMarkBits { + let bits: GcMarkBitsRepr = self.raw_value(); + GcRawMarkBits::new_with_raw_value(if state.mark_bits_inverted.get() { + GcRawMarkBits::invert_bits(bits) + } else { + bits + }) + } +} + +#[bitenum(u1, exhaustive = true)] +pub enum GcRawMarkBits { + Red = 0, + Green = 1, +} +impl GcRawMarkBits { + #[inline] + pub fn resolve(&self, state: &CollectorState) -> GcMarkBits { + let bits: GcMarkBitsRepr = self.raw_value(); + GcMarkBits::new_with_raw_value(if state.mark_bits_inverted.get() { + Self::invert_bits(bits) + } else { + bits + }) + } + + #[inline] + fn invert_bits(bits: GcMarkBitsRepr) -> GcMarkBitsRepr { + ::MAX - bits + } +} + +/// A bitfield for the garbage collector's state. +/// +/// ## Default +/// The `DEFAULT` value isn't valid here. +/// However, it currently needs to exist fo +/// the macro to generate the `builder` field +#[bitfield(u32, default = 0)] +pub struct GcStateBits { + #[bit(0, rw)] + forwarded: bool, + #[bit(1, rw)] + generation: GenerationId, + #[bit(2, rw)] + array: bool, + #[bit(3, rw)] + raw_mark_bits: GcRawMarkBits, +} +pub union HeaderMetadata { + pub type_info: &'static GcTypeInfo, + pub array_type_info: &'static GcArrayTypeInfo, + pub forward_ptr: NonNull>, +} +pub union AllocInfo { + /// The [overall size][`GcTypeLayout::overall_layout`] of this object. + /// + /// This is used to iterate over objects in the young generation. + /// + /// Objects whose size cannot fit into a `u32` + /// can never be allocated in the young generation. + /// + /// If this object is an array, + /// this is the overall size of + /// the header and all elements. + pub this_object_overall_size: u32, + /// The index of the object within the vector of live objects. + /// + /// This is used in the old generation. + pub live_object_index: u32, +} + +#[repr(C, align(8))] +pub(crate) struct GcHeader { + pub(super) state_bits: Cell, + pub(super) alloc_info: AllocInfo, + pub(super) metadata: HeaderMetadata, + /// The id for the collector where this object is allocated. + /// + /// If the collector is a singleton (either global or thread-local), + /// this will be a zero sized type. + /// + /// ## Safety + /// The alignment of this type must be smaller than [`GcHeader::FIXED_ALIGNMENT`]. + pub collector_id: Id, +} +impl GcHeader { + #[inline] + pub(crate) unsafe fn update_state_bits(&self, func: impl FnOnce(&mut GcStateBits)) { + let mut bits = self.state_bits.get(); + func(&mut bits); + self.state_bits.set(bits); + } + + /// The fixed alignment for all GC types + /// + /// Allocating a type with an alignment greater than this is an error. + pub const FIXED_ALIGNMENT: usize = 8; + /// The fixed offset from the start of the GcHeader to a regular value + pub const REGULAR_VALUE_OFFSET: usize = std::mem::size_of::(); + pub const ARRAY_VALUE_OFFSET: usize = std::mem::size_of::>(); + pub const REGULAR_HEADER_LAYOUT: Layout = Layout::new::(); + pub const ARRAY_HEADER_LAYOUT: Layout = Layout::new::>(); + + #[inline] + pub fn id(&self) -> Id { + self.collector_id + } + + #[inline] + fn resolve_type_info(&self) -> &'static GcTypeInfo { + unsafe { + if self.state_bits.get().forwarded() { + let forward_ptr = self.metadata.forward_ptr; + let forward_header = forward_ptr.as_ref(); + debug_assert!(!forward_header.state_bits.get().forwarded()); + forward_header.metadata.type_info + } else { + self.metadata.type_info + } + } + } + + #[inline] + pub fn regular_value_ptr(&self) -> NonNull { + unsafe { + NonNull::new_unchecked( + (self as *const Self as *mut Self as *mut u8).add(Self::REGULAR_VALUE_OFFSET), + ) + } + } + + #[inline] + pub unsafe fn assume_array_header(&self) -> &'_ GcArrayHeader { + &*(self as *const Self as *const GcArrayHeader) + } +} + +#[repr(C, align(8))] +pub struct GcArrayHeader { + pub(super) main_header: GcHeader, + /// The length of the array in elements + pub(super) len_elements: usize, +} + +impl GcArrayHeader { + #[inline] + fn resolve_type_info(&self) -> &'static GcArrayTypeInfo { + unsafe { + &*(self.main_header.resolve_type_info() as *const GcTypeInfo + as *const GcArrayTypeInfo) + } + } + + #[inline] + pub fn array_value_ptr(&self) -> NonNull { + unsafe { + NonNull::new_unchecked( + (self as *const Self as *mut Self as *mut u8) + .add(GcHeader::::ARRAY_VALUE_OFFSET), + ) + } + } + + #[inline] + pub fn layout_info(&self) -> GcArrayLayoutInfo { + GcArrayLayoutInfo { + element_layout: self.element_layout(), + len_elements: self.len_elements, + marker: PhantomData, + } + } + + #[inline] + fn element_layout(&self) -> Layout { + self.resolve_type_info() + .element_type_info + .layout + .value_layout + } + + #[inline] + fn value_layout(&self) -> Layout { + self.layout_info().value_layout() + } + + #[inline] + fn overall_layout(&self) -> Layout { + self.layout_info().overall_layout() + } +} +pub struct GcArrayLayoutInfo { + element_layout: Layout, + len_elements: usize, + marker: PhantomData<&'static Id>, +} +impl GcArrayLayoutInfo { + #[inline] + pub unsafe fn new_unchecked(element_layout: Layout, len_elements: usize) -> Self { + #[cfg(debug_assertions)] + { + match Self::new(element_layout, len_elements) { + Ok(_success) => {} + Err(GcArrayLayoutError::ArraySizeOverflow) => { + panic!("Invalid array layout: size overflow") + } + Err(_) => panic!("invalid array layout: other issue"), + } + } + GcArrayLayoutInfo { + element_layout, + len_elements, + marker: PhantomData, + } + } + + // See Layout::max_size_for_align + const MAX_VALUE_SIZE: usize = ((isize::MAX as usize) - GcHeader::::FIXED_ALIGNMENT - 1) + - GcHeader::::ARRAY_VALUE_OFFSET; + + #[cfg_attr(not(debug_assertions), inline)] + pub const fn new( + element_layout: Layout, + len_elements: usize, + ) -> Result { + if element_layout.align() > GcHeader::::FIXED_ALIGNMENT { + return Err(GcArrayLayoutError::InvalidElementAlign); + } + if LayoutExt(element_layout).pad_to_align().size() != element_layout.size() { + return Err(GcArrayLayoutError::ElementMissingPadding); + } + let Some(repeated_value_size) = element_layout.size().checked_mul(len_elements) else { + return Err(GcArrayLayoutError::ArraySizeOverflow); + }; + if repeated_value_size >= Self::MAX_VALUE_SIZE { + return Err(GcArrayLayoutError::ArraySizeOverflow); + } + if cfg!(debug_assertions) { + // double check above calculations + match Layout::from_size_align(repeated_value_size, GcHeader::::FIXED_ALIGNMENT) { + Ok(repeated_value) => { + match LayoutExt(GcHeader::::ARRAY_HEADER_LAYOUT).extend(repeated_value) { + Ok((overall_layout, actual_offset)) => { + debug_assert!(actual_offset == GcHeader::::ARRAY_VALUE_OFFSET); + debug_assert!( + overall_layout.size() + == match repeated_value_size + .checked_add(GcHeader::::ARRAY_VALUE_OFFSET) + { + Some(size) => size, + None => panic!("checked add overflow"), + } + ); + } + Err(e) => panic!("Overall value overflows layout"), + } + } + Err(_) => panic!("Repeated value overflows layout!"), + } + } + return Ok(GcArrayLayoutInfo { + element_layout, + len_elements, + marker: PhantomData, + }); + } + + #[inline] + pub const fn len_elements(&self) -> usize { + self.len_elements + } + + #[inline] + pub const fn element_layout(&self) -> Layout { + self.element_layout + } + + #[inline] + pub fn value_layout(&self) -> Layout { + let element_layout = self.element_layout(); + unsafe { + Layout::from_size_align_unchecked( + element_layout.size().unchecked_mul(self.len_elements), + element_layout.align(), + ) + } + } + + #[inline] + pub fn overall_layout(&self) -> Layout { + let value_layout = self.value_layout(); + unsafe { + Layout::from_size_align_unchecked( + value_layout + .size() + .unchecked_add(GcHeader::::ARRAY_VALUE_OFFSET), + GcHeader::::FIXED_ALIGNMENT, + ) + .pad_to_align() + } + } +} +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum GcArrayLayoutError { + #[error("Invalid element alignment")] + InvalidElementAlign, + #[error("Element layout missing trailing padding")] + ElementMissingPadding, + #[error("Size overflow for array layout")] + ArraySizeOverflow, +} diff --git a/src/context/old.rs b/src/context/old.rs index 69b6083..13c2610 100644 --- a/src/context/old.rs +++ b/src/context/old.rs @@ -1,15 +1,10 @@ use allocator_api2::alloc::Allocator; -use std::alloc::Layout; use std::cell::{Cell, UnsafeCell}; use std::ptr::NonNull; use zerogc_next_mimalloc_semisafe::heap::MimallocHeap; -use crate::context::young::YoungGenerationSpace; -use crate::context::{ - AllocInfo, CollectStage, CollectStageTracker, GcArrayTypeInfo, GcHeader, GcMarkBits, - GcStateBits, GcTypeInfo, GenerationId, HeaderMetadata, -}; -use crate::gcptr::Gc; +use crate::context::layout::{AllocInfo, GcHeader, GcMarkBits}; +use crate::context::{CollectStage, CollectStageTracker, CollectorState, GenerationId}; use crate::CollectorId; pub struct OldGenerationSpace { @@ -19,7 +14,7 @@ pub struct OldGenerationSpace { stage: CollectStageTracker, } impl OldGenerationSpace { - pub unsafe fn sweep(&mut self) { + pub unsafe fn sweep(&mut self, state: &CollectorState) { self.stage .begin_stage(Some(CollectStage::Mark), CollectStage::Sweep); let mut next_index: u32 = 0; @@ -27,7 +22,7 @@ impl OldGenerationSpace { let header = &mut *func.as_ptr(); debug_assert_eq!(header.collector_id, self.collector_id); debug_assert_eq!(header.state_bits.get().generation(), GenerationId::Old); - let mark_bits = header.state_bits.get().raw_mark_bits().resolve(self); + let mark_bits = header.state_bits.get().raw_mark_bits().resolve(state); match mark_bits { GcMarkBits::White => { // unmarked @@ -69,21 +64,14 @@ impl OldGenerationSpace { { let live_objects = &mut *self.live_objects.get(); live_object_index = u32::try_from(live_objects.len()).unwrap(); - live_objects.push(header_ptr); + live_objects.push(header_ptr.cast::>()); } target.init_header( header_ptr, GcHeader { - state_bits: Cell::new( - GcStateBits::builder() - .with_forwarded(false) - .with_generation(GenerationId::Old) - .with_array(T::ARRAY) - .with_raw_mark_bits(GcMarkBits::White.to_raw(self)) - .build(), - ), + state_bits: Cell::new(target.init_state_bits(GenerationId::Old)), alloc_info: AllocInfo { live_object_index }, - metadata: HeaderMetadata { type_info }, + metadata: target.header_metadata(), collector_id: self.collector_id, }, ); @@ -92,21 +80,7 @@ impl OldGenerationSpace { } #[derive(Debug, thiserror::Error)] -enum OldAllocError { +pub enum OldAllocError { #[error("Out of memory (oldgen)")] OutOfMemory, } - -unsafe impl super::Generation for OldGenerationSpace { - const ID: GenerationId = GenerationId::Old; - - #[inline] - fn cast_young(&self) -> Option<&'_ YoungGenerationSpace> { - None - } - - #[inline] - fn cast_old(&self) -> Option<&'_ OldGenerationSpace> { - Some(self) - } -} diff --git a/src/context/young.rs b/src/context/young.rs index d4029b9..437293b 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -1,15 +1,11 @@ -use std::alloc::Layout; use std::cell::Cell; use std::marker::PhantomData; use std::ptr::NonNull; use bumpalo::ChunkRawIter; -use crate::context::old::OldGenerationSpace; -use crate::context::{ - AllocInfo, CollectStage, CollectStageTracker, GcHeader, GcStateBits, GenerationId, - HeaderMetadata, -}; +use crate::context::layout::{AllocInfo, GcHeader}; +use crate::context::{CollectStage, CollectStageTracker, GenerationId}; use crate::utils::bumpalo_raw::{BumpAllocRaw, BumpAllocRawConfig}; use crate::utils::Alignment; use crate::CollectorId; @@ -54,17 +50,11 @@ impl YoungGenerationSpace { target.init_header( header_ptr, GcHeader { - state_bits: Cell::new( - GcStateBits::builder() - .with_forwarded(false) - .with_generation(GenerationId::Young) - .with_array(T::ARRAY) - .build(), - ), + state_bits: Cell::new(target.init_state_bits(GenerationId::Young)), alloc_info: AllocInfo { this_object_overall_size: overall_layout.size() as u32, }, - metadata: HeaderMetadata { type_info }, + metadata: target.header_metadata(), collector_id: self.collector_id, }, ); diff --git a/src/gcptr.rs b/src/gcptr.rs index 2e497e4..39a5094 100644 --- a/src/gcptr.rs +++ b/src/gcptr.rs @@ -1,9 +1,8 @@ -use log::debug; use std::marker::PhantomData; use std::ops::Deref; use std::ptr::NonNull; -use crate::context::{GcHeader, GcTypeInfo}; +use crate::context::layout::{GcHeader, GcTypeInfo}; use crate::{Collect, CollectContext, CollectorId, GarbageCollector}; pub struct Gc<'gc, T, Id: CollectorId> { diff --git a/src/utils/layout_helpers.rs b/src/utils/layout_helpers.rs index f1d5fc0..0f60dd2 100644 --- a/src/utils/layout_helpers.rs +++ b/src/utils/layout_helpers.rs @@ -32,7 +32,7 @@ impl Alignment { } #[inline] - pub fn value(&self) -> usize { + pub const fn value(&self) -> usize { self.0.get() } } @@ -117,7 +117,7 @@ impl LayoutExt { #[derive(Debug, thiserror::Error)] #[error("Layout error")] #[non_exhaustive] -struct LayoutExtError; +pub struct LayoutExtError; #[inline] const fn const_max(first: usize, second: usize) -> usize { From 5efc76a60bcee19ec3c4310b823976649d395fdb Mon Sep 17 00:00:00 2001 From: Techcable Date: Sat, 4 May 2024 19:23:02 -0700 Subject: [PATCH 10/35] Get basic example to compile Some strange errors with invalid Layout alignment (when using miri) --- Cargo.lock | 295 ++++++++++++++++++++++++++++++ Cargo.toml | 9 + build.rs | 5 + examples/binary_trees.rs | 128 +++++++++++++ src/context.rs | 345 ++++++++++++++++++++++++++++++------ src/context/layout.rs | 36 +++- src/context/old.rs | 128 +++++++++++-- src/context/young.rs | 26 ++- src/lib.rs | 2 + src/utils.rs | 93 ++++++++++ src/utils/bumpalo_raw.rs | 12 ++ src/utils/layout_helpers.rs | 2 +- 12 files changed, 999 insertions(+), 82 deletions(-) create mode 100644 build.rs create mode 100644 examples/binary_trees.rs diff --git a/Cargo.lock b/Cargo.lock index e5a475d..03b820e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,12 @@ dependencies = [ "syn", ] +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + [[package]] name = "bumpalo" version = "3.16.0" @@ -50,12 +56,53 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "equivalent" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -68,6 +115,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "indexmap" version = "2.2.6" @@ -78,6 +131,23 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + [[package]] name = "libc" version = "0.2.154" @@ -95,12 +165,40 @@ dependencies = [ "libc", ] +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags", + "libc", +] + [[package]] name = "log" version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro-kwargs" version = "0.2.0" @@ -153,6 +251,68 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "redox_users" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + +[[package]] +name = "rustversion" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092474d1a01ea8278f69e6a358998405fae5b8b963ddaeb2b0b04a128bf1dfb0" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.200" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.200" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-term" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" +dependencies = [ + "is-terminal", + "slog", + "term", + "thread_local", + "time", +] + [[package]] name = "stacker" version = "0.1.15" @@ -177,6 +337,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.60" @@ -197,12 +368,59 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "winapi" version = "0.3.9" @@ -225,6 +443,79 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + [[package]] name = "zerog-next-macros" version = "0.1.0-alpha.1" @@ -245,6 +536,10 @@ dependencies = [ "bitbybit", "bumpalo", "log", + "rustversion", + "scopeguard", + "slog", + "slog-term", "stacker", "thiserror", "zerogc-next-mimalloc-semisafe", diff --git a/Cargo.toml b/Cargo.toml index 509daa6..e18ca38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,11 +10,20 @@ allocator-api2 = "0.2.18" bitbybit = "1.3.2" arbitrary-int = "1.2.7" thiserror = "1" +rustversion = "1" # Easier to debug recusion than an explicit queue stacker = "0.1" # Internal bindings to mimalloc zerogc-next-mimalloc-semisafe = { version = "0.1.0-alpha.1", path = "libs/mimalloc-semisafe" } log = "0.4.21" +scopeguard = "1.2" + +[build-dependencies] +rustversion = "1" + +[dev-dependencies] +slog = "2.7.0" +slog-term = "2.9.1" [workspace] resolver = "2" diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..9cd6111 --- /dev/null +++ b/build.rs @@ -0,0 +1,5 @@ +pub fn main() { + if rustversion::cfg!(nightly) { + println!("cargo:rustc-cfg=zerogc_next_nightly") + } +} diff --git a/examples/binary_trees.rs b/examples/binary_trees.rs new file mode 100644 index 0000000..327bd23 --- /dev/null +++ b/examples/binary_trees.rs @@ -0,0 +1,128 @@ +use slog::{o, Drain, Logger}; +use std::cell::Cell; +use std::ptr::NonNull; +use zerogc_next::context::SingletonStatus; +use zerogc_next::{Collect, CollectContext, CollectorId, GarbageCollector, Gc, NullCollect}; + +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +struct ThisCollectorId; + +unsafe impl CollectorId for ThisCollectorId { + const SINGLETON: Option = Some(SingletonStatus::Global); + + #[inline] + unsafe fn summon_singleton() -> Option { + Some(ThisCollectorId) + } +} + +struct Tree<'gc> { + children: Cell< + Option<( + Gc<'gc, Tree<'gc>, ThisCollectorId>, + Gc<'gc, Tree<'gc>, ThisCollectorId>, + )>, + >, +} + +unsafe impl<'gc> Collect for Tree<'gc> { + type Collected<'newgc> = Tree<'newgc>; + const NEEDS_COLLECT: bool = true; + + unsafe fn collect_inplace( + target: NonNull, + context: &mut CollectContext<'_, ThisCollectorId>, + ) { + let mut children = target.as_ref().children.get(); + match &mut children { + Some((left, right)) => { + Gc::collect_inplace(NonNull::from(left), context); + Gc::collect_inplace(NonNull::from(right), context); + } + None => {} // no need to traxe + } + target.as_ref().children.set(children); + } +} + +fn item_check(tree: &Tree) -> i32 { + if let Some((left, right)) = tree.children.get() { + 1 + item_check(&right) + item_check(&left) + } else { + 1 + } +} + +fn bottom_up_tree<'gc>( + collector: &'gc GarbageCollector, + depth: i32, +) -> Gc<'gc, Tree<'gc>, ThisCollectorId> { + let tree = collector.alloc(Tree { + children: Cell::new(None), + }); + if depth > 0 { + let right = bottom_up_tree(collector, depth - 1); + let left = bottom_up_tree(collector, depth - 1); + tree.children.set(Some((left, right))); + } + tree +} + +fn inner(gc: &mut GarbageCollector, depth: i32, iterations: u32) -> String { + let chk: i32 = (0..iterations) + .into_iter() + .map(|_| { + let a = bottom_up_tree(&gc, depth); + let res = item_check(&a); + gc.collect(); + res + }) + .sum(); + format!("{}\t trees of depth {}\t check: {}", iterations, depth, chk) +} + +fn main() { + let n = std::env::args() + .nth(1) + .and_then(|n| n.parse().ok()) + .unwrap_or(10); + let min_depth = 4; + let max_depth = if min_depth + 2 > n { min_depth + 2 } else { n }; + + let plain = slog_term::PlainSyncDecorator::new(std::io::stdout()); + let logger = Logger::root( + slog_term::FullFormat::new(plain).build().fuse(), + o!("bench" => file!()), + ); + let mut collector = unsafe { GarbageCollector::with_id(ThisCollectorId) }; + { + let depth = max_depth + 1; + let tree = bottom_up_tree(&collector, depth); + println!( + "stretch tree of depth {}\t check: {}", + depth, + item_check(&tree) + ); + } + collector.collect(); + + let long_lived_tree = collector.root(bottom_up_tree(&collector, max_depth)); + + { + (min_depth / 2..max_depth / 2 + 1) + .into_iter() + .for_each(|half_depth| { + let depth = half_depth * 2; + let iterations = 1 << ((max_depth - depth + min_depth) as u32); + let message = inner(&mut collector, depth, iterations); + collector.collect(); + println!("{}", message); + }) + } + + println!( + "long lived tree of depth {}\t check: {}", + max_depth, + item_check(&*long_lived_tree.resolve(&collector)) + ); +} diff --git a/src/context.rs b/src/context.rs index b01dfaf..07f2c17 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,8 +1,10 @@ use std::alloc::Layout; -use std::cell::Cell; +use std::cell::{Cell, RefCell}; +use std::error::Error; use std::fmt::Debug; use std::marker::PhantomData; use std::ptr::NonNull; +use std::rc::{Rc, Weak}; use bitbybit::bitenum; @@ -11,8 +13,9 @@ use crate::context::layout::{ GcTypeInfo, HeaderMetadata, TraceFuncPtr, }; use crate::context::old::OldGenerationSpace; -use crate::context::young::YoungGenerationSpace; +use crate::context::young::{YoungAllocError, YoungGenerationSpace}; use crate::gcptr::Gc; +use crate::utils::AbortFailureGuard; use crate::Collect; pub(crate) mod layout; @@ -45,7 +48,9 @@ pub enum SingletonStatus { /// This type must be `#[repr(C)`] and its alignment must be at most eight bytes. pub unsafe trait CollectorId: Copy + Debug + Eq + 'static { const SINGLETON: Option; - unsafe fn resolve_collector(&self) -> *mut GarbageCollector; + + // TODO :This method is unsafe because of mutable aliasing + // unsafe fn resolve_collector(&self) -> *mut GarbageCollector; unsafe fn summon_singleton() -> Option; } @@ -92,15 +97,54 @@ pub(crate) struct CollectorState { mark_bits_inverted: Cell, } +struct GcRootBox { + header: Cell>>, +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +struct GenerationSizes { + young_generation_size: usize, + old_generation_size: usize, +} +impl GenerationSizes { + const INITIAL_COLLECT_THRESHOLD: Self = GenerationSizes { + young_generation_size: 12 * 1024, + old_generation_size: 12 * 1204, + }; + + #[inline] + pub fn meets_either_threshold(&self, threshold: GenerationSizes) -> bool { + self.young_generation_size >= threshold.young_generation_size + || self.old_generation_size >= threshold.old_generation_size + } +} + pub struct GarbageCollector { state: CollectorState, young_generation: YoungGenerationSpace, old_generation: OldGenerationSpace, + roots: RefCell>>>, + last_collect_size: Option, + collector_id: Id, } impl GarbageCollector { + pub unsafe fn with_id(id: Id) -> Self { + GarbageCollector { + state: CollectorState { + collector_id: id, + mark_bits_inverted: Cell::new(false), + }, + young_generation: YoungGenerationSpace::new(id), + old_generation: OldGenerationSpace::new(id), + roots: RefCell::new(Vec::new()), + last_collect_size: None, + collector_id: id, + } + } + #[inline] pub fn id(&self) -> Id { - self.state.collector_id + self.collector_id } #[inline(always)] @@ -108,9 +152,156 @@ impl GarbageCollector { self.alloc_with(|| value) } + /// Allocate a GC object, initializng it with the specified closure. #[inline(always)] + #[track_caller] pub fn alloc_with>(&self, func: impl FnOnce() -> T) -> Gc<'_, T, Id> { - todo!() + unsafe { + let header = self.alloc_raw(&RegularAlloc { + state: &self.state, + type_info: GcTypeInfo::new::(), + }); + let initialization_guard = DestroyUninitValueGuard { + header, + old_generation: &self.old_generation, + }; + let value_ptr = header.as_ref().regular_value_ptr().cast::(); + value_ptr.as_ptr().write(func()); + header + .as_ref() + .update_state_bits(|state| state.with_value_initialized(true)); + initialization_guard.defuse(); // successful initialization; + Gc::from_raw_ptr(value_ptr) + } + } + + #[inline] + unsafe fn alloc_raw>(&self, target: &T) -> NonNull { + match self.young_generation.alloc_raw(target) { + Ok(res) => res, + Err(YoungAllocError::SizeExceedsLimit) => self.alloc_raw_fallback(target), + Err(error @ YoungAllocError::OutOfMemory) => Self::oom(error), + } + } + + #[cold] + unsafe fn alloc_raw_fallback>(&self, target: &T) -> NonNull { + self.old_generation + .alloc_raw(target) + .unwrap_or_else(|err| Self::oom(err)) + } + + #[cold] + #[inline(never)] + fn oom(error: E) -> ! { + panic!("Fatal allocation error: {error}") + } + + #[inline] + pub fn root<'gc, T: Collect>( + &'gc self, + val: Gc<'gc, T, Id>, + ) -> GcHandle, Id> { + let mut roots = self.roots.borrow_mut(); + let root = Rc::new(GcRootBox { + header: Cell::new(NonNull::from(val.header())), + }); + roots.push(Rc::downgrade(&root)); + drop(roots); // drop refcell guard + GcHandle { + ptr: root, + id: self.id(), + marker: PhantomData, + } + } + + #[inline] + pub fn collect(&mut self) { + if self.needs_collection() { + self.force_collect(); + } + } + + #[cold] + pub fn force_collect(&mut self) { + // mark roots + let mut context = CollectContext { + garbage_collector: self, + id: self.collector_id, + }; + let failure_guard = AbortFailureGuard::new("GC failure to trace is fatal"); + let mut roots = self.roots.borrow_mut(); + roots.retain(|root| { + match root.upgrade() { + Some(root) => { + root.header + .set(unsafe { context.collect_gcheader(root.header.get()) }); + true // keep live root + } + None => false, // delete dead root + } + }); + drop(roots); // release guard + // tracing failure is fatal, but sweeping fatal is fine + failure_guard.defuse(); + // now sweep + unsafe { + self.young_generation.sweep(); + self.old_generation.sweep(&self.state); + } + // invert meaning of the mark bits + self.state + .mark_bits_inverted + .set(!self.state.mark_bits_inverted.get()); + // count size to trigger next gc + self.last_collect_size = Some(self.current_size()); + } + + #[inline] + fn current_size(&self) -> GenerationSizes { + GenerationSizes { + old_generation_size: self.old_generation.allocated_bytes(), + young_generation_size: self.young_generation.allocated_bytes(), + } + } + + #[inline] + fn threshold_size(&self) -> GenerationSizes { + match self.last_collect_size { + None => GenerationSizes::INITIAL_COLLECT_THRESHOLD, + Some(last_sizes) => GenerationSizes { + young_generation_size: last_sizes.young_generation_size * 2, + old_generation_size: last_sizes.old_generation_size * 2, + }, + } + } + + #[inline] + fn needs_collection(&self) -> bool { + self.current_size() + .meets_either_threshold(self.threshold_size()) + } +} + +pub struct GcHandle, Id: CollectorId> { + ptr: Rc>, + id: Id, + marker: PhantomData, +} +impl, Id: CollectorId> GcHandle { + /// Resolve this handle into a [`Gc`] smart-pointer. + /// + /// ## Safety + /// Even if this handle is dropped, the value will live until the next collection. + /// This makes it valid for `'gc`. + #[inline] + pub fn resolve<'gc>( + &self, + collector: &'gc GarbageCollector, + ) -> Gc<'gc, T::Collected<'gc>, Id> { + assert_eq!(self.id, collector.id()); + // reload from GcRootBox in case pointer moved + unsafe { Gc::from_raw_ptr(self.ptr.header.get().as_ref().regular_value_ptr().cast()) } } } @@ -127,6 +318,7 @@ unsafe trait RawAllocTarget { .with_generation(gen) .with_array(Self::ARRAY) .with_raw_mark_bits(GcMarkBits::White.to_raw(self.collector_state())) + .with_value_initialized(false) .build() } @@ -215,7 +407,7 @@ enum GenerationId { pub struct CollectContext<'newgc, Id: CollectorId> { id: Id, - garbage_collector: &'newgc mut GarbageCollector, + garbage_collector: &'newgc GarbageCollector, } impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { #[inline] @@ -231,60 +423,66 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { .write(self.collect_gc_ptr(target.read())); } - #[cfg_attr(not(debug_assertions), inline)] + #[inline] unsafe fn collect_gc_ptr<'gc, T: Collect>( &mut self, target: Gc<'gc, T, Id>, ) -> Gc<'newgc, T::Collected<'newgc>, Id> { - let header = target.header(); - assert_eq!(header.collector_id, self.id, "Mismatched collector ids"); - debug_assert!( - !header.state_bits.get().array(), - "Incorrectly marked as an array" - ); - if header.state_bits.get().forwarded() { - debug_assert_eq!(header.state_bits.get().generation(), GenerationId::Young); - debug_assert_eq!( - header - .state_bits - .get() - .raw_mark_bits() - .resolve(&self.garbage_collector.state), - GcMarkBits::Black - ); - return Gc::from_raw_ptr( - header - .metadata - .forward_ptr - .as_ref() - .regular_value_ptr() - .cast(), - ); - } - match header - .state_bits - .get() - .raw_mark_bits() - .resolve(&self.garbage_collector.state) + Gc::from_raw_ptr( + self.collect_gcheader(NonNull::from(target.header())) + .as_ref() + .regular_value_ptr() + .cast(), + ) + } + + #[cfg_attr(not(debug_assertions), inline)] + #[must_use] + unsafe fn collect_gcheader(&mut self, header: NonNull>) -> NonNull> { + let mark_bits: GcMarkBits; { - GcMarkBits::White => { - let new_header = self.fallback_collect_gc_header(NonNull::from(header)); - Gc::from_raw_ptr(new_header.as_ref().regular_value_ptr().cast()) - } - GcMarkBits::Black => { - // already traced, can skip it - Gc::from_raw_ptr(target.as_raw_ptr().cast()) + let header = header.as_ref(); + assert_eq!(header.collector_id, self.id, "Mismatched collector ids"); + debug_assert!( + !header.state_bits.get().array(), + "Incorrectly marked as an array" + ); + if header.state_bits.get().forwarded() { + debug_assert_eq!(header.state_bits.get().generation(), GenerationId::Young); + debug_assert_eq!( + header + .state_bits + .get() + .raw_mark_bits() + .resolve(&self.garbage_collector.state), + GcMarkBits::Black + ); + return header.metadata.forward_ptr; } + mark_bits = header + .state_bits + .get() + .raw_mark_bits() + .resolve(&self.garbage_collector.state); + } + match mark_bits { + GcMarkBits::White => self.fallback_collect_gc_header(header), + GcMarkBits::Black => header, } } #[cold] - unsafe fn fallback_collect_gc_header<'gc>( + unsafe fn fallback_collect_gc_header( &mut self, header_ptr: NonNull>, ) -> NonNull> { let type_info: &'static GcTypeInfo; let array = header_ptr.as_ref().state_bits.get().array(); + debug_assert!( + header_ptr.as_ref().state_bits.get().value_initialized(), + "Traced value must be initialized: {:?}", + header_ptr.as_ref() + ); let prev_generation: GenerationId; { let header = header_ptr.as_ref(); @@ -299,7 +497,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { // mark as black header.update_state_bits(|state_bits| { state_bits - .with_raw_mark_bits(GcMarkBits::Black.to_raw(&self.garbage_collector.state)); + .with_raw_mark_bits(GcMarkBits::Black.to_raw(&self.garbage_collector.state)) }); prev_generation = header.state_bits.get().generation(); type_info = header.metadata.type_info; @@ -321,7 +519,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { array_value_size = Some(array_layout.value_layout().size()); self.garbage_collector .old_generation - .alloc_raw(ArrayAlloc { + .alloc_raw(&ArrayAlloc { layout_info: array_layout, type_info: array_type_info, state: &self.garbage_collector.state, @@ -331,7 +529,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { array_value_size = None; self.garbage_collector .old_generation - .alloc_raw(RegularAlloc { + .alloc_raw(&RegularAlloc { type_info, state: &self.garbage_collector.state, }) @@ -346,11 +544,12 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { .set(header_ptr.as_ref().state_bits.get()); copied_ptr.as_ref().update_state_bits(|bits| { debug_assert!(!bits.forwarded()); - bits.with_generation(GenerationId::Old); - }); - header_ptr.as_ref().update_state_bits(|bits| { - bits.with_forwarded(true); + bits.with_generation(GenerationId::Old) + .with_value_initialized(true) }); + header_ptr + .as_ref() + .update_state_bits(|bits| bits.with_forwarded(true)); (&mut *header_ptr.as_ptr()).metadata.forward_ptr = copied_ptr.cast(); // NOTE: Copy uninitialized bytes is safe here, as long as they are not read in dest if array { @@ -422,6 +621,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { trace_func(header.as_ref().regular_value_ptr().cast(), self); } } + unsafe fn trace_children_array( &mut self, header: NonNull>, @@ -441,3 +641,44 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { } } } + +/// A RAII guard to destroy an uninitialized GC allocation. +/// +/// Must explicitly call `defuse` after a successful initialization. +#[must_use] +struct DestroyUninitValueGuard<'a, Id: CollectorId> { + header: NonNull>, + old_generation: &'a OldGenerationSpace, +} +impl<'a, Id: CollectorId> DestroyUninitValueGuard<'a, Id> { + #[inline] + pub fn defuse(self) { + debug_assert!( + unsafe { self.header.as_ref().state_bits.get().value_initialized() }, + "Value not initialized" + ); + std::mem::forget(self); + } +} +impl Drop for DestroyUninitValueGuard<'_, Id> { + #[cold] + fn drop(&mut self) { + // should only be called on failure + unsafe { + assert!( + !self.header.as_ref().state_bits.get().value_initialized(), + "Value successfully initialized but guard not defused" + ); + match self.header.as_ref().state_bits.get().generation() { + GenerationId::Old => { + // old-gen needs an explicit free + self.old_generation.destroy_uninit_object(self.header); + } + GenerationId::Young => { + // In young-gen, marking uninitialized is sufficient + // it will be automatically freed next sweep + } + } + } + } +} diff --git a/src/context/layout.rs b/src/context/layout.rs index ac97d56..96e30b0 100644 --- a/src/context/layout.rs +++ b/src/context/layout.rs @@ -4,10 +4,12 @@ use crate::{Collect, CollectContext, CollectorId}; use bitbybit::{bitenum, bitfield}; use std::alloc::Layout; use std::cell::Cell; +use std::fmt::{Debug, Formatter}; use std::marker::PhantomData; use std::ptr::NonNull; /// The layout of a "regular" (non-array) type +#[derive(Debug)] pub(crate) struct GcTypeLayout { /// The layout of the underlying value /// @@ -104,6 +106,7 @@ impl GcArrayTypeInfo { pub type TraceFuncPtr = unsafe fn(NonNull<()>, &mut CollectContext); #[repr(C)] +#[derive(Debug)] pub(crate) struct GcTypeInfo { pub(super) layout: GcTypeLayout, pub(super) drop_func: Option, @@ -178,7 +181,7 @@ pub enum GcMarkBits { impl GcMarkBits { #[inline] - pub fn to_raw(&self, state: &CollectorState) -> GcRawMarkBits { + pub fn to_raw(self, state: &CollectorState) -> GcRawMarkBits { let bits: GcMarkBitsRepr = self.raw_value(); GcRawMarkBits::new_with_raw_value(if state.mark_bits_inverted.get() { GcRawMarkBits::invert_bits(bits) @@ -217,6 +220,7 @@ impl GcRawMarkBits { /// However, it currently needs to exist fo /// the macro to generate the `builder` field #[bitfield(u32, default = 0)] +#[derive(Debug)] pub struct GcStateBits { #[bit(0, rw)] forwarded: bool, @@ -226,6 +230,8 @@ pub struct GcStateBits { array: bool, #[bit(3, rw)] raw_mark_bits: GcRawMarkBits, + #[bit(4, rw)] + value_initialized: bool, } pub union HeaderMetadata { pub type_info: &'static GcTypeInfo, @@ -264,12 +270,26 @@ pub(crate) struct GcHeader { /// The alignment of this type must be smaller than [`GcHeader::FIXED_ALIGNMENT`]. pub collector_id: Id, } +impl Debug for GcHeader { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GcHeader") + .field("state_bits", &self.state_bits.get()) + .field( + "metadata", + if self.state_bits.get().forwarded() { + unsafe { &self.metadata.forward_ptr } + } else { + unsafe { &self.metadata.type_info } + }, + ) + .field("collector_id", &self.collector_id) + .finish_non_exhaustive() + } +} impl GcHeader { #[inline] - pub(crate) unsafe fn update_state_bits(&self, func: impl FnOnce(&mut GcStateBits)) { - let mut bits = self.state_bits.get(); - func(&mut bits); - self.state_bits.set(bits); + pub(crate) unsafe fn update_state_bits(&self, func: impl FnOnce(GcStateBits) -> GcStateBits) { + self.state_bits.set(func(self.state_bits.get())); } /// The fixed alignment for all GC types @@ -432,17 +452,17 @@ impl GcArrayLayoutInfo { } ); } - Err(e) => panic!("Overall value overflows layout"), + Err(_e) => panic!("Overall value overflows layout"), } } Err(_) => panic!("Repeated value overflows layout!"), } } - return Ok(GcArrayLayoutInfo { + Ok(GcArrayLayoutInfo { element_layout, len_elements, marker: PhantomData, - }); + }) } #[inline] diff --git a/src/context/old.rs b/src/context/old.rs index 13c2610..1d5151e 100644 --- a/src/context/old.rs +++ b/src/context/old.rs @@ -1,25 +1,75 @@ -use allocator_api2::alloc::Allocator; +use allocator_api2::alloc::{AllocError, Allocator}; +use std::alloc::Layout; use std::cell::{Cell, UnsafeCell}; use std::ptr::NonNull; use zerogc_next_mimalloc_semisafe::heap::MimallocHeap; use crate::context::layout::{AllocInfo, GcHeader, GcMarkBits}; -use crate::context::{CollectStage, CollectStageTracker, CollectorState, GenerationId}; +use crate::context::{CollectorState, GenerationId}; use crate::CollectorId; +mod miri { + use allocator_api2::alloc::AllocError; + use std::alloc::Layout; + use std::ptr::NonNull; + + pub struct HeapAllocFallback; + impl HeapAllocFallback { + pub fn new() -> Self { + HeapAllocFallback + } + } + + unsafe impl allocator_api2::alloc::Allocator for HeapAllocFallback { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + unsafe { + let ptr = allocator_api2::alloc::alloc(layout); + if ptr.is_null() { + Err(AllocError) + } else { + Ok(NonNull::slice_from_raw_parts( + NonNull::new_unchecked(ptr), + layout.size(), + )) + } + } + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + allocator_api2::alloc::dealloc(ptr.as_ptr(), layout) + } + } +} + +#[cfg(miri)] +type HeapAllocator = miri::HeapAllocFallback; +#[cfg(not(miri))] +type HeapAllocator = zerogc_next_mimalloc_semisafe::heap::MimallocHeap; + pub struct OldGenerationSpace { - heap: MimallocHeap, - live_objects: UnsafeCell>>>, + // TODO: Add allocation count wrapper? + heap: HeapAllocator, + live_objects: UnsafeCell>>>>, collector_id: Id, - stage: CollectStageTracker, + allocated_bytes: Cell, } impl OldGenerationSpace { + pub unsafe fn new(id: Id) -> Self { + OldGenerationSpace { + heap: HeapAllocator::new(), + live_objects: UnsafeCell::new(Vec::new()), + collector_id: id, + allocated_bytes: Cell::new(0), + } + } + pub unsafe fn sweep(&mut self, state: &CollectorState) { - self.stage - .begin_stage(Some(CollectStage::Mark), CollectStage::Sweep); let mut next_index: u32 = 0; self.live_objects.get_mut().retain(|func| { - let header = &mut *func.as_ptr(); + if func.is_none() { + return false; // skip null objects, deallocated early + } + let header = &mut *func.unwrap().as_ptr(); debug_assert_eq!(header.collector_id, self.collector_id); debug_assert_eq!(header.state_bits.get().generation(), GenerationId::Old); let mark_bits = header.state_bits.get().raw_mark_bits().resolve(state); @@ -29,6 +79,19 @@ impl OldGenerationSpace { if cfg!(debug_assertions) { header.alloc_info.live_object_index = u32::MAX; } + let overall_layout = if header.state_bits.get().array() { + header.assume_array_header().layout_info().overall_layout() + } else { + header.metadata.type_info.layout.overall_layout() + }; + self.allocated_bytes.set( + self.allocated_bytes + .get() + .checked_sub(overall_layout.size()) + .expect("allocated size underflow"), + ); + self.heap + .deallocate(NonNull::from(header).cast(), overall_layout); false } GcMarkBits::Black => { @@ -43,28 +106,64 @@ impl OldGenerationSpace { if cfg!(debug_assertions) { // second pass to check indexes for (index, live) in self.live_objects.get_mut().iter().enumerate() { + let live = live.expect("All `None` objects should be removed"); assert_eq!(live.as_ref().alloc_info.live_object_index as usize, index); } } - self.stage.finish_stage(CollectStage::Sweep); + } + + /// Destroy an object whose value has not been initialized + #[cold] + pub(super) unsafe fn destroy_uninit_object(&self, header: NonNull>) { + assert!(!header.as_ref().state_bits.get().value_initialized()); + let array = header.as_ref().state_bits.get().array(); + let overall_layout = if array { + header + .as_ref() + .assume_array_header() + .layout_info() + .overall_layout() + } else { + header.as_ref().metadata.type_info.layout.overall_layout() + }; + { + let live_objects = &mut *self.live_objects.get(); + let live_object_index = header.as_ref().alloc_info.live_object_index as usize; + let obj_ref = &mut live_objects[live_object_index]; + assert_eq!(*obj_ref, Some(header)); + *obj_ref = None; // null out remaining reference + } + self.heap.deallocate(header.cast(), overall_layout); + self.allocated_bytes.set( + self.allocated_bytes + .get() + .checked_sub(overall_layout.size()) + .expect("dealloc size overflow"), + ) } #[inline] pub unsafe fn alloc_raw>( &self, - target: T, + target: &T, ) -> Result, OldAllocError> { let overall_layout = target.overall_layout(); let raw_ptr = match self.heap.allocate(overall_layout) { Ok(raw_ptr) => raw_ptr, Err(allocator_api2::alloc::AllocError) => return Err(OldAllocError::OutOfMemory), }; + self.allocated_bytes.set( + self.allocated_bytes + .get() + .checked_add(overall_layout.size()) + .expect("allocated size overflow"), + ); let header_ptr = raw_ptr.cast::(); let live_object_index: u32; { let live_objects = &mut *self.live_objects.get(); live_object_index = u32::try_from(live_objects.len()).unwrap(); - live_objects.push(header_ptr.cast::>()); + live_objects.push(Some(header_ptr.cast::>())); } target.init_header( header_ptr, @@ -77,10 +176,15 @@ impl OldGenerationSpace { ); Ok(header_ptr) } + + #[inline] + pub fn allocated_bytes(&self) -> usize { + self.allocated_bytes.get() + } } #[derive(Debug, thiserror::Error)] pub enum OldAllocError { - #[error("Out of memory (oldgen)")] + #[error("Out of memory (old-gen)")] OutOfMemory, } diff --git a/src/context/young.rs b/src/context/young.rs index 437293b..3d08bb0 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -5,7 +5,7 @@ use std::ptr::NonNull; use bumpalo::ChunkRawIter; use crate::context::layout::{AllocInfo, GcHeader}; -use crate::context::{CollectStage, CollectStageTracker, GenerationId}; +use crate::context::GenerationId; use crate::utils::bumpalo_raw::{BumpAllocRaw, BumpAllocRawConfig}; use crate::utils::Alignment; use crate::CollectorId; @@ -19,25 +19,28 @@ use crate::CollectorId; pub struct YoungGenerationSpace { bump: BumpAllocRaw>, collector_id: Id, - stage: CollectStageTracker, } impl YoungGenerationSpace { + pub unsafe fn new(id: Id) -> Self { + YoungGenerationSpace { + bump: BumpAllocRaw::new(), + collector_id: id, + } + } + /// The maximum size to allocate in the young generation. /// /// Anything larger than this is immediately sent to the old generation. pub const SIZE_LIMIT: usize = 1024; pub unsafe fn sweep(&mut self) { - self.stage - .begin_stage(Some(CollectStage::Mark), CollectStage::Sweep); self.bump.reset(); - self.stage.finish_stage(CollectStage::Sweep); } #[inline] pub unsafe fn alloc_raw>( &self, - target: T, + target: &T, ) -> Result, YoungAllocError> { let overall_layout = target.overall_layout(); if overall_layout.size() > Self::SIZE_LIMIT { @@ -69,16 +72,21 @@ impl YoungGenerationSpace { marker: PhantomData, } } + + #[inline] + pub fn allocated_bytes(&self) -> usize { + self.bump.allocated_bytes() + } } #[derive(Debug, thiserror::Error)] -enum YoungAllocError { - #[error("Out of memory")] +pub enum YoungAllocError { + #[error("Out of memory (young-gen)")] OutOfMemory, #[error("Size exceeds young-alloc limit")] SizeExceedsLimit, } -struct IterRawAllocations<'bump, Id: CollectorId> { +pub(crate) struct IterRawAllocations<'bump, Id: CollectorId> { chunk_iter: ChunkRawIter<'bump>, remaining_chunk_info: Option<(NonNull, usize)>, marker: PhantomData, diff --git a/src/lib.rs b/src/lib.rs index 609eec2..5581014 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,3 +7,5 @@ pub(crate) mod utils; pub use self::collect::{Collect, NullCollect}; pub use self::context::{CollectContext, CollectorId, GarbageCollector}; + +pub use self::gcptr::Gc; diff --git a/src/utils.rs b/src/utils.rs index 7de3e7f..0254518 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,10 +1,103 @@ +use std::backtrace::{Backtrace, BacktraceStatus}; +use std::fmt::Display; use std::mem::ManuallyDrop; +use std::panic::Location; pub(crate) mod bumpalo_raw; mod layout_helpers; pub use self::layout_helpers::{Alignment, LayoutExt}; +enum AbortReason { + Message(M), + FailedAbort, +} + +/// A RAII guard that aborts the process if the operation fails/panics. +/// +/// Can be used to avoid exception safety problems. +/// +/// This guard must be explicitly dropped with [`defuse`](AbortFailureGuard::defuse). +#[must_use] +pub struct AbortFailureGuard { + reason: AbortReason, + location: Option<&'static Location<'static>>, +} +impl AbortFailureGuard { + #[inline] + #[track_caller] + pub fn new(reason: M) -> Self { + AbortFailureGuard { + reason: AbortReason::Message(reason), + location: Some(Location::caller()), + } + } + + #[inline] + pub fn defuse(mut self) { + // replace with a dummy value and drop the real value + drop(std::mem::replace( + &mut self.reason, + AbortReason::FailedAbort, + )); + std::mem::forget(self); + } + + #[inline] + pub fn fail(&self) -> ! { + self.erase().fail_impl() + } + + #[inline] + fn erase(&self) -> AbortFailureGuard<&'_ dyn Display> { + AbortFailureGuard { + reason: match self.reason { + AbortReason::Message(ref reason) => AbortReason::Message(reason as &'_ dyn Display), + AbortReason::FailedAbort => AbortReason::FailedAbort, + }, + location: self.location, + } + } +} +impl<'a> AbortFailureGuard<&'a dyn Display> { + #[cold] + #[inline(never)] + pub fn fail_impl(&self) -> ! { + match self.reason { + AbortReason::Message(msg) => { + let secondary_abort_guard = AbortFailureGuard { + reason: AbortReason::::FailedAbort, + location: self.location, + }; + eprintln!("Aborting: {msg}"); + let backtrace = Backtrace::capture(); + if let Some(location) = self.location { + eprintln!("Location: {location}") + } + if !std::thread::panicking() { + eprintln!( + "WARNING: Thread not panicking (forgot to defuse AbortFailureGuard?)" + ); + } + if matches!(backtrace.status(), BacktraceStatus::Captured) { + eprintln!("Backtrace: {backtrace}"); + } + secondary_abort_guard.defuse(); + } + // don't do anything, failed to print primary abort message + AbortReason::FailedAbort => {} + } + std::process::abort(); + } +} +impl Drop for AbortFailureGuard { + #[cold] + #[inline] + fn drop(&mut self) { + self.fail() + } +} + /// Transmute one type into another, /// without doing compile-time checks for sizes. /// diff --git a/src/utils/bumpalo_raw.rs b/src/utils/bumpalo_raw.rs index 8f1a71e..24b5754 100644 --- a/src/utils/bumpalo_raw.rs +++ b/src/utils/bumpalo_raw.rs @@ -10,6 +10,13 @@ pub struct BumpAllocRaw { marker: PhantomData, } impl BumpAllocRaw { + pub fn new() -> Self { + BumpAllocRaw { + inner: Bump::new(), + marker: PhantomData, + } + } + #[inline(always)] pub fn try_alloc_layout(&self, layout: Layout) -> Result, AllocErr> { assert_eq!(layout.align(), Config::FIXED_ALIGNMENT.value()); @@ -25,6 +32,11 @@ impl BumpAllocRaw { pub unsafe fn iter_allocated_chunks_raw(&self) -> bumpalo::ChunkRawIter<'_> { self.inner.iter_allocated_chunks_raw() } + + #[inline] + pub fn allocated_bytes(&self) -> usize { + self.inner.allocated_bytes() + } } pub trait BumpAllocRawConfig { diff --git a/src/utils/layout_helpers.rs b/src/utils/layout_helpers.rs index 0f60dd2..0baa3b1 100644 --- a/src/utils/layout_helpers.rs +++ b/src/utils/layout_helpers.rs @@ -98,7 +98,7 @@ impl LayoutExt { * is we skip the usize::is_power_of_two check. */ if new_size > Self::max_size_for_align(unsafe { Alignment::new_unchecked(new_align) }) { - return Err(LayoutExtError); + Err(LayoutExtError) } else { Ok(( unsafe { Layout::from_size_align_unchecked(new_size, new_align) }, From 304f4df00bad4f365be8dd0b1e7f18b85c2c6b08 Mon Sep 17 00:00:00 2001 From: Techcable Date: Sat, 4 May 2024 21:59:11 -0700 Subject: [PATCH 11/35] Avoid GcHeader::regular_value_ptr to satisfy miri --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + src/context.rs | 46 +++++++++++++++++++++---------------------- src/context/layout.rs | 1 + src/context/old.rs | 11 +++++++++-- src/context/young.rs | 12 +++++++++-- 6 files changed, 51 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03b820e..6e55e99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,6 +148,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.154" @@ -535,6 +541,7 @@ dependencies = [ "arbitrary-int", "bitbybit", "bumpalo", + "lazy_static", "log", "rustversion", "scopeguard", diff --git a/Cargo.toml b/Cargo.toml index e18ca38..a1e3602 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ stacker = "0.1" zerogc-next-mimalloc-semisafe = { version = "0.1.0-alpha.1", path = "libs/mimalloc-semisafe" } log = "0.4.21" scopeguard = "1.2" +lazy_static = "1.4.0" [build-dependencies] rustversion = "1" diff --git a/src/context.rs b/src/context.rs index 07f2c17..fc4be14 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,4 +1,5 @@ use std::alloc::Layout; +use std::array; use std::cell::{Cell, RefCell}; use std::error::Error; use std::fmt::Debug; @@ -157,7 +158,7 @@ impl GarbageCollector { #[track_caller] pub fn alloc_with>(&self, func: impl FnOnce() -> T) -> Gc<'_, T, Id> { unsafe { - let header = self.alloc_raw(&RegularAlloc { + let (header, value_ptr) = self.alloc_raw(&RegularAlloc { state: &self.state, type_info: GcTypeInfo::new::(), }); @@ -165,18 +166,20 @@ impl GarbageCollector { header, old_generation: &self.old_generation, }; - let value_ptr = header.as_ref().regular_value_ptr().cast::(); - value_ptr.as_ptr().write(func()); + value_ptr.as_ptr().cast::().write(func()); header .as_ref() .update_state_bits(|state| state.with_value_initialized(true)); initialization_guard.defuse(); // successful initialization; - Gc::from_raw_ptr(value_ptr) + Gc::from_raw_ptr(value_ptr.cast()) } } #[inline] - unsafe fn alloc_raw>(&self, target: &T) -> NonNull { + unsafe fn alloc_raw>( + &self, + target: &T, + ) -> (NonNull, NonNull) { match self.young_generation.alloc_raw(target) { Ok(res) => res, Err(YoungAllocError::SizeExceedsLimit) => self.alloc_raw_fallback(target), @@ -185,7 +188,10 @@ impl GarbageCollector { } #[cold] - unsafe fn alloc_raw_fallback>(&self, target: &T) -> NonNull { + unsafe fn alloc_raw_fallback>( + &self, + target: &T, + ) -> (NonNull, NonNull) { self.old_generation .alloc_raw(target) .unwrap_or_else(|err| Self::oom(err)) @@ -506,7 +512,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { GenerationId::Young => { let array_value_size: Option; // reallocate in oldgen - let copied_ptr = if array { + let (copied_header_ptr, copied_value_ptr) = if array { let array_type_info = type_info.assume_array_info(); debug_assert!(std::ptr::eq( array_type_info, @@ -524,7 +530,9 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { type_info: array_type_info, state: &self.garbage_collector.state, }) - .map(NonNull::cast::>) + .map(|(array_header_ptr, value_ptr)| { + (array_header_ptr.cast::>(), value_ptr) + }) } else { array_value_size = None; self.garbage_collector @@ -535,28 +543,22 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { }) } .unwrap_or_else(|_| { - // TODO: This panic is fatal, will cause an abort + // This panic is fatal, will cause an abort panic!("Oldgen alloc failure") }); - copied_ptr - .as_ref() - .state_bits - .set(header_ptr.as_ref().state_bits.get()); - copied_ptr.as_ref().update_state_bits(|bits| { + copied_header_ptr.as_ref().update_state_bits(|bits| { debug_assert!(!bits.forwarded()); bits.with_generation(GenerationId::Old) .with_value_initialized(true) + .with_raw_mark_bits(GcMarkBits::Black.to_raw(&self.garbage_collector.state)) }); header_ptr .as_ref() .update_state_bits(|bits| bits.with_forwarded(true)); - (&mut *header_ptr.as_ptr()).metadata.forward_ptr = copied_ptr.cast(); + (&mut *header_ptr.as_ptr()).metadata.forward_ptr = copied_header_ptr.cast(); // NOTE: Copy uninitialized bytes is safe here, as long as they are not read in dest if array { - copied_ptr - .cast::>() - .as_ref() - .array_value_ptr() + copied_value_ptr .cast::() .as_ptr() .copy_from_nonoverlapping( @@ -568,9 +570,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { array_value_size.unwrap(), ) } else { - copied_ptr - .as_ref() - .regular_value_ptr() + copied_value_ptr .cast::() .as_ptr() .copy_from_nonoverlapping( @@ -582,7 +582,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { type_info.layout.value_layout().size(), ); } - copied_ptr + copied_header_ptr } GenerationId::Old => header_ptr, // no copying needed for oldgen }; diff --git a/src/context/layout.rs b/src/context/layout.rs index 96e30b0..e78ebfd 100644 --- a/src/context/layout.rs +++ b/src/context/layout.rs @@ -322,6 +322,7 @@ impl GcHeader { } #[inline] + #[deprecated(note = "miri doesn't like this")] pub fn regular_value_ptr(&self) -> NonNull { unsafe { NonNull::new_unchecked( diff --git a/src/context/old.rs b/src/context/old.rs index 1d5151e..f8f954e 100644 --- a/src/context/old.rs +++ b/src/context/old.rs @@ -146,7 +146,7 @@ impl OldGenerationSpace { pub unsafe fn alloc_raw>( &self, target: &T, - ) -> Result, OldAllocError> { + ) -> Result<(NonNull, NonNull), OldAllocError> { let overall_layout = target.overall_layout(); let raw_ptr = match self.heap.allocate(overall_layout) { Ok(raw_ptr) => raw_ptr, @@ -174,7 +174,14 @@ impl OldGenerationSpace { collector_id: self.collector_id, }, ); - Ok(header_ptr) + Ok(( + header_ptr, + NonNull::new_unchecked(raw_ptr.cast::().as_ptr().add(if T::ARRAY { + GcHeader::::ARRAY_VALUE_OFFSET + } else { + GcHeader::::REGULAR_VALUE_OFFSET + })), + )) } #[inline] diff --git a/src/context/young.rs b/src/context/young.rs index 3d08bb0..d3afd61 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -3,6 +3,7 @@ use std::marker::PhantomData; use std::ptr::NonNull; use bumpalo::ChunkRawIter; +use lazy_static::lazy_static; use crate::context::layout::{AllocInfo, GcHeader}; use crate::context::GenerationId; @@ -41,7 +42,7 @@ impl YoungGenerationSpace { pub unsafe fn alloc_raw>( &self, target: &T, - ) -> Result, YoungAllocError> { + ) -> Result<(NonNull, NonNull), YoungAllocError> { let overall_layout = target.overall_layout(); if overall_layout.size() > Self::SIZE_LIMIT { return Err(YoungAllocError::SizeExceedsLimit); @@ -61,7 +62,14 @@ impl YoungGenerationSpace { collector_id: self.collector_id, }, ); - Ok(header_ptr) + Ok(( + header_ptr, + NonNull::new_unchecked(raw_ptr.as_ptr().add(if T::ARRAY { + GcHeader::::ARRAY_VALUE_OFFSET + } else { + GcHeader::::REGULAR_VALUE_OFFSET + })), + )) } #[inline] From 09e90725c18b937753b32392ec59b1154f67c9c4 Mon Sep 17 00:00:00 2001 From: Techcable Date: Sat, 4 May 2024 21:59:18 -0700 Subject: [PATCH 12/35] Revert "Avoid GcHeader::regular_value_ptr to satisfy miri" These address calculations can't really be avoided :/ This reverts commit 2f0926f95db863829159608e82bf7c5f4c98adf2 --- Cargo.lock | 7 ------- Cargo.toml | 1 - src/context.rs | 46 +++++++++++++++++++++---------------------- src/context/layout.rs | 1 - src/context/old.rs | 11 ++--------- src/context/young.rs | 12 ++--------- 6 files changed, 27 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e55e99..03b820e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,12 +148,6 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - [[package]] name = "libc" version = "0.2.154" @@ -541,7 +535,6 @@ dependencies = [ "arbitrary-int", "bitbybit", "bumpalo", - "lazy_static", "log", "rustversion", "scopeguard", diff --git a/Cargo.toml b/Cargo.toml index a1e3602..e18ca38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,6 @@ stacker = "0.1" zerogc-next-mimalloc-semisafe = { version = "0.1.0-alpha.1", path = "libs/mimalloc-semisafe" } log = "0.4.21" scopeguard = "1.2" -lazy_static = "1.4.0" [build-dependencies] rustversion = "1" diff --git a/src/context.rs b/src/context.rs index fc4be14..07f2c17 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,5 +1,4 @@ use std::alloc::Layout; -use std::array; use std::cell::{Cell, RefCell}; use std::error::Error; use std::fmt::Debug; @@ -158,7 +157,7 @@ impl GarbageCollector { #[track_caller] pub fn alloc_with>(&self, func: impl FnOnce() -> T) -> Gc<'_, T, Id> { unsafe { - let (header, value_ptr) = self.alloc_raw(&RegularAlloc { + let header = self.alloc_raw(&RegularAlloc { state: &self.state, type_info: GcTypeInfo::new::(), }); @@ -166,20 +165,18 @@ impl GarbageCollector { header, old_generation: &self.old_generation, }; - value_ptr.as_ptr().cast::().write(func()); + let value_ptr = header.as_ref().regular_value_ptr().cast::(); + value_ptr.as_ptr().write(func()); header .as_ref() .update_state_bits(|state| state.with_value_initialized(true)); initialization_guard.defuse(); // successful initialization; - Gc::from_raw_ptr(value_ptr.cast()) + Gc::from_raw_ptr(value_ptr) } } #[inline] - unsafe fn alloc_raw>( - &self, - target: &T, - ) -> (NonNull, NonNull) { + unsafe fn alloc_raw>(&self, target: &T) -> NonNull { match self.young_generation.alloc_raw(target) { Ok(res) => res, Err(YoungAllocError::SizeExceedsLimit) => self.alloc_raw_fallback(target), @@ -188,10 +185,7 @@ impl GarbageCollector { } #[cold] - unsafe fn alloc_raw_fallback>( - &self, - target: &T, - ) -> (NonNull, NonNull) { + unsafe fn alloc_raw_fallback>(&self, target: &T) -> NonNull { self.old_generation .alloc_raw(target) .unwrap_or_else(|err| Self::oom(err)) @@ -512,7 +506,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { GenerationId::Young => { let array_value_size: Option; // reallocate in oldgen - let (copied_header_ptr, copied_value_ptr) = if array { + let copied_ptr = if array { let array_type_info = type_info.assume_array_info(); debug_assert!(std::ptr::eq( array_type_info, @@ -530,9 +524,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { type_info: array_type_info, state: &self.garbage_collector.state, }) - .map(|(array_header_ptr, value_ptr)| { - (array_header_ptr.cast::>(), value_ptr) - }) + .map(NonNull::cast::>) } else { array_value_size = None; self.garbage_collector @@ -543,22 +535,28 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { }) } .unwrap_or_else(|_| { - // This panic is fatal, will cause an abort + // TODO: This panic is fatal, will cause an abort panic!("Oldgen alloc failure") }); - copied_header_ptr.as_ref().update_state_bits(|bits| { + copied_ptr + .as_ref() + .state_bits + .set(header_ptr.as_ref().state_bits.get()); + copied_ptr.as_ref().update_state_bits(|bits| { debug_assert!(!bits.forwarded()); bits.with_generation(GenerationId::Old) .with_value_initialized(true) - .with_raw_mark_bits(GcMarkBits::Black.to_raw(&self.garbage_collector.state)) }); header_ptr .as_ref() .update_state_bits(|bits| bits.with_forwarded(true)); - (&mut *header_ptr.as_ptr()).metadata.forward_ptr = copied_header_ptr.cast(); + (&mut *header_ptr.as_ptr()).metadata.forward_ptr = copied_ptr.cast(); // NOTE: Copy uninitialized bytes is safe here, as long as they are not read in dest if array { - copied_value_ptr + copied_ptr + .cast::>() + .as_ref() + .array_value_ptr() .cast::() .as_ptr() .copy_from_nonoverlapping( @@ -570,7 +568,9 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { array_value_size.unwrap(), ) } else { - copied_value_ptr + copied_ptr + .as_ref() + .regular_value_ptr() .cast::() .as_ptr() .copy_from_nonoverlapping( @@ -582,7 +582,7 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { type_info.layout.value_layout().size(), ); } - copied_header_ptr + copied_ptr } GenerationId::Old => header_ptr, // no copying needed for oldgen }; diff --git a/src/context/layout.rs b/src/context/layout.rs index e78ebfd..96e30b0 100644 --- a/src/context/layout.rs +++ b/src/context/layout.rs @@ -322,7 +322,6 @@ impl GcHeader { } #[inline] - #[deprecated(note = "miri doesn't like this")] pub fn regular_value_ptr(&self) -> NonNull { unsafe { NonNull::new_unchecked( diff --git a/src/context/old.rs b/src/context/old.rs index f8f954e..1d5151e 100644 --- a/src/context/old.rs +++ b/src/context/old.rs @@ -146,7 +146,7 @@ impl OldGenerationSpace { pub unsafe fn alloc_raw>( &self, target: &T, - ) -> Result<(NonNull, NonNull), OldAllocError> { + ) -> Result, OldAllocError> { let overall_layout = target.overall_layout(); let raw_ptr = match self.heap.allocate(overall_layout) { Ok(raw_ptr) => raw_ptr, @@ -174,14 +174,7 @@ impl OldGenerationSpace { collector_id: self.collector_id, }, ); - Ok(( - header_ptr, - NonNull::new_unchecked(raw_ptr.cast::().as_ptr().add(if T::ARRAY { - GcHeader::::ARRAY_VALUE_OFFSET - } else { - GcHeader::::REGULAR_VALUE_OFFSET - })), - )) + Ok(header_ptr) } #[inline] diff --git a/src/context/young.rs b/src/context/young.rs index d3afd61..3d08bb0 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -3,7 +3,6 @@ use std::marker::PhantomData; use std::ptr::NonNull; use bumpalo::ChunkRawIter; -use lazy_static::lazy_static; use crate::context::layout::{AllocInfo, GcHeader}; use crate::context::GenerationId; @@ -42,7 +41,7 @@ impl YoungGenerationSpace { pub unsafe fn alloc_raw>( &self, target: &T, - ) -> Result<(NonNull, NonNull), YoungAllocError> { + ) -> Result, YoungAllocError> { let overall_layout = target.overall_layout(); if overall_layout.size() > Self::SIZE_LIMIT { return Err(YoungAllocError::SizeExceedsLimit); @@ -62,14 +61,7 @@ impl YoungGenerationSpace { collector_id: self.collector_id, }, ); - Ok(( - header_ptr, - NonNull::new_unchecked(raw_ptr.as_ptr().add(if T::ARRAY { - GcHeader::::ARRAY_VALUE_OFFSET - } else { - GcHeader::::REGULAR_VALUE_OFFSET - })), - )) + Ok(header_ptr) } #[inline] From bf5da49727093d29f64009e280a457f4454f9594 Mon Sep 17 00:00:00 2001 From: Techcable Date: Sun, 5 May 2024 13:31:17 -0700 Subject: [PATCH 13/35] Add allocator debug mode, avoiding bumpalo Allows using AddressSanitizer --- Cargo.lock | 3 + Cargo.toml | 5 +- src/context.rs | 1 + src/context/alloc.rs | 118 ++++++++++++++++++++++++++++++++++++ src/context/young.rs | 139 +++++++++++++++++++------------------------ src/utils.rs | 1 - 6 files changed, 188 insertions(+), 79 deletions(-) create mode 100644 src/context/alloc.rs diff --git a/Cargo.lock b/Cargo.lock index 03b820e..c597f01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,9 @@ name = "bumpalo" version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +dependencies = [ + "allocator-api2", +] [[package]] name = "cc" diff --git a/Cargo.toml b/Cargo.toml index e18ca38..4e98091 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ version.workspace = true edition.workspace = true [dependencies] -bumpalo = "3.16" +bumpalo = { version = "3.16", features = ["allocator-api2"] } allocator-api2 = "0.2.18" bitbybit = "1.3.2" arbitrary-int = "1.2.7" @@ -25,6 +25,9 @@ rustversion = "1" slog = "2.7.0" slog-term = "2.9.1" +[features] +debug-alloc = [] + [workspace] resolver = "2" members = [".", "libs/*"] diff --git a/src/context.rs b/src/context.rs index 07f2c17..3dc2a73 100644 --- a/src/context.rs +++ b/src/context.rs @@ -18,6 +18,7 @@ use crate::gcptr::Gc; use crate::utils::AbortFailureGuard; use crate::Collect; +mod alloc; pub(crate) mod layout; mod old; mod young; diff --git a/src/context/alloc.rs b/src/context/alloc.rs new file mode 100644 index 0000000..71d1a0e --- /dev/null +++ b/src/context/alloc.rs @@ -0,0 +1,118 @@ +use std::alloc::Layout; +use std::cell::{Cell, RefCell}; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::ptr::NonNull; + +use allocator_api2::alloc::{AllocError, Allocator}; + +pub struct CountingAlloc { + alloc: A, + allocated_bytes: Cell, +} +impl CountingAlloc { + #[inline] + pub fn new(alloc: A) -> Self { + CountingAlloc { + alloc, + allocated_bytes: Cell::new(0), + } + } + + #[inline] + pub fn as_inner(&self) -> &A { + &self.alloc + } + + #[inline] + pub fn as_inner_mut(&mut self) -> &mut A { + &mut self.alloc + } + + #[inline] + pub fn allocated_bytes(&self) -> usize { + self.allocated_bytes.get() + } +} + +unsafe impl Allocator for CountingAlloc { + #[inline] + fn allocate(&self, layout: Layout) -> Result, AllocError> { + let res = self.as_inner().allocate(layout)?; + self.allocated_bytes + .set(self.allocated_bytes.get() + res.len()); + Ok(res) + } + + #[inline] + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + self.as_inner().deallocate(ptr, layout); + self.allocated_bytes + .set(self.allocated_bytes.get() - layout.size()); + } +} + +#[derive(Debug, Eq, PartialEq)] +struct AllocObject { + ptr: NonNull, + layout: Layout, +} + +/// An allocator that supports freeing objects in bulk via the [`GroupAlloc::clear`] method. +pub struct GroupAlloc { + alloc: A, + allocated_objects: RefCell, AllocObject>>, +} +impl GroupAlloc { + pub fn new(alloc: A) -> Self { + GroupAlloc { + alloc, + allocated_objects: Default::default(), + } + } + + pub unsafe fn reset(&self) { + let mut objects = self.allocated_objects.borrow_mut(); + for (&ptr, obj) in objects.iter() { + assert_eq!(ptr, obj.ptr); + self.alloc.deallocate(ptr, obj.layout); + } + objects.clear(); + } +} +unsafe impl Allocator for GroupAlloc { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + let res = self.alloc.allocate(layout)?; + let mut objects = self.allocated_objects.borrow_mut(); + match objects.entry(res.cast::()) { + Entry::Occupied(_) => panic!("Duplicate entries for {res:?}"), + Entry::Vacant(entry) => { + entry.insert(AllocObject { + ptr: res.cast(), + layout: Layout::from_size_align(res.len(), layout.align()).unwrap(), + }); + } + } + Ok(res) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + let mut allocated = self.allocated_objects.borrow_mut(); + match allocated.entry(ptr) { + Entry::Occupied(entry) => { + assert_eq!(*entry.get(), AllocObject { ptr, layout }); + entry.remove(); + } + Entry::Vacant(_) => panic!("Missing entry for {ptr:?} w/ {layout:?}"), + } + drop(allocated); // release guard + self.alloc.deallocate(ptr, layout); + } +} +impl Drop for GroupAlloc { + fn drop(&mut self) { + unsafe { + self.reset(); + } + } +} diff --git a/src/context/young.rs b/src/context/young.rs index 3d08bb0..db0fc40 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -1,15 +1,67 @@ +use allocator_api2::alloc::{AllocError, Allocator}; +use bumpalo::Bump; +use std::alloc::Layout; use std::cell::Cell; use std::marker::PhantomData; +use std::mem::ManuallyDrop; use std::ptr::NonNull; -use bumpalo::ChunkRawIter; - +use crate::context::alloc::{CountingAlloc, GroupAlloc}; use crate::context::layout::{AllocInfo, GcHeader}; use crate::context::GenerationId; -use crate::utils::bumpalo_raw::{BumpAllocRaw, BumpAllocRawConfig}; use crate::utils::Alignment; use crate::CollectorId; +struct YoungAlloc { + #[cfg(feature = "debug-alloc")] + group: GroupAlloc, + #[cfg(not(feature = "debug-alloc"))] + bump: Bump, +} +impl YoungAlloc { + pub fn new() -> Self { + #[cfg(feature = "debug-alloc")] + { + YoungAlloc { + group: GroupAlloc::new(allocator_api2::alloc::Global), + } + } + #[cfg(not(feature = "debug-alloc"))] + { + YoungAlloc { bump: Bump::new() } + } + } + fn alloc_impl(&self) -> impl Allocator + '_ { + #[cfg(feature = "debug-alloc")] + { + &self.group + } + #[cfg(not(feature = "debug-alloc"))] + { + &self.bump + } + } + unsafe fn reset(&mut self) { + #[cfg(feature = "debug-alloc")] + { + self.group.reset(); + } + #[cfg(not(feature = "debug-alloc"))] + { + self.bump.reset(); + } + } +} +unsafe impl Allocator for YoungAlloc { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + self.alloc_impl().allocate(layout) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + self.alloc_impl().deallocate(ptr, layout) + } +} + /// A young-generation object-space /// /// If copying is in progress, @@ -17,13 +69,15 @@ use crate::CollectorId; /// /// The design of the allocator is heavily based on [`bumpalo`](https://crates.io/crates/bumpalo) pub struct YoungGenerationSpace { - bump: BumpAllocRaw>, + alloc: CountingAlloc, collector_id: Id, } impl YoungGenerationSpace { pub unsafe fn new(id: Id) -> Self { + #[cfg(not(feature = "debug-alloc"))] + let bump = ManuallyDrop::new(Box::new(Bump::new())); YoungGenerationSpace { - bump: BumpAllocRaw::new(), + alloc: CountingAlloc::new(YoungAlloc::new()), collector_id: id, } } @@ -34,7 +88,7 @@ impl YoungGenerationSpace { pub const SIZE_LIMIT: usize = 1024; pub unsafe fn sweep(&mut self) { - self.bump.reset(); + self.alloc.as_inner_mut().reset(); } #[inline] @@ -46,7 +100,7 @@ impl YoungGenerationSpace { if overall_layout.size() > Self::SIZE_LIMIT { return Err(YoungAllocError::SizeExceedsLimit); } - let Ok(raw_ptr) = self.bump.try_alloc_layout(overall_layout) else { + let Ok(raw_ptr) = self.alloc.allocate(overall_layout) else { return Err(YoungAllocError::OutOfMemory); }; let header_ptr = raw_ptr.cast::(); @@ -64,18 +118,9 @@ impl YoungGenerationSpace { Ok(header_ptr) } - #[inline] - pub unsafe fn iter_raw_allocations(&self) -> IterRawAllocations<'_, Id> { - IterRawAllocations { - chunk_iter: self.bump.iter_allocated_chunks_raw(), - remaining_chunk_info: None, - marker: PhantomData, - } - } - #[inline] pub fn allocated_bytes(&self) -> usize { - self.bump.allocated_bytes() + self.alloc.allocated_bytes() } } #[derive(Debug, thiserror::Error)] @@ -85,63 +130,3 @@ pub enum YoungAllocError { #[error("Size exceeds young-alloc limit")] SizeExceedsLimit, } - -pub(crate) struct IterRawAllocations<'bump, Id: CollectorId> { - chunk_iter: ChunkRawIter<'bump>, - remaining_chunk_info: Option<(NonNull, usize)>, - marker: PhantomData, -} -impl Iterator for IterRawAllocations<'_, Id> { - type Item = NonNull>; - - #[inline] - fn next(&mut self) -> Option { - loop { - if let Some((ref mut remaining_chunk_ptr, ref mut remaining_chunk_size)) = - self.remaining_chunk_info - { - if *remaining_chunk_size == 0 { - continue; - } - debug_assert!( - *remaining_chunk_size >= GcHeader::::REGULAR_HEADER_LAYOUT.size() - ); - debug_assert_eq!( - // TODO: Use `is_aligned_to` once stabilized - remaining_chunk_ptr - .as_ptr() - .align_offset(GcHeader::::FIXED_ALIGNMENT), - 0 - ); - unsafe { - let header = &*remaining_chunk_ptr.as_ptr().cast::>(); - debug_assert_eq!(header.state_bits.get().generation(), GenerationId::Young); - let overall_object_size = header.alloc_info.this_object_overall_size as usize; - *remaining_chunk_ptr = NonNull::new_unchecked( - remaining_chunk_ptr.as_ptr().add(overall_object_size), - ); - *remaining_chunk_size = remaining_chunk_size.unchecked_sub(overall_object_size); - return Some(NonNull::from(header)); - } - } else { - match self.chunk_iter.next() { - None => return None, - Some((remaining_chunk_ptr, remaining_chunk_size)) => { - self.remaining_chunk_info = Some(( - unsafe { NonNull::new_unchecked(remaining_chunk_ptr) }, - remaining_chunk_size, - )); - } - } - } - } - } -} - -struct BumpConfig(PhantomData<&'static Id>); -impl BumpAllocRawConfig for BumpConfig { - const FIXED_ALIGNMENT: Alignment = match Alignment::new(GcHeader::::FIXED_ALIGNMENT) { - Ok(alignment) => alignment, - Err(_) => panic!("GcHeader alignment must be valid"), - }; -} diff --git a/src/utils.rs b/src/utils.rs index 0254518..b220ac5 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -3,7 +3,6 @@ use std::fmt::Display; use std::mem::ManuallyDrop; use std::panic::Location; -pub(crate) mod bumpalo_raw; mod layout_helpers; pub use self::layout_helpers::{Alignment, LayoutExt}; From 3aa6904b94dcf2a5972f01b7cba27894477317fb Mon Sep 17 00:00:00 2001 From: Techcable Date: Sun, 5 May 2024 17:27:39 -0700 Subject: [PATCH 14/35] In debug mode, touch roots after trace Triggers errors earlier in AddressSanitizer --- src/context.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/context.rs b/src/context.rs index 3dc2a73..b83fe63 100644 --- a/src/context.rs +++ b/src/context.rs @@ -235,8 +235,8 @@ impl GarbageCollector { roots.retain(|root| { match root.upgrade() { Some(root) => { - root.header - .set(unsafe { context.collect_gcheader(root.header.get()) }); + let new_header = unsafe { context.collect_gcheader(root.header.get()) }; + root.header.set(new_header); true // keep live root } None => false, // delete dead root @@ -250,6 +250,22 @@ impl GarbageCollector { self.young_generation.sweep(); self.old_generation.sweep(&self.state); } + // touch roots to verify validity + #[cfg(debug_assertions)] + for root in self.roots.get_mut().iter() { + unsafe { + assert!(!root + .upgrade() + .unwrap() + .header + .get() + .as_ref() + .state_bits + .get() + .forwarded()); + } + } + // invert meaning of the mark bits self.state .mark_bits_inverted From 661f3be7c957f690febeccbb1fd8bf56d1d42d76 Mon Sep 17 00:00:00 2001 From: Techcable Date: Sun, 5 May 2024 18:17:07 -0700 Subject: [PATCH 15/35] Disable stacker::grow under miri Because FFI is forbidden --- src/context.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/context.rs b/src/context.rs index b83fe63..716ffa2 100644 --- a/src/context.rs +++ b/src/context.rs @@ -613,11 +613,14 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { * NOTE: Cannot have aliasing &mut header references during this recursion * The parameters to maybe_grow are completely arbitrary right now. */ + #[cfg(not(miri))] stacker::maybe_grow( 4096, // 4KB 128 * 1024, // 128KB || self.trace_children(forwarded_ptr, trace_func), ); + #[cfg(miri)] + self.trace_children(forwarded_ptr, trace_func); } forwarded_ptr } From 9bc55d93ed6469bbb824e3164147f8de80d8569f Mon Sep 17 00:00:00 2001 From: Techcable Date: Sun, 5 May 2024 18:17:45 -0700 Subject: [PATCH 16/35] Fix Gc::header offset calculation UB This causes the to pass miri (onlt with tree borrows, not stacked borrows) --- src/gcptr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gcptr.rs b/src/gcptr.rs index 39a5094..86b1883 100644 --- a/src/gcptr.rs +++ b/src/gcptr.rs @@ -22,7 +22,7 @@ impl<'gc, T: Collect, Id: CollectorId> Gc<'gc, T, Id> { #[inline] pub(crate) fn header(&self) -> &'_ GcHeader { unsafe { - &*((self.ptr.as_ptr() as *mut u8).sub(GcHeader::::FIXED_ALIGNMENT) + &*((self.ptr.as_ptr() as *mut u8).sub(GcHeader::::REGULAR_VALUE_OFFSET) as *mut GcHeader) } } From d17c4620450afd3c237007e9f2e8b6abc48ae96d Mon Sep 17 00:00:00 2001 From: Techcable Date: Sun, 5 May 2024 18:18:56 -0700 Subject: [PATCH 17/35] In debug-alloc, ArenaAlloc cannot frees individual allocs This emulates bumpalo::Bump behavior and avoids a slow HashMap --- src/context/alloc.rs | 52 +++++++++++++++++--------------------------- src/context/young.rs | 6 ++--- 2 files changed, 23 insertions(+), 35 deletions(-) diff --git a/src/context/alloc.rs b/src/context/alloc.rs index 71d1a0e..ab7eea4 100644 --- a/src/context/alloc.rs +++ b/src/context/alloc.rs @@ -58,58 +58,46 @@ struct AllocObject { layout: Layout, } -/// An allocator that supports freeing objects in bulk via the [`GroupAlloc::clear`] method. -pub struct GroupAlloc { +/// An arena allocator that only supports freeing objects in bulk. +/// +/// This emulates [`bumpalo::Bump`], +/// but allocates each object individually for better tracking. +pub struct ArenaAlloc { alloc: A, - allocated_objects: RefCell, AllocObject>>, + allocated_objects: RefCell>, } -impl GroupAlloc { +impl ArenaAlloc { pub fn new(alloc: A) -> Self { - GroupAlloc { + ArenaAlloc { alloc, allocated_objects: Default::default(), } } - pub unsafe fn reset(&self) { - let mut objects = self.allocated_objects.borrow_mut(); - for (&ptr, obj) in objects.iter() { - assert_eq!(ptr, obj.ptr); - self.alloc.deallocate(ptr, obj.layout); + pub unsafe fn reset(&mut self) { + let objects = self.allocated_objects.get_mut(); + for obj in objects.iter() { + self.alloc.deallocate(obj.ptr, obj.layout); } objects.clear(); } } -unsafe impl Allocator for GroupAlloc { +unsafe impl Allocator for ArenaAlloc { fn allocate(&self, layout: Layout) -> Result, AllocError> { let res = self.alloc.allocate(layout)?; - let mut objects = self.allocated_objects.borrow_mut(); - match objects.entry(res.cast::()) { - Entry::Occupied(_) => panic!("Duplicate entries for {res:?}"), - Entry::Vacant(entry) => { - entry.insert(AllocObject { - ptr: res.cast(), - layout: Layout::from_size_align(res.len(), layout.align()).unwrap(), - }); - } - } + + self.allocated_objects.borrow_mut().push(AllocObject { + ptr: res.cast(), + layout: Layout::from_size_align(res.len(), layout.align()).unwrap(), + }); Ok(res) } unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - let mut allocated = self.allocated_objects.borrow_mut(); - match allocated.entry(ptr) { - Entry::Occupied(entry) => { - assert_eq!(*entry.get(), AllocObject { ptr, layout }); - entry.remove(); - } - Entry::Vacant(_) => panic!("Missing entry for {ptr:?} w/ {layout:?}"), - } - drop(allocated); // release guard - self.alloc.deallocate(ptr, layout); + // dealloc is nop } } -impl Drop for GroupAlloc { +impl Drop for ArenaAlloc { fn drop(&mut self) { unsafe { self.reset(); diff --git a/src/context/young.rs b/src/context/young.rs index db0fc40..8a93b6e 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -6,7 +6,7 @@ use std::marker::PhantomData; use std::mem::ManuallyDrop; use std::ptr::NonNull; -use crate::context::alloc::{CountingAlloc, GroupAlloc}; +use crate::context::alloc::{ArenaAlloc, CountingAlloc}; use crate::context::layout::{AllocInfo, GcHeader}; use crate::context::GenerationId; use crate::utils::Alignment; @@ -14,7 +14,7 @@ use crate::CollectorId; struct YoungAlloc { #[cfg(feature = "debug-alloc")] - group: GroupAlloc, + group: ArenaAlloc, #[cfg(not(feature = "debug-alloc"))] bump: Bump, } @@ -23,7 +23,7 @@ impl YoungAlloc { #[cfg(feature = "debug-alloc")] { YoungAlloc { - group: GroupAlloc::new(allocator_api2::alloc::Global), + group: ArenaAlloc::new(allocator_api2::alloc::Global), } } #[cfg(not(feature = "debug-alloc"))] From 68ae6353ba8909ee186e41a677478115ff695a54 Mon Sep 17 00:00:00 2001 From: Techcable Date: Sun, 5 May 2024 18:35:28 -0700 Subject: [PATCH 18/35] Attempt to explicitly Drop old-gen objects under miri Unfortunately, miri still reports leaks --- src/context/old.rs | 91 +++++++++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 29 deletions(-) diff --git a/src/context/old.rs b/src/context/old.rs index 1d5151e..71770ec 100644 --- a/src/context/old.rs +++ b/src/context/old.rs @@ -8,9 +8,10 @@ use crate::context::layout::{AllocInfo, GcHeader, GcMarkBits}; use crate::context::{CollectorState, GenerationId}; use crate::CollectorId; -mod miri { +mod fallback { use allocator_api2::alloc::AllocError; use std::alloc::Layout; + use std::collections::HashMap; use std::ptr::NonNull; pub struct HeapAllocFallback; @@ -41,11 +42,24 @@ mod miri { } } -#[cfg(miri)] -type HeapAllocator = miri::HeapAllocFallback; -#[cfg(not(miri))] +#[cfg(any(miri, feature = "debug-alloc"))] +type HeapAllocator = fallback::HeapAllocFallback; +#[cfg(not(any(miri, feature = "debug-alloc")))] type HeapAllocator = zerogc_next_mimalloc_semisafe::heap::MimallocHeap; +const DROP_NEEDS_EXPLICIT_FREE: bool = cfg!(any(miri, feature = "debug-alloc")); + +enum ObjectFreeCondition<'a, Id: CollectorId> { + /// Free the object if it has not been marked. + /// + /// Used to sweep objects. + Unmarked { state: &'a CollectorState }, + /// Unconditionally free the object. + /// + /// Used to destroy the + Always, +} + pub struct OldGenerationSpace { // TODO: Add allocation count wrapper? heap: HeapAllocator, @@ -64,6 +78,10 @@ impl OldGenerationSpace { } pub unsafe fn sweep(&mut self, state: &CollectorState) { + self.free_live_objects(ObjectFreeCondition::Unmarked { state }); + } + + unsafe fn free_live_objects(&mut self, cond: ObjectFreeCondition<'_, Id>) { let mut next_index: u32 = 0; self.live_objects.get_mut().retain(|func| { if func.is_none() { @@ -72,34 +90,40 @@ impl OldGenerationSpace { let header = &mut *func.unwrap().as_ptr(); debug_assert_eq!(header.collector_id, self.collector_id); debug_assert_eq!(header.state_bits.get().generation(), GenerationId::Old); - let mark_bits = header.state_bits.get().raw_mark_bits().resolve(state); - match mark_bits { - GcMarkBits::White => { - // unmarked - if cfg!(debug_assertions) { - header.alloc_info.live_object_index = u32::MAX; + let should_free = match cond { + ObjectFreeCondition::Unmarked { state } => { + let mark_bits = header.state_bits.get().raw_mark_bits().resolve(state); + match mark_bits { + GcMarkBits::White => true, // should free + GcMarkBits::Black => false, // should not free } - let overall_layout = if header.state_bits.get().array() { - header.assume_array_header().layout_info().overall_layout() - } else { - header.metadata.type_info.layout.overall_layout() - }; - self.allocated_bytes.set( - self.allocated_bytes - .get() - .checked_sub(overall_layout.size()) - .expect("allocated size underflow"), - ); - self.heap - .deallocate(NonNull::from(header).cast(), overall_layout); - false } - GcMarkBits::Black => { - // marked - header.alloc_info.live_object_index = next_index; - next_index += 1; - true + ObjectFreeCondition::Always => true, // always free + }; + if should_free { + // unmarked (should free) + if cfg!(debug_assertions) { + header.alloc_info.live_object_index = u32::MAX; } + let overall_layout = if header.state_bits.get().array() { + header.assume_array_header().layout_info().overall_layout() + } else { + header.metadata.type_info.layout.overall_layout() + }; + self.allocated_bytes.set( + self.allocated_bytes + .get() + .checked_sub(overall_layout.size()) + .expect("allocated size underflow"), + ); + self.heap + .deallocate(NonNull::from(header).cast(), overall_layout); + false + } else { + // marked (should not free) + header.alloc_info.live_object_index = next_index; + next_index += 1; + true } }); assert_eq!(next_index as usize, self.live_objects.get_mut().len()); @@ -182,6 +206,15 @@ impl OldGenerationSpace { self.allocated_bytes.get() } } +impl Drop for OldGenerationSpace { + fn drop(&mut self) { + if DROP_NEEDS_EXPLICIT_FREE { + unsafe { + self.free_live_objects(ObjectFreeCondition::Always); + } + } + } +} #[derive(Debug, thiserror::Error)] pub enum OldAllocError { From f087b8914e466a5d1457ddb4fa5d7b8feda67f00 Mon Sep 17 00:00:00 2001 From: Techcable Date: Sun, 5 May 2024 18:36:04 -0700 Subject: [PATCH 19/35] Add lefthook.yml to check rust formatting --- lefthook.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 lefthook.yml diff --git a/lefthook.yml b/lefthook.yml new file mode 100644 index 0000000..84d13e9 --- /dev/null +++ b/lefthook.yml @@ -0,0 +1,11 @@ +skip_output: + - meta + - success + - summary +pre-commit: + commands: + rustfmt: + tags: formatter + glob: "*.rs" + run: cargo fmt --check -- {staged_files} + From 38a3b1e13e2f2a8258aa325378952900d697c3ab Mon Sep 17 00:00:00 2001 From: Techcable Date: Mon, 6 May 2024 08:56:17 -0700 Subject: [PATCH 20/35] Invoke destructors for dead objects This adds some extra overhead in young-gen, but not much in old --- src/context.rs | 31 ++++++++++++----- src/context/layout.rs | 76 ++++++++++++++++++++++++++++++++++------ src/context/old.rs | 7 ++++ src/context/young.rs | 81 +++++++++++++++++++++++++++++++++++++++---- src/lib.rs | 2 -- 5 files changed, 170 insertions(+), 27 deletions(-) diff --git a/src/context.rs b/src/context.rs index 716ffa2..d9454d0 100644 --- a/src/context.rs +++ b/src/context.rs @@ -247,7 +247,7 @@ impl GarbageCollector { failure_guard.defuse(); // now sweep unsafe { - self.young_generation.sweep(); + self.young_generation.sweep(&self.state); self.old_generation.sweep(&self.state); } // touch roots to verify validity @@ -326,6 +326,7 @@ unsafe trait RawAllocTarget { const ARRAY: bool; type Header: Sized; fn header_metadata(&self) -> HeaderMetadata; + fn needs_drop(&self) -> bool; unsafe fn init_header(&self, header_ptr: NonNull, base_header: GcHeader); fn overall_layout(&self) -> Layout; #[inline] @@ -356,6 +357,11 @@ unsafe impl RawAllocTarget for RegularAlloc<'_, Id> { } } + #[inline] + fn needs_drop(&self) -> bool { + self.type_info.drop_func.is_some() + } + #[inline] unsafe fn init_header(&self, header_ptr: NonNull>, base_header: GcHeader) { header_ptr.as_ptr().write(base_header) @@ -392,6 +398,11 @@ unsafe impl RawAllocTarget for ArrayAlloc<'_, Id> { } } + #[inline] + fn needs_drop(&self) -> bool { + self.type_info.element_type_info.drop_func.is_some() + } + #[inline] unsafe fn init_header( &self, @@ -568,6 +579,14 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { .as_ref() .update_state_bits(|bits| bits.with_forwarded(true)); (&mut *header_ptr.as_ptr()).metadata.forward_ptr = copied_ptr.cast(); + // determine if drop is needed from header_ptr, avoiding an indirection to type_info + let needs_drop = header_ptr.as_ref().alloc_info.nontrivial_drop_index < u32::MAX; + debug_assert_eq!(needs_drop, type_info.drop_func.is_some()); + if needs_drop { + self.garbage_collector + .young_generation + .remove_destruction_queue(header_ptr, &self.garbage_collector.state); + } // NOTE: Copy uninitialized bytes is safe here, as long as they are not read in dest if array { copied_ptr @@ -650,14 +669,8 @@ impl<'newgc, Id: CollectorId> CollectContext<'newgc, Id> { let type_info = header.as_ref().main_header.metadata.type_info; debug_assert_eq!(type_info.trace_func, Some(trace_func)); let array_header = header.cast::>(); - let element_layout = type_info.layout.value_layout(); - let len = array_header.as_ref().len_elements; - let element_start_ptr = array_header.as_ref().array_value_ptr(); - for i in 0..len { - let element = element_start_ptr - .as_ptr() - .add(i.unchecked_mul(element_layout.size())); - trace_func(NonNull::new_unchecked(element as *mut ()), self); + for element in array_header.as_ref().iter_elements() { + trace_func(element.cast::<()>(), self); } } } diff --git a/src/context/layout.rs b/src/context/layout.rs index 96e30b0..c20c73b 100644 --- a/src/context/layout.rs +++ b/src/context/layout.rs @@ -5,8 +5,11 @@ use bitbybit::{bitenum, bitfield}; use std::alloc::Layout; use std::cell::Cell; use std::fmt::{Debug, Formatter}; +use std::iter::FusedIterator; use std::marker::PhantomData; +use std::path::Iter; use std::ptr::NonNull; +use std::thread::current; /// The layout of a "regular" (non-array) type #[derive(Debug)] @@ -239,17 +242,13 @@ pub union HeaderMetadata { pub forward_ptr: NonNull>, } pub union AllocInfo { - /// The [overall size][`GcTypeLayout::overall_layout`] of this object. + /// The index of this object within the vector of objects which need to be dropped. /// - /// This is used to iterate over objects in the young generation. + /// If this object doesn't need to be dropped, + /// then this is `u32::MAX` /// - /// Objects whose size cannot fit into a `u32` - /// can never be allocated in the young generation. - /// - /// If this object is an array, - /// this is the overall size of - /// the header and all elements. - pub this_object_overall_size: u32, + /// This is used in the young generation. + pub nontrivial_drop_index: u32, /// The index of the object within the vector of live objects. /// /// This is used in the old generation. @@ -308,7 +307,7 @@ impl GcHeader { } #[inline] - fn resolve_type_info(&self) -> &'static GcTypeInfo { + pub fn resolve_type_info(&self) -> &'static GcTypeInfo { unsafe { if self.state_bits.get().forwarded() { let forward_ptr = self.metadata.forward_ptr; @@ -334,6 +333,13 @@ impl GcHeader { pub unsafe fn assume_array_header(&self) -> &'_ GcArrayHeader { &*(self as *const Self as *const GcArrayHeader) } + + #[inline] + pub unsafe fn invoke_destructor(&self) { + if let Some(drop_func) = self.resolve_type_info().drop_func { + drop_func(self.regular_value_ptr().as_ptr() as *mut ()); + } + } } #[repr(C, align(8))] @@ -388,7 +394,57 @@ impl GcArrayHeader { fn overall_layout(&self) -> Layout { self.layout_info().overall_layout() } + + #[inline] + pub unsafe fn iter_elements(&self) -> IterArrayElementPtr { + let len = self.len_elements; + IterArrayElementPtr { + element_size: self.element_layout().size(), + current_ptr: self.array_value_ptr(), + remaining_elements: len, + } + } + + pub unsafe fn invoke_destructor(&self) { + if let Some(drop_func) = self.resolve_type_info().element_type_info.drop_func { + for element in self.iter_elements() { + drop_func(element.as_ptr() as *mut ()); + } + } + } +} + +pub struct IterArrayElementPtr { + element_size: usize, + current_ptr: NonNull, + remaining_elements: usize, +} +impl Iterator for IterArrayElementPtr { + type Item = NonNull; + + #[inline] + fn next(&mut self) -> Option { + if self.remaining_elements > 0 { + let element_ptr = self.current_ptr; + unsafe { + self.current_ptr = + NonNull::new_unchecked(element_ptr.as_ptr().add(self.element_size)); + } + self.remaining_elements -= 1; + Some(element_ptr) + } else { + None + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (self.remaining_elements, Some(self.remaining_elements)) + } } +impl ExactSizeIterator for IterArrayElementPtr {} +impl FusedIterator for IterArrayElementPtr {} + pub struct GcArrayLayoutInfo { element_layout: Layout, len_elements: usize, diff --git a/src/context/old.rs b/src/context/old.rs index 71770ec..c1e52c7 100644 --- a/src/context/old.rs +++ b/src/context/old.rs @@ -116,6 +116,13 @@ impl OldGenerationSpace { .checked_sub(overall_layout.size()) .expect("allocated size underflow"), ); + // run destructors + if header.state_bits.get().array() { + header.assume_array_header().invoke_destructor(); + } else { + header.invoke_destructor(); + } + // deallocate memory self.heap .deallocate(NonNull::from(header).cast(), overall_layout); false diff --git a/src/context/young.rs b/src/context/young.rs index 8a93b6e..46c3059 100644 --- a/src/context/young.rs +++ b/src/context/young.rs @@ -1,16 +1,16 @@ use allocator_api2::alloc::{AllocError, Allocator}; use bumpalo::Bump; use std::alloc::Layout; -use std::cell::Cell; +use std::cell::{Cell, UnsafeCell}; use std::marker::PhantomData; use std::mem::ManuallyDrop; use std::ptr::NonNull; use crate::context::alloc::{ArenaAlloc, CountingAlloc}; -use crate::context::layout::{AllocInfo, GcHeader}; -use crate::context::GenerationId; +use crate::context::layout::{AllocInfo, GcHeader, GcMarkBits}; +use crate::context::{CollectorState, GenerationId}; use crate::utils::Alignment; -use crate::CollectorId; +use crate::{CollectorId, Gc}; struct YoungAlloc { #[cfg(feature = "debug-alloc")] @@ -70,6 +70,8 @@ unsafe impl Allocator for YoungAlloc { /// The design of the allocator is heavily based on [`bumpalo`](https://crates.io/crates/bumpalo) pub struct YoungGenerationSpace { alloc: CountingAlloc, + /// A set of objects which need destructors to be run. + destruction_queue: UnsafeCell>>>>, collector_id: Id, } impl YoungGenerationSpace { @@ -78,6 +80,7 @@ impl YoungGenerationSpace { let bump = ManuallyDrop::new(Box::new(Bump::new())); YoungGenerationSpace { alloc: CountingAlloc::new(YoungAlloc::new()), + destruction_queue: UnsafeCell::new(Vec::new()), collector_id: id, } } @@ -87,10 +90,58 @@ impl YoungGenerationSpace { /// Anything larger than this is immediately sent to the old generation. pub const SIZE_LIMIT: usize = 1024; - pub unsafe fn sweep(&mut self) { + pub unsafe fn sweep(&mut self, state: &CollectorState) { + for &element in self.destruction_queue.get_mut().iter() { + if let Some(header) = element { + debug_assert_eq!( + header + .as_ref() + .state_bits + .get() + .raw_mark_bits() + .resolve(state), + GcMarkBits::White, + "Only white objects should be in destruction queue" + ); + header.as_ref().invoke_destructor(); + } + } + self.destruction_queue.get_mut().clear(); self.alloc.as_inner_mut().reset(); } + #[inline] + pub unsafe fn remove_destruction_queue( + &self, + header: NonNull>, + state: &CollectorState, + ) { + debug_assert_ne!( + header + .as_ref() + .state_bits + .get() + .raw_mark_bits() + .resolve(state), + GcMarkBits::White, + "Only marked objects should be removed from the detruction queue" + ); + debug_assert_eq!( + header.as_ref().state_bits.get().generation(), + GenerationId::Young + ); + let drop_index = header.as_ref().alloc_info.nontrivial_drop_index; + if drop_index == u32::MAX { + debug_assert!(header.as_ref().resolve_type_info().drop_func.is_none()); + } else { + debug_assert!(header.as_ref().resolve_type_info().drop_func.is_some()); + (*self.destruction_queue.get())[drop_index as usize] = None; + if cfg!(debug_assertions) { + (*header.as_ptr()).alloc_info.nontrivial_drop_index = u32::MAX - 1; + } + } + } + #[inline] pub unsafe fn alloc_raw>( &self, @@ -104,12 +155,20 @@ impl YoungGenerationSpace { return Err(YoungAllocError::OutOfMemory); }; let header_ptr = raw_ptr.cast::(); + let drop_index = if target.needs_drop() { + let index = (*self.destruction_queue.get()).len(); + (*self.destruction_queue.get()).push(Some(header_ptr.cast::>())); + assert!(index < u32::MAX as usize); + index as u32 + } else { + u32::MAX + }; target.init_header( header_ptr, GcHeader { state_bits: Cell::new(target.init_state_bits(GenerationId::Young)), alloc_info: AllocInfo { - this_object_overall_size: overall_layout.size() as u32, + nontrivial_drop_index: drop_index, }, metadata: target.header_metadata(), collector_id: self.collector_id, @@ -123,6 +182,16 @@ impl YoungGenerationSpace { self.alloc.allocated_bytes() } } +impl Drop for YoungGenerationSpace { + fn drop(&mut self) { + // drop all pending objects + for header in self.destruction_queue.get_mut().iter() { + if let Some(header) = header { + unsafe { header.as_ref().invoke_destructor() } + } + } + } +} #[derive(Debug, thiserror::Error)] pub enum YoungAllocError { #[error("Out of memory (young-gen)")] diff --git a/src/lib.rs b/src/lib.rs index 5581014..1491bc2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,3 @@ -extern crate core; - pub mod collect; pub mod context; mod gcptr; From 8cae9b7a71c5836328315043b613866783c5e080 Mon Sep 17 00:00:00 2001 From: Techcable Date: Tue, 7 May 2024 17:05:35 -0700 Subject: [PATCH 21/35] Remove context and simple crates In the v0.3 API, the context crate will no longer exist and the simple crate needs to be rewritten. --- libs/context/Cargo.toml | 38 - libs/context/src/collector.rs | 546 ------- libs/context/src/handle.rs | 688 --------- libs/context/src/lib.rs | 275 ---- libs/context/src/state/mod.rs | 112 -- libs/context/src/state/nosync.rs | 232 --- libs/context/src/state/sync.rs | 667 --------- libs/context/src/utils.rs | 164 --- libs/simple/Cargo.toml | 69 - libs/simple/examples/binary_trees.rs | 97 -- libs/simple/examples/binary_trees_parallel.rs | 101 -- libs/simple/src/alloc.rs | 433 ------ libs/simple/src/layout.rs | 704 --------- libs/simple/src/lib.rs | 1282 ----------------- libs/simple/tests/arrays.rs | 105 -- libs/simple/tests/errors.rs | 57 - libs/simple/tests/trait_objects.rs | 87 -- 17 files changed, 5657 deletions(-) delete mode 100644 libs/context/Cargo.toml delete mode 100644 libs/context/src/collector.rs delete mode 100644 libs/context/src/handle.rs delete mode 100644 libs/context/src/lib.rs delete mode 100644 libs/context/src/state/mod.rs delete mode 100644 libs/context/src/state/nosync.rs delete mode 100644 libs/context/src/state/sync.rs delete mode 100644 libs/context/src/utils.rs delete mode 100644 libs/simple/Cargo.toml delete mode 100644 libs/simple/examples/binary_trees.rs delete mode 100644 libs/simple/examples/binary_trees_parallel.rs delete mode 100644 libs/simple/src/alloc.rs delete mode 100644 libs/simple/src/layout.rs delete mode 100644 libs/simple/src/lib.rs delete mode 100644 libs/simple/tests/arrays.rs delete mode 100644 libs/simple/tests/errors.rs delete mode 100644 libs/simple/tests/trait_objects.rs diff --git a/libs/context/Cargo.toml b/libs/context/Cargo.toml deleted file mode 100644 index aee89a8..0000000 --- a/libs/context/Cargo.toml +++ /dev/null @@ -1,38 +0,0 @@ -[package] -name = "zerogc-context" -description = "Handles the context of a zerogc collector." -version.workspace = true -authors.workspace = true -repository.workspace = true -edition.workspace = true -readme = "../../README.md" -license = "MIT" - -[dependencies] -zerogc = { path = "../..", version = "0.2.0-alpha.6" } -zerogc-derive = { path = "../derive", version = "0.2.0-alpha.6" } -once_cell = { version = "1.5", optional = true } -# Concurrency -parking_lot = { version = "0.11", optional = true } -crossbeam-utils = { version = "0.8", optional = true } -# Logging -slog = "2.7" - -[features] -default = [ - "sync", # Support thread-safety by default - "std" -] -# Use the standard library (required for `sync`) -std = [] -# This will allow multiple threads to access the garbage collector -# by creating a separate context for each. -# -# Thread safe collectors can have increased overhead -# by requiring communication between threads. -sync = [ - "parking_lot", - "crossbeam-utils", - "std" -] - diff --git a/libs/context/src/collector.rs b/libs/context/src/collector.rs deleted file mode 100644 index 28f6eef..0000000 --- a/libs/context/src/collector.rs +++ /dev/null @@ -1,546 +0,0 @@ -//! The interface to a collector -#![allow(clippy::missing_safety_doc)] - -use core::fmt::{self, Debug, Formatter}; -use core::hash::{Hash, Hasher}; -use core::marker::PhantomData; -use core::ptr::NonNull; - -use alloc::sync::Arc; - -use slog::{o, Logger}; - -use zerogc::{Gc, GcArray, GcSafe, GcSimpleAlloc, GcSystem, Trace}; - -use crate::state::{CollectionManager, RawContext}; -use crate::CollectorContext; -use zerogc::vec::raw::GcRawVec; - -pub unsafe trait ConstRawCollectorImpl: RawCollectorImpl { - fn resolve_array_len_const(gc: &GcArray>) -> usize; -} - -/// A specific implementation of a collector -pub unsafe trait RawCollectorImpl: 'static + Sized { - /// A dynamic pointer to a `Trace` root - /// - /// The simple collector implements this as - /// a trait object pointer. - type DynTracePtr: Copy + Debug + 'static; - /// The configuration - type Config: Sized + Default; - - /// A pointer to this collector - /// - /// Must be a ZST if the collector is a singleton. - type Ptr: CollectorPtr; - - /// The type that manages this collector's state - type Manager: CollectionManager; - - /// The context - type RawContext: RawContext; - /// The raw representation of a vec - type RawVec<'gc, T: GcSafe<'gc, CollectorId>>: GcRawVec<'gc, T, Id = CollectorId>; - - /// True if this collector is a singleton - /// - /// If the collector allows multiple instances, - /// this *must* be false - const SINGLETON: bool; - - /// True if this collector is thread-safe. - const SYNC: bool; - - fn id_for_gc<'a, 'gc, T>(gc: &'a Gc<'gc, T, CollectorId>) -> &'a CollectorId - where - 'gc: 'a, - T: ?Sized + 'gc; - - // TODO: What if we want to customize 'GcArrayRepr'?? - - fn id_for_array<'a, 'gc, T>( - gc: &'a GcArray<'gc, T, CollectorId>, - ) -> &'a CollectorId - where - 'gc: 'a; - - fn resolve_array_len(repr: &GcArray>) -> usize; - - /// Convert the specified value into a dyn pointer - unsafe fn as_dyn_trace_pointer(t: *mut T) -> Self::DynTracePtr; - - /// Initialize an instance of the collector - /// - /// Must panic if the collector is not a singleton - fn init(config: Self::Config, logger: Logger) -> NonNull; - - /// The id of this collector - #[inline] - fn id(&self) -> CollectorId { - CollectorId { - ptr: unsafe { Self::Ptr::from_raw(self as *const _ as *mut _) }, - } - } - unsafe fn gc_write_barrier<'gc, O, V>( - owner: &Gc<'gc, O, CollectorId>, - value: &Gc<'gc, V, CollectorId>, - field_offset: usize, - ) where - O: GcSafe<'gc, CollectorId> + ?Sized, - V: GcSafe<'gc, CollectorId> + ?Sized; - /// The logger associated with this collector - fn logger(&self) -> &Logger; - - fn manager(&self) -> &Self::Manager; - - fn should_collect(&self) -> bool; - - fn allocated_size(&self) -> crate::utils::MemorySize; - - unsafe fn perform_raw_collection(&self, contexts: &[*mut Self::RawContext]); -} - -/// A thread safe collector -pub unsafe trait SyncCollector: RawCollectorImpl + Sync {} - -/// A collector implemented as a singleton -/// -/// This only has one instance -pub unsafe trait SingletonCollector: - RawCollectorImpl> -{ - /// When the collector is a singleton, - /// return the global implementation - fn global_ptr() -> *const Self; - - /// Initialize the global singleton - /// - /// Panics if already initialized - fn init_global(config: Self::Config, logger: Logger); -} - -impl PartialEq for CollectorId { - #[inline] - fn eq(&self, other: &Self) -> bool { - self.ptr == other.ptr - } -} -impl Hash for CollectorId { - #[inline] - fn hash(&self, hasher: &mut H) { - hasher.write_usize(self.ptr.as_ptr() as usize); - } -} -impl Eq for CollectorId {} -impl Clone for CollectorId { - #[inline] - fn clone(&self) -> Self { - *self - } -} -impl Copy for CollectorId {} -impl Debug for CollectorId { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - let mut debug = f.debug_struct("CollectorId"); - if !C::SINGLETON { - debug.field("ptr", &format_args!("{:p}", self.ptr.as_ptr())); - } - debug.finish() - } -} - -/// An unchecked pointer to a collector -pub unsafe trait CollectorPtr>: - Copy + Eq + self::sealed::Sealed + 'static -{ - /// A weak reference to the pointer - type Weak: Clone + 'static; - - unsafe fn from_raw(ptr: *mut C) -> Self; - unsafe fn clone_owned(&self) -> Self; - fn as_ptr(&self) -> *mut C; - unsafe fn drop(self); - fn upgrade_weak_raw(weak: &Self::Weak) -> Option; - #[inline] - fn upgrade_weak(weak: &Self::Weak) -> Option> { - Self::upgrade_weak_raw(weak).map(|ptr| CollectorRef { ptr }) - } - unsafe fn assume_weak_valid(weak: &Self::Weak) -> Self; - unsafe fn create_weak(&self) -> Self::Weak; -} -/// This is implemented as a -/// raw pointer via [Arc::into_raw] -unsafe impl> CollectorPtr for NonNull { - type Weak = alloc::sync::Weak; - - #[inline] - unsafe fn from_raw(ptr: *mut C) -> Self { - assert!(!C::SINGLETON, "Collector is a singleton!"); - debug_assert!(!ptr.is_null()); - NonNull::new_unchecked(ptr) - } - - #[inline] - unsafe fn clone_owned(&self) -> Self { - let original = Arc::from_raw(self.as_ptr()); - let cloned = Arc::clone(&original); - core::mem::forget(original); - NonNull::new_unchecked(Arc::into_raw(cloned) as *mut _) - } - - #[inline] - fn as_ptr(&self) -> *mut C { - NonNull::as_ptr(*self) - } - - #[inline] - unsafe fn drop(self) { - drop(Arc::from_raw(self.as_ptr() as *const _)) - } - - #[inline] - fn upgrade_weak_raw(weak: &Self::Weak) -> Option { - weak.upgrade() - .map(|arc| unsafe { Self::from_raw(Arc::into_raw(arc) as *mut _) }) - } - - #[inline] - unsafe fn assume_weak_valid(weak: &Self::Weak) -> Self { - debug_assert!(weak.upgrade().is_some(), "Dead collector"); - NonNull::new_unchecked(weak.as_ptr() as *mut _) - } - - #[inline] - unsafe fn create_weak(&self) -> Self::Weak { - let arc = Arc::from_raw(self.as_ptr()); - let weak = Arc::downgrade(&arc); - core::mem::forget(arc); - weak - } -} -/// Dummy implementation -impl self::sealed::Sealed for NonNull {} -unsafe impl> CollectorPtr for PhantomData<&'static C> { - type Weak = PhantomData<&'static C>; - - #[inline] - unsafe fn from_raw(ptr: *mut C) -> Self { - assert!(C::SINGLETON, "Expected a singleton"); - debug_assert_eq!(ptr, C::global_ptr() as *mut _); - PhantomData - } - - #[inline] - unsafe fn clone_owned(&self) -> Self { - *self - } - - #[inline] - fn as_ptr(&self) -> *mut C { - assert!(C::SINGLETON, "Expected a singleton"); - C::global_ptr() as *mut C - } - - #[inline] - unsafe fn drop(self) {} - - #[inline] - fn upgrade_weak_raw(weak: &Self::Weak) -> Option { - assert!(C::SINGLETON); - Some(*weak) // gloal is always valid - } - - #[inline] - unsafe fn assume_weak_valid(weak: &Self::Weak) -> Self { - assert!(C::SINGLETON); // global is always valid - *weak - } - - #[inline] - unsafe fn create_weak(&self) -> Self::Weak { - *self - } -} -/// Dummy implementation -impl self::sealed::Sealed for PhantomData<&'static C> {} - -/// Uniquely identifies the collector in case there are -/// multiple collectors. -/// -/// If there are multiple collectors `cfg!(feature="multiple-collectors")`, -/// we need to use a pointer to tell them apart. -/// Otherwise, this is a zero-sized structure. -/// -/// As long as our memory is valid, -/// it implies this pointer is too. -#[repr(C)] -pub struct CollectorId { - /// This is in essence a borrowed reference to - /// the collector. - /// - /// Depending on whether or not the collector is a singleton, - /// - /// We don't know whether the underlying memory will be valid. - ptr: C::Ptr, -} -impl CollectorId { - #[inline] - pub const unsafe fn from_raw(ptr: C::Ptr) -> CollectorId { - CollectorId { ptr } - } - #[inline] - pub unsafe fn as_ref(&self) -> &C { - &*self.ptr.as_ptr() - } - #[inline] - pub unsafe fn weak_ref(&self) -> WeakCollectorRef { - WeakCollectorRef { - weak: self.ptr.create_weak(), - } - } -} -unsafe impl Sync for CollectorId {} -unsafe impl Send for CollectorId {} -unsafe impl ::zerogc::CollectorId for CollectorId { - type System = CollectorRef; - type Context = CollectorContext; - type RawVec<'gc, T: GcSafe<'gc, Self>> = C::RawVec<'gc, T>; - // TODO: What if clients want to customize this? - type ArrayPtr = zerogc::array::repr::ThinArrayPtr; - - #[inline] - fn from_gc_ptr<'a, 'gc, T>(gc: &'a Gc<'gc, T, Self>) -> &'a Self - where - T: ?Sized, - 'gc: 'a, - { - C::id_for_gc(gc) - } - - #[inline] - fn resolve_array_id<'a, 'gc, T>(gc: &'a GcArray<'gc, T, Self>) -> &'a Self - where - 'gc: 'a, - { - C::id_for_array(gc) - } - - #[inline] - fn resolve_array_len(repr: &GcArray<'_, T, Self>) -> usize { - C::resolve_array_len(repr) - } - - #[inline(always)] - unsafe fn gc_write_barrier<'gc, O, V>( - owner: &Gc<'gc, O, Self>, - value: &Gc<'gc, V, Self>, - field_offset: usize, - ) where - O: GcSafe<'gc, Self> + ?Sized, - V: GcSafe<'gc, Self> + ?Sized, - { - C::gc_write_barrier(owner, value, field_offset) - } - - #[inline] - unsafe fn assume_valid_system(&self) -> &Self::System { - // TODO: Make the API nicer? (avoid borrowing and indirection) - assert_eq!( - core::mem::size_of::(), - core::mem::size_of::>() - ); - &*(self as *const CollectorId as *const CollectorRef) - } -} -zerogc::impl_nulltrace_for_static!(CollectorId, params => [C: RawCollectorImpl]); - -pub struct WeakCollectorRef { - weak: >::Weak, -} -impl WeakCollectorRef { - #[inline] - pub unsafe fn assume_valid(&self) -> CollectorId { - CollectorId { - ptr: C::Ptr::assume_weak_valid(&self.weak), - } - } - pub fn ensure_valid(&self, func: impl FnOnce(CollectorId) -> R) -> R { - self.try_ensure_valid(|id| match id { - Some(id) => func(id), - None => panic!("Dead collector"), - }) - } - #[inline] - pub fn try_ensure_valid(&self, func: impl FnOnce(Option>) -> R) -> R { - func(C::Ptr::upgrade_weak(&self.weak).map(|r| r.id())) - } -} - -pub unsafe trait RawSimpleAlloc: RawCollectorImpl { - unsafe fn alloc_uninit<'gc, T: GcSafe<'gc, CollectorId>>( - context: &'gc CollectorContext, - ) -> *mut T; - unsafe fn alloc_uninit_slice<'gc, T>( - context: &'gc CollectorContext, - len: usize, - ) -> *mut T - where - T: GcSafe<'gc, CollectorId>; - fn alloc_raw_vec_with_capacity<'gc, T>( - context: &'gc CollectorContext, - capacity: usize, - ) -> Self::RawVec<'gc, T> - where - T: GcSafe<'gc, CollectorId>; -} -unsafe impl GcSimpleAlloc for CollectorContext -where - C: RawSimpleAlloc, -{ - #[inline] - unsafe fn alloc_uninit<'gc, T>(&'gc self) -> *mut T - where - T: GcSafe<'gc, CollectorId>, - { - C::alloc_uninit(self) - } - - #[inline] - unsafe fn alloc_uninit_slice<'gc, T>(&'gc self, len: usize) -> *mut T - where - T: GcSafe<'gc, CollectorId>, - { - C::alloc_uninit_slice(self, len) - } - - #[inline] - fn alloc_raw_vec_with_capacity<'gc, T>(&'gc self, capacity: usize) -> C::RawVec<'gc, T> - where - T: GcSafe<'gc, CollectorId>, - { - C::alloc_raw_vec_with_capacity::(self, capacity) - } -} - -/// A reference to the collector. -/// -/// TODO: Devise better name -#[repr(C)] -pub struct CollectorRef { - /// When using singleton collectors, this is a ZST. - /// - /// When using multiple collectors, this is just an [Arc]. - /// - /// It is implemented as a raw pointer around [Arc::into_raw] - ptr: C::Ptr, -} -/// We actually are thread safe ;) -unsafe impl Send for CollectorRef {} -#[cfg(feature = "sync")] -unsafe impl Sync for CollectorRef {} - -/// Internal trait for initializing a collector -#[doc(hidden)] -pub trait CollectorInit>: CollectorPtr { - fn create() -> CollectorRef { - Self::with_logger(C::Config::default(), Logger::root(slog::Discard, o!())) - } - fn with_logger(config: C::Config, logger: Logger) -> CollectorRef; -} - -impl>> CollectorInit for NonNull { - fn with_logger(config: C::Config, logger: Logger) -> CollectorRef { - assert!(!C::SINGLETON); - let raw_ptr = C::init(config, logger); - CollectorRef { ptr: raw_ptr } - } -} -impl CollectorInit for PhantomData<&'static C> -where - C: SingletonCollector, -{ - fn with_logger(config: C::Config, logger: Logger) -> CollectorRef { - assert!(C::SINGLETON); - C::init_global(config, logger); // TODO: Is this safe? - // NOTE: The raw pointer is implicit (now that we're leaked) - CollectorRef { ptr: PhantomData } - } -} - -impl CollectorRef { - #[inline] - pub fn create() -> Self - where - C::Ptr: CollectorInit, - { - >::create() - } - - #[inline] - pub fn with_logger(logger: Logger) -> Self - where - C::Ptr: CollectorInit, - { - Self::with_config(C::Config::default(), logger) - } - - pub fn with_config(config: C::Config, logger: Logger) -> Self - where - C::Ptr: CollectorInit, - { - >::with_logger(config, logger) - } - - #[inline] - pub(crate) fn clone_internal(&self) -> CollectorRef { - CollectorRef { - ptr: unsafe { self.ptr.clone_owned() }, - } - } - - #[inline] - pub fn as_raw(&self) -> &C { - unsafe { &*self.ptr.as_ptr() } - } - - /// The id of this collector - #[inline] - pub fn id(&self) -> CollectorId { - CollectorId { ptr: self.ptr } - } - - /// Convert this collector into a unique context - /// - /// The single-threaded implementation only allows a single context, - /// so this method is nessicary to support it. - pub fn into_context(self) -> CollectorContext { - unsafe { CollectorContext::register_root(&self) } - } -} -impl CollectorRef { - /// Create a new context bound to this collector - /// - /// Warning: Only one collector should be created per thread. - /// Doing otherwise can cause deadlocks/panics. - pub fn create_context(&self) -> CollectorContext { - unsafe { CollectorContext::register_root(self) } - } -} -impl Drop for CollectorRef { - #[inline] - fn drop(&mut self) { - unsafe { - self.ptr.drop(); - } - } -} - -unsafe impl GcSystem for CollectorRef { - type Id = CollectorId; - type Context = CollectorContext; -} - -mod sealed { - pub trait Sealed {} -} diff --git a/libs/context/src/handle.rs b/libs/context/src/handle.rs deleted file mode 100644 index 06c656f..0000000 --- a/libs/context/src/handle.rs +++ /dev/null @@ -1,688 +0,0 @@ -//! Implementation of [::zerogc::GcHandle] -//! -//! Inspired by [Mono's Lock free Gc Handles](https://www.mono-project.com/news/2016/08/16/lock-free-gc-handles/) -use core::marker::PhantomData; -use core::mem::ManuallyDrop; -use core::ptr::{self, NonNull, Pointee}; -use core::sync::atomic::{self, AtomicPtr, AtomicUsize, Ordering}; - -use alloc::boxed::Box; -use alloc::vec::Vec; - -use crate::collector::RawCollectorImpl; -use crate::{CollectionManager, CollectorId, CollectorRef, Gc, WeakCollectorRef}; -use zerogc::{ - GcRebrand, GcSafe, GcVisitor, HandleCollectorId, NullTrace, Trace, TraceImmutable, TrustedDrop, -}; - -const INITIAL_HANDLE_CAPACITY: usize = 64; - -/// A [RawCollectorImpl] that supports handles -pub unsafe trait RawHandleImpl: RawCollectorImpl { - /// Type information - type TypeInfo: Sized; - - fn type_info_of<'gc, T: GcSafe<'gc, CollectorId>>() -> &'static Self::TypeInfo; - - fn resolve_type_info<'gc, T: ?Sized + GcSafe<'gc, CollectorId>>( - gc: Gc<'gc, T, CollectorId>, - ) -> &'static Self::TypeInfo; - - fn handle_list(&self) -> &GcHandleList; -} - -/// Concurrent list of [GcHandle]s -/// -/// Each bucket in the linked list is twice the size of -/// the previous one, ensuring amortized growth -/// -/// The list can not be appended to while a collection -/// is in progress. -/// -/// TODO: This list only grows. It never shrinks!! -pub struct GcHandleList { - last_bucket: AtomicPtr>, - /// Pointer to the last free slot in the free list - /// - /// If the list is empty, this is null - last_free_slot: AtomicPtr>, -} -#[allow(clippy::new_without_default)] -impl GcHandleList { - pub fn new() -> Self { - use core::ptr::null_mut; - GcHandleList { - last_bucket: AtomicPtr::new(null_mut()), - last_free_slot: AtomicPtr::new(null_mut()), - } - } - /// Append the specified slot to this list - /// - /// The specified slot must be logically owned - /// and not already part of this list - unsafe fn append_free_slot(&self, slot: *mut HandleSlot) { - // Verify it's actually free... - debug_assert_eq!((*slot).valid.value.load(Ordering::SeqCst), ptr::null_mut()); - let mut last_free = self.last_free_slot.load(Ordering::Acquire); - loop { - /* - * NOTE: Must update `prev_freed_slot` - * BFEORE we write to the free-list. - * Other threads must see a consistent state - * the moment we're present in the free-list - */ - (*slot) - .freed - .prev_free_slot - .store(last_free, Ordering::Release); - /* - * We really dont want surprise failures because we're going to - * have to redo the above store if that happens. - * Likewise we want acquire ordering so we don't fail unnecessarily - * on retry. - * In theory this is premature optimization, but I really want to - * make this as straightforward as possible. - * Maybe we should look into the efficiency of this on ARM? - */ - match self.last_free_slot.compare_exchange( - last_free, - slot, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(actual) => { - debug_assert_eq!(actual, last_free); - return; // Success - } - Err(actual) => { - last_free = actual; - } - } - } - } - /// Allocate a raw handle that points to the given value. - /// - /// ## Safety - /// A collection may not currently be in progress. - /// - /// The returned handle must be fully initialized - /// before the next collection begins. - #[inline] - pub(crate) unsafe fn alloc_raw_handle(&self, value: *mut ()) -> &GcRawHandle { - // TODO: Should we weaken these orderings? - let mut slot = self.last_free_slot.load(Ordering::Acquire); - while !slot.is_null() { - /* - * NOTE: If another thread has raced us and already initialized - * this free slot, this pointer could be nonsense. - * That's okay - it's still initialized memory. - */ - let prev = (*slot).freed.prev_free_slot.load(Ordering::Acquire); - /* - * If this CAS succeeds, we have ownership. - * Otherwise another thread beat us and - * we must try again. - * - * Avoid relaxed ordering and compare_exchange_weak - * to make this straightforward. - */ - match self.last_free_slot.compare_exchange( - slot, - prev, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(actual_slot) => { - debug_assert_eq!(actual_slot, slot); - // Verify it's actually free... - debug_assert_eq!((*slot).valid.value.load(Ordering::SeqCst), ptr::null_mut()); - /* - * We own the slot, initialize it to point to - * the provided pointer. The user is responsible - * for any remaining initialization. - */ - (*slot).valid.value.store(value, Ordering::Release); - return &(*slot).valid; - } - Err(actual_slot) => { - // Try again - slot = actual_slot; - } - } - } - // Empty free list - self.alloc_handle_fallback(value) - } - /// Fallback to creating more buckets - /// if the free-list is empty - #[cold] - #[inline(never)] - unsafe fn alloc_handle_fallback(&self, value: *mut ()) -> &GcRawHandle { - let mut bucket = self.last_bucket.load(Ordering::Acquire); - loop { - // TODO: Should we be retrying the free-list? - let new_size: usize; - if bucket.is_null() { - new_size = INITIAL_HANDLE_CAPACITY; - } else { - if let Some(slot) = (*bucket).blindly_alloc_slot(value) { - /* - * NOTE: The caller is responsible - * for finishing up initializing this. - * We've merely set the pointer to `value` - */ - return &slot.valid; - } - // Double capacity for amortized growth - new_size = (*bucket).slots.len() * 2; - } - match self.init_bucket(bucket, new_size) { - /* - * We honestly don't care what thread - * allocated the new bucket. We just want - * to use it */ - Ok(new_bucket) | Err(new_bucket) => { - bucket = new_bucket as *const _ as *mut _; - } - } - } - } - unsafe fn init_bucket( - &self, - prev_bucket: *mut GcHandleBucket, - desired_size: usize, - ) -> Result<&GcHandleBucket, &GcHandleBucket> { - let mut slots: Vec> = Vec::with_capacity(desired_size); - // Zero initialize slots - assuming this is safe - slots.as_mut_ptr().write_bytes(0, desired_size); - slots.set_len(desired_size); - let allocated_bucket = Box::into_raw(Box::new(GcHandleBucket { - slots: slots.into_boxed_slice(), - last_alloc: AtomicUsize::new(0), - prev: AtomicPtr::new(prev_bucket), - })); - match self.last_bucket.compare_exchange( - prev_bucket, - allocated_bucket, - Ordering::SeqCst, - Ordering::SeqCst, - ) { - Ok(actual_bucket) => { - assert_eq!(actual_bucket, prev_bucket); - Ok(&*actual_bucket) - } - Err(actual_bucket) => { - /* - * Someone else beat us to creating the bucket. - * - * Free the bucket we've created and return - * their bucket - */ - drop(Box::from_raw(allocated_bucket)); - Err(&*actual_bucket) - } - } - } - /// Trace the [GcHandle] using the specified closure. - /// - /// ## Safety - /// Assumes the visitor function is well behaved. - /// - /// TODO: Can the 'unsafe' be removed? - /// I think we were only using it for 'trace_inner' - /// because it assumes the fences were already applied. - /// Now that's behind a layer of abstraction, - /// the unsafety has technically been moved to the caller. - pub unsafe fn trace(&mut self, mut visitor: F) -> Result<(), E> - where - F: FnMut(*mut (), &C::TypeInfo) -> Result<(), E>, - { - /* - * TODO: This fence seems unnecessary since we should - * already have exclusive access..... - */ - atomic::fence(Ordering::Acquire); - let mut bucket = self.last_bucket.load(Ordering::Relaxed); - while !bucket.is_null() { - // We should have exclusive access! - let slots = &mut *(*bucket).slots; - for slot in slots { - if slot.is_valid(Ordering::Relaxed) { - slot.valid.trace_inner(&mut visitor)?; - } - } - bucket = (*bucket).prev.load(Ordering::Relaxed); - } - /* - * Release any pending writes (for relocated pointers) - * TODO: We should have exclusive access, like above! - */ - atomic::fence(Ordering::Release); - Ok(()) - } -} -impl Drop for GcHandleList { - fn drop(&mut self) { - let mut bucket = self.last_bucket.load(Ordering::Acquire); - while !bucket.is_null() { - unsafe { - drop(Box::from_raw(bucket)); - bucket = (*bucket).prev.load(Ordering::Acquire); - } - } - } -} -struct GcHandleBucket { - slots: Box<[HandleSlot]>, - /// A pointer to the last allocated slot - /// - /// This doesn't take into account freed slots, - /// so should only be used as a fallback if - /// the free-list is empty - last_alloc: AtomicUsize, - /// Pointer to the last bucket in - /// the linked list - /// - /// This must be freed manually. - /// Dropping this bucket doesn't drop the - /// last one. - prev: AtomicPtr>, -} -impl GcHandleBucket { - /// Acquire a new raw handle from this bucket, - /// or `None` if the bucket is believed to be empty - /// - /// This **doesn't reused freed values**, so should - /// only be used as a fallback if the free-list is empty. - /// - /// ## Safety - /// See docs on [GcHandleList::alloc_raw_bucket] - unsafe fn blindly_alloc_slot(&self, value: *mut ()) -> Option<&HandleSlot> { - let last_alloc = self.last_alloc.load(Ordering::Relaxed); - for (i, slot) in self.slots.iter().enumerate().skip(last_alloc) { - // TODO: All these fences must be horrible on ARM - if slot - .valid - .value - .compare_exchange(ptr::null_mut(), value, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() - { - // We acquired ownership! - self.last_alloc.fetch_max(i, Ordering::AcqRel); - return Some(&*slot); - } - } - None - } -} -/// A slot in the array of handles -/// -/// This may or may not be valid, -/// depending on whether the value pointer is null. -#[repr(C)] -pub union HandleSlot { - freed: ManuallyDrop>, - valid: ManuallyDrop>, -} -impl HandleSlot { - /// Load the current value of this pointer - #[inline] - fn is_valid(&self, ord: Ordering) -> bool { - /* - * Check if the value pointer is null - * - * The pointer is present in both variants - * of the enum. - */ - unsafe { !self.valid.value.load(ord).is_null() } - } -} - -/// A handle that is free -#[repr(C)] -pub struct FreedHandleSlot { - /// This corresponds to a [GcRawHandle::value] - /// - /// It must be null for this object to be truly invalid! - _invalid_value: AtomicPtr<()>, - /// The previous slot in the free list - prev_free_slot: AtomicPtr>, -} - -/// The underlying value of a handle. -/// -/// These are reused -#[repr(C)] -pub struct GcRawHandle { - /// Refers to the underlying value of this handle. - /// - /// If it's null, it's invalid, and is actually - /// a freed handle - /// - /// The underlying value can only be safely accessed - /// if there isn't a collection in progress - value: AtomicPtr<()>, - /// I think this should be protected by the other atomic - /// accesses. Regardless, I'll put it in an AtomicPtr anyways. - // TODO: Encapsulate - pub(crate) type_info: AtomicPtr, - /// The reference count to the handle - /// - /// If this is zero the value can be freed - /// and this memory can be used. - // TODO: Encapsulate - pub(crate) refcnt: AtomicUsize, -} -impl GcRawHandle { - /// Trace this handle, assuming collection is in progress - /// - /// ## Safety - /// - Trace function must be reasonable - /// - A collection must currently be in progress - /// - It is assumed that the appropriate atomic fences (if any) - /// have already been applied (TODO: Don't we have exclusive access?) - unsafe fn trace_inner(&self, trace: &mut F) -> Result<(), E> - where - F: FnMut(*mut (), &C::TypeInfo) -> Result<(), E>, - { - let value = self.value.load(Ordering::Relaxed); - if value.is_null() { - debug_assert_eq!(self.refcnt.load(Ordering::Relaxed), 0); - return Ok(()); // Nothing to trace - } - debug_assert_ne!(self.refcnt.load(Ordering::Relaxed), 0); - let type_info = &*self.type_info.load(Ordering::Relaxed); - trace(value, type_info) - } -} -pub struct GcHandle>, C: RawHandleImpl> { - inner: NonNull>, - collector: WeakCollectorRef, - /// The pointer metadata for the type. - /// - /// Assumed to be immutable - /// and not change - /// SAFETY: - /// 1. slices - Length never changes - /// 2. dyn pointers - Never needs - metadata: ::Metadata, - marker: PhantomData<*mut T>, -} -impl>, C: RawHandleImpl> GcHandle { - #[inline] - pub(crate) unsafe fn new( - inner: NonNull>, - collector: WeakCollectorRef, - metadata: ::Metadata, - ) -> Self { - GcHandle { - inner, - collector, - metadata, - marker: PhantomData, - } - } - #[inline] - unsafe fn assume_valid(&self) -> *mut T { - ptr::from_raw_parts_mut( - self.inner.as_ref().value.load(Ordering::Acquire) as *mut (), - self.metadata, - ) - } -} -unsafe impl>, C: RawHandleImpl> ::zerogc::GcHandle - for GcHandle -{ - type System = CollectorRef; - type Id = CollectorId; - - fn use_critical(&self, func: impl FnOnce(&T) -> R) -> R { - self.collector.ensure_valid(|collector| unsafe { - /* - * This should be sufficient to ensure - * the value won't be collected or relocated. - * - * Note that this is implemented using a read lock, - * so recursive calls will deadlock. - * This is preferable to using `recursive_read`, - * since that could starve writers (collectors). - */ - C::Manager::prevent_collection(collector.as_ref(), || { - let value = self.assume_valid(); - func(&*value) - }) - }) - } - #[inline] - fn bind_to<'new_gc>( - &self, - context: &'new_gc ::Context, - ) -> Gc<'new_gc, >::Branded, Self::Id> - where - T: GcRebrand<'new_gc, Self::Id>, - { - /* - * We can safely assume the object will - * be as valid as long as the context. - * By binding it to the lifetime of the context, - * we ensure that the user will have to properly - * track it with safepoints from now on. - * - * Instead of dynamically tracking the root - * with runtime-overhead, - * its tracked like any other object normally - * allocated from a context. It's zero-cost - * from now until the next safepoint. - */ - unsafe { - let collector = self.collector.assume_valid(); - assert_eq!( - collector.as_ref() as *const C, - context.collector() as *const C, - "Collectors mismatch" - ); - /* - * NOTE: Can't use regular pointer-cast - * because of potentially mismatched vtables. - */ - let value = - crate::utils::transmute_mismatched::<*mut T, *mut T::Branded>(self.assume_valid()); - debug_assert!(!value.is_null()); - Gc::from_raw(NonNull::new_unchecked(value)) - } - } -} -unsafe impl>, C: RawHandleImpl> Trace - for GcHandle -{ - /// See docs on reachability - const NEEDS_TRACE: bool = false; - const NEEDS_DROP: bool = true; - #[inline(always)] - fn trace(&mut self, _visitor: &mut V) -> Result<(), V::Err> - where - V: zerogc::GcVisitor, - { - Ok(()) - } -} -unsafe impl>, C: RawHandleImpl> TraceImmutable - for GcHandle -{ - #[inline(always)] - fn trace_immutable(&self, _visitor: &mut V) -> Result<(), V::Err> - where - V: GcVisitor, - { - Ok(()) - } -} -unsafe impl>, C: RawHandleImpl> NullTrace - for GcHandle -{ -} -unsafe impl<'gc, T: ?Sized + GcSafe<'static, CollectorId>, C: RawHandleImpl> - GcSafe<'gc, CollectorId> for GcHandle -{ - #[inline] - unsafe fn trace_inside_gc( - gc: &mut Gc<'gc, Self, CollectorId>, - visitor: &mut V, - ) -> Result<(), V::Err> - where - V: GcVisitor, - { - // Fine to stuff inside a pointer. We're a `Sized` type - visitor.trace_gc(gc) - } -} -unsafe impl>, C: RawHandleImpl> TrustedDrop - for GcHandle -{ -} -impl>, C: RawHandleImpl> Clone for GcHandle { - fn clone(&self) -> Self { - // NOTE: Dead collector -> invalid handle - let collector = self.collector.ensure_valid(|id| unsafe { id.weak_ref() }); - let inner = unsafe { self.inner.as_ref() }; - debug_assert!( - !inner.value.load(Ordering::SeqCst).is_null(), - "Pointer is invalid" - ); - let mut old_refcnt = inner.refcnt.load(Ordering::Relaxed); - loop { - assert_ne!( - old_refcnt, - isize::max_value() as usize, - "Reference count overflow" - ); - /* - * NOTE: Relaxed is sufficient for failure since we have no - * expectations about the new state. Weak exchange is okay - * since we retry in a loop. - * - * NOTE: We do **not** use fetch_add because we are afraid - * of refcount overflow. We should possibly consider it - */ - match inner.refcnt.compare_exchange_weak( - old_refcnt, - old_refcnt + 1, - Ordering::AcqRel, - Ordering::Relaxed, - ) { - Ok(_) => break, - Err(val) => { - old_refcnt = val; - } - } - } - GcHandle { - inner: self.inner, - metadata: self.metadata, - collector, - marker: PhantomData, - } - } -} -impl>, C: RawHandleImpl> Drop for GcHandle { - fn drop(&mut self) { - self.collector.try_ensure_valid(|id| { - let collector = match id { - None => { - /* - * The collector is dead. - * Our memory has already been freed - */ - return; - } - Some(ref id) => unsafe { id.as_ref() }, - }; - let inner = unsafe { self.inner.as_ref() }; - debug_assert!( - !inner.value.load(Ordering::SeqCst).is_null(), - "Pointer already invalid" - ); - let prev = inner.refcnt.fetch_sub(1, Ordering::AcqRel); - match prev { - 0 => { - /* - * This should be impossible. - * - * I believe it's undefined behavior! - */ - panic!("UB: GcHandle refcnt overflow") - } - 1 => { - // Free underlying memory - } - _ => {} // Other references - } - // Mark the value as freed - inner.value.store(ptr::null_mut(), Ordering::Release); - unsafe { - collector - .handle_list() - .append_free_slot(self.inner.as_ptr() as *mut HandleSlot); - } - }); - } -} -/// In order to send *references* between threads, -/// the underlying type must be sync. -/// -/// This is the same reason that `Arc: Send` requires `T: Sync` -/// -/// Requires that the collector is thread-safe. -unsafe impl> + Sync, C: RawHandleImpl + Sync> Send - for GcHandle -{ -} - -/// If the underlying type is Sync, -/// it's safe to share garbage collected references between threads. -/// -/// Requires that the collector is thread-safe. -unsafe impl> + Sync, C: RawHandleImpl + Sync> Sync - for GcHandle -{ -} - -/// We support handles -unsafe impl HandleCollectorId for CollectorId -where - C: RawHandleImpl, -{ - type Handle + ?Sized> = GcHandle; - - #[inline] - fn create_handle<'gc, T>(gc: Gc<'gc, T, CollectorId>) -> Self::Handle - where - T: ?Sized + GcSafe<'gc, Self> + GcRebrand<'static, Self>, - T::Branded: GcSafe<'static, Self>, - { - unsafe { - let collector = gc.collector_id(); - let value = gc.as_raw_ptr(); - let raw = collector - .as_ref() - .handle_list() - .alloc_raw_handle(value as *mut ()); - /* - * WARN: Undefined Behavior - * if we don't finish initializing - * the handle!!! - */ - raw.type_info.store( - C::resolve_type_info(gc) as *const C::TypeInfo as *mut C::TypeInfo, - Ordering::Release, - ); - raw.refcnt.store(1, Ordering::Release); - let weak_collector = collector.weak_ref(); - let metadata = crate::utils::transmute_mismatched::< - ::Metadata, - ::Metadata, - >(ptr::metadata(value)); - GcHandle::new(NonNull::from(raw), weak_collector, metadata) - } - } -} diff --git a/libs/context/src/lib.rs b/libs/context/src/lib.rs deleted file mode 100644 index db5938b..0000000 --- a/libs/context/src/lib.rs +++ /dev/null @@ -1,275 +0,0 @@ -#![feature( - negative_impls, // !Send is much cleaner than `PhantomData` - ptr_metadata -)] -#![allow( - clippy::missing_safety_doc, // Entirely internal code -)] -#![cfg_attr(not(feature = "std"), no_std)] -//! The implementation of (GcContext)[`::zerogc::GcContext`] that is -//! shared among both thread-safe and thread-unsafe code. - -/* - * NOTE: Allocation is still needed for internals - * - * Uses: - * 1. `Box` for each handle - * 2. `Vec` for listing buckets of handles - * 3. `Arc` and `Box` for boxing context state - * - * TODO: Should we drop these uses entirely? - */ -extern crate alloc; - -use core::fmt::{self, Debug, Formatter}; -use core::mem::ManuallyDrop; - -use alloc::boxed::Box; -use alloc::vec::Vec; - -use zerogc::prelude::*; - -pub mod state; - -#[macro_use] -pub mod utils; -pub mod collector; -pub mod handle; - -use crate::collector::RawCollectorImpl; - -pub use crate::collector::{CollectorId, CollectorRef, WeakCollectorRef}; -pub use crate::state::{CollectionManager, RawContext}; - -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub enum ContextState { - /// The context is active. - /// - /// Its contents are potentially being mutated, - /// so the `shadow_stack` doesn't necessarily - /// reflect the actual set of thread roots. - /// - /// New objects could be allocated that are not - /// actually being tracked in the `shadow_stack`. - Active, - /// The context is waiting at a safepoint - /// for a collection to complete. - /// - /// The mutating thread is blocked for the - /// duration of the safepoint (until collection completes). - /// - /// Therefore, its `shadow_stack` is guarenteed to reflect - /// the actual set of thread roots. - SafePoint { - /// The id of the collection we are waiting for - collection_id: u64, - }, - /// The context is frozen. - /// Allocation or mutation can't happen - /// but the mutator thread isn't actually blocked. - /// - /// Unlike a safepoint, this is explicitly unfrozen at the - /// user's discretion. - /// - /// Because no allocation or mutation can happen, - /// its shadow_stack stack is guarenteed to - /// accurately reflect the roots of the context. - #[cfg_attr(not(feature = "sync"), allow(unused))] - // TODO: Implement frozen for simple contexts? - Frozen, -} -impl ContextState { - #[cfg_attr(not(feature = "sync"), allow(unused))] // TODO: Implement frozen for simple contexts? - fn is_frozen(&self) -> bool { - matches!(*self, ContextState::Frozen) - } -} - -/* - * These form a stack of contexts, - * which all share owns a pointer to the RawContext, - * The raw context is implicitly bound to a single thread - * and manages the state of all the contexts. - * - * https://llvm.org/docs/GarbageCollection.html#the-shadow-stack-gc - * Essentially these objects maintain a shadow stack - * - * The pointer to the RawContext must be Arc, since the - * collector maintains a weak reference to it. - * I use double indirection with a `Rc` because I want - * `recurse_context` to avoid the cost of atomic operations. - * - * SimpleCollectorContexts mirror the application stack. - * They can be stack allocated inside `recurse_context`. - * All we would need to do is internally track ownership of the original - * context. The sub-collector in `recurse_context` is very clearly - * restricted to the lifetime of the closure - * which is a subset of the parent's lifetime. - * - * We still couldn't be Send, since we use interior mutablity - * inside of RawContext that is not thread-safe. - */ -// TODO: Rename to remove 'Simple' from name -pub struct CollectorContext { - raw: *mut C::RawContext, - /// Whether we are the root context - /// - /// Only the root actually owns the `Arc` - /// and is responsible for dropping it - root: bool, -} -impl CollectorContext { - pub(crate) unsafe fn register_root(collector: &CollectorRef) -> Self { - CollectorContext { - raw: Box::into_raw(ManuallyDrop::into_inner(C::RawContext::register_new( - collector, - ))), - root: true, // We are responsible for unregistering - } - } - #[inline] - pub fn collector(&self) -> &C { - unsafe { (*self.raw).collector() } - } - #[inline(always)] - unsafe fn with_shadow_stack( - &self, - value: *mut &mut T, - func: impl FnOnce() -> R, - ) -> R { - let old_link = (*(*self.raw).shadow_stack_ptr()).last; - let new_link = ShadowStackLink { - element: C::as_dyn_trace_pointer(value), - prev: old_link, - }; - (*(*self.raw).shadow_stack_ptr()).last = &new_link; - let result = func(); - debug_assert_eq!((*(*self.raw).shadow_stack_ptr()).last, &new_link); - (*(*self.raw).shadow_stack_ptr()).last = new_link.prev; - result - } - #[cold] - unsafe fn trigger_basic_safepoint(&self, element: &mut &mut T) { - self.with_shadow_stack(element, || { - (*self.raw).trigger_safepoint(); - }) - } -} -impl Drop for CollectorContext { - #[inline] - fn drop(&mut self) { - if self.root { - unsafe { - C::Manager::free_context(self.collector(), self.raw); - } - } - } -} -unsafe impl GcContext for CollectorContext { - type System = CollectorRef; - type Id = CollectorId; - - #[inline] - unsafe fn unchecked_safepoint(&self, value: &mut &mut T) { - debug_assert_eq!((*self.raw).state(), ContextState::Active); - if (*self.raw).collector().should_collect() { - self.trigger_basic_safepoint(value); - } - debug_assert_eq!((*self.raw).state(), ContextState::Active); - } - - unsafe fn freeze(&mut self) { - (*self.raw).collector().manager().freeze_context(&*self.raw); - } - - unsafe fn unfreeze(&mut self) { - (*self.raw) - .collector() - .manager() - .unfreeze_context(&*self.raw); - } - - #[inline] - unsafe fn recurse_context(&self, value: &mut &mut T, func: F) -> R - where - T: Trace, - F: for<'gc> FnOnce(&'gc mut Self, &'gc mut T) -> R, - { - debug_assert_eq!((*self.raw).state(), ContextState::Active); - self.with_shadow_stack(value, || { - let mut sub_context = ManuallyDrop::new(CollectorContext { - /* - * safe to copy because we wont drop it - * Lifetime is guarenteed to be restricted to - * the closure. - */ - raw: self.raw, - root: false, /* don't drop our pointer!!! */ - }); - let result = func(&mut *sub_context, value); - debug_assert!(!sub_context.root); - // No need to run drop code on context..... - core::mem::forget(sub_context); - debug_assert_eq!((*self.raw).state(), ContextState::Active); - result - }) - } - - #[inline] - fn system(&self) -> &'_ Self::System { - unsafe { (&*self.raw).collector_ref() } - } - - #[inline] - fn id(&self) -> Self::Id { - unsafe { (&*self.raw).collector() }.id() - } -} - -/// It's not safe for a context to be sent across threads. -/// -/// We use (thread-unsafe) interior mutability to maintain the -/// shadow stack. Since we could potentially be cloned via `safepoint_recurse!`, -/// implementing `Send` would allow another thread to obtain a -/// reference to our internal `&RefCell`. Further mutation/access -/// would be undefined..... -impl !Send for CollectorContext {} - -// -// Root tracking -// - -#[repr(C)] -#[derive(Debug)] -pub(crate) struct ShadowStackLink { - pub element: T, - /// The previous link in the chain, - /// or NULL if there isn't any - pub prev: *const ShadowStackLink, -} - -impl Debug for ShadowStack { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("ShadowStack") - .field("last", &format_args!("{:p}", self.last)) - .finish() - } -} -#[derive(Clone)] -pub struct ShadowStack { - /// The last element in the shadow stack, - /// or NULL if it's empty - pub(crate) last: *const ShadowStackLink, -} -impl ShadowStack { - unsafe fn as_vec(&self) -> Vec { - let mut result: Vec<_> = self.reverse_iter().collect(); - result.reverse(); - result - } - #[inline] - pub unsafe fn reverse_iter(&self) -> impl Iterator + '_ { - core::iter::successors(self.last.as_ref(), |link| link.prev.as_ref()) - .map(|link| link.element) - } -} diff --git a/libs/context/src/state/mod.rs b/libs/context/src/state/mod.rs deleted file mode 100644 index 18e5960..0000000 --- a/libs/context/src/state/mod.rs +++ /dev/null @@ -1,112 +0,0 @@ -use crate::collector::RawCollectorImpl; -use crate::{CollectorRef, ContextState, ShadowStack}; - -use core::fmt::Debug; -use core::mem::ManuallyDrop; - -use alloc::boxed::Box; - -pub mod nosync; -/// The internal state of the collector -/// -/// Has a thread-safe and thread-unsafe implementation. - -#[cfg(feature = "sync")] -pub mod sync; - -/// Manages coordination of garbage collections -pub unsafe trait CollectionManager: self::sealed::Sealed -where - C: RawCollectorImpl, -{ - type Context: RawContext; - fn new() -> Self; - fn is_collecting(&self) -> bool; - fn should_trigger_collection(&self) -> bool; - /// Freeze this context - /// - /// ## Safety - /// See [GcContext::free_context] - unsafe fn freeze_context(&self, context: &Self::Context); - /// Unfreeze the context - /// - /// ## Safety - /// See [GcContext::unfreeze_context] - unsafe fn unfreeze_context(&self, context: &Self::Context); - - // - // Extension methods on collector - // - - /// Attempt to prevent garbage collection for the duration of the closure - /// - /// This method is **OPTIONAL** and will panic if unimplemented. - fn prevent_collection(collector: &C, func: impl FnOnce() -> R) -> R; - - /// Free the specified context - /// - /// ## Safety - /// - Assumes the specified pointer is valid - /// - Assumes there are no more outstanding borrows to values in the context - unsafe fn free_context(collector: &C, context: *mut Self::Context); -} - -/// The underlying state of a context -/// -/// Each context is bound to one and only one thread, -/// even if the collector supports multi-threading. -pub unsafe trait RawContext: Debug + self::sealed::Sealed -where - C: RawCollectorImpl, -{ - unsafe fn register_new(collector: &CollectorRef) -> ManuallyDrop>; - /// Trigger a safepoint for this context. - /// - /// This implicitly attempts a collection, - /// potentially blocking until completion.. - /// - /// Undefined behavior if mutated during collection - /// - /// ## Safety - /// See [GcContext::unchecked_safepoint] - unsafe fn trigger_safepoint(&self); - /// Borrow a reference to the shadow stack, - /// assuming this context is valid (not active). - /// - /// A context is valid if it is either frozen - /// or paused at a safepoint. - /// - /// ## Safety - /// The context must be "inactive", - /// either frozen or paused at a safepoint. - #[inline] - unsafe fn assume_valid_shadow_stack(&self) -> &ShadowStack { - match self.state() { - ContextState::Active => unreachable!("active context: {:?}", self), - ContextState::SafePoint { .. } | ContextState::Frozen { .. } => {} - } - &*self.shadow_stack_ptr() - } - /// Get a pointer to the shadow stack - fn shadow_stack_ptr(&self) -> *mut ShadowStack; - /// Get a reference to the collector as a [CollectorRef] - /// - /// ## Safety - /// Assumes the underlying collector is still valid. - unsafe fn collector_ref(&self) -> &'_ CollectorRef; - /// Get a reference to the collector, - /// assuming that it's valid - /// - /// ## Safety - /// Assumes that the underlying collector - /// is still valid. - #[inline] - unsafe fn collector(&self) -> &C { - self.collector_ref().as_raw() - } - fn state(&self) -> ContextState; -} - -mod sealed { - pub trait Sealed {} -} diff --git a/libs/context/src/state/nosync.rs b/libs/context/src/state/nosync.rs deleted file mode 100644 index 60349c8..0000000 --- a/libs/context/src/state/nosync.rs +++ /dev/null @@ -1,232 +0,0 @@ -//! A simpler implementation of (GcContext)[`::zerogc::GcContext`] -//! that doesn't support multiple threads/contexts. -//! -//! In exchange, there is no locking :) -//! -//! Also, there is `#![no_std]` support - -use core::cell::{Cell, RefCell, UnsafeCell}; -use core::fmt::{self, Debug, Formatter}; -use core::marker::PhantomData; -use core::mem::ManuallyDrop; - -use alloc::boxed::Box; - -use slog::{o, trace, FnValue, Logger}; - -use crate::collector::RawCollectorImpl; -use crate::{CollectorRef, ContextState, ShadowStack}; - -/// Manages coordination of garbage collections -/// -/// This is factored out of the main code mainly due to -/// differences from single-threaded collection -pub struct CollectionManager { - /// Implicit collector ref - _marker: PhantomData>, - /// Access to the internal state - state: RefCell, - /// Whether a collection is currently in progress - /// - /// Used for debugging only - collecting: Cell, - /// Sanity check to ensure there's only one context - has_existing_context: Cell, -} -impl super::sealed::Sealed for CollectionManager {} -unsafe impl super::CollectionManager for CollectionManager -where - C: RawCollectorImpl>, -{ - type Context = RawContext; - - fn new() -> Self { - assert!(!C::SYNC); - CollectionManager { - _marker: PhantomData, - state: RefCell::new(CollectorState::new()), - collecting: Cell::new(false), - has_existing_context: Cell::new(false), - } - } - #[inline] - fn is_collecting(&self) -> bool { - self.collecting.get() - } - #[inline] - fn should_trigger_collection(&self) -> bool { - /* - * Unlike the sync context manager, we can assume - * there is only a single thread. - * Therefore we don't need to check for other threads - * having a collection in progress when we're at a safepoint. - * - * If we were having a collection, control flow is already - * taken over by the collector ;) - */ - false - } - unsafe fn freeze_context(&self, context: &RawContext) { - debug_assert_eq!(context.state.get(), ContextState::Active); - unimplemented!("Freezing single-threaded contexts") - } - unsafe fn unfreeze_context(&self, _context: &RawContext) { - // We can't freeze, so we sure can't unfreeze - unreachable!("Can't unfreeze a single-threaded context") - } - - fn prevent_collection(_collector: &C, _func: impl FnOnce() -> R) -> R { - unimplemented!("Preventing collections for non-sync collectors") - } - - #[inline] - unsafe fn free_context(_collector: &C, _context: *mut Self::Context) { - assert!(!C::SYNC); - // No extra work to do - automatic Drop handles everything - } -} -pub struct RawContext { - /// Since we're the only context, we should (logically) - /// be the only owner. - /// - /// This is still an Arc for easier use alongside the - /// thread-safe implementation - pub(crate) collector: CollectorRef, - // NOTE: We are Send, not Sync - pub(super) shadow_stack: UnsafeCell>, - // TODO: Does the collector access this async? - pub(super) state: Cell, - logger: Logger, -} -impl Debug for RawContext { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - f.debug_struct("RawContext") - .field("collector", &format_args!("{:p}", &self.collector)) - .field( - "shadow_stacks", - // We're assuming this is valid.... - unsafe { &*self.shadow_stack.get() }, - ) - .field("state", &self.state.get()) - .finish() - } -} -impl super::sealed::Sealed for RawContext {} -unsafe impl super::RawContext for RawContext -where - C: RawCollectorImpl>, -{ - unsafe fn register_new(collector: &CollectorRef) -> ManuallyDrop> { - assert!(!C::SYNC); - // NOTE: Nosync collector must have only **ONE** context - assert!( - !collector - .as_raw() - .manager() - .has_existing_context - .replace(true), - "Already created a context for the collector!" - ); - // Assume ownership - let collector = collector.clone_internal(); - let logger = collector.as_raw().logger().new(o!()); - let context = ManuallyDrop::new(Box::new(RawContext { - logger: logger.clone(), - collector, - shadow_stack: UnsafeCell::new(ShadowStack { - last: core::ptr::null_mut(), - }), - state: Cell::new(ContextState::Active), - })); - trace!( - logger, "Initializing context"; - "ptr" => format_args!("{:p}", &**context), - ); - context - } - #[cold] - #[inline(never)] - unsafe fn trigger_safepoint(&self) { - /* - * Begin a collection. - * - * Since we are the only collector we don't have - * to worry about awaiting other threads stopping - * at a safepoint. - * This simplifies the implementation considerably. - */ - assert!(!self.collector.as_raw().manager().collecting.get()); - self.collector.as_raw().manager().collecting.set(true); - let collection_id = self - .collector - .as_raw() - .manager() - .state - .borrow_mut() - .next_pending_id(); - trace!( - self.logger, - "Creating collector"; - "id" => collection_id, - "ctx_ptr" => format_args!("{:?}", self) - ); - let shadow_stack = &*self.shadow_stack.get(); - let ptr = self as *const RawContext as *mut RawContext; - // Change our state to mark we are now waiting at a safepoint - assert_eq!( - self.state - .replace(ContextState::SafePoint { collection_id }), - ContextState::Active - ); - trace!( - self.logger, "Beginning collection"; - "ptr" => ?ptr, - "shadow_stack" => FnValue(|_| alloc::format!("{:?}", shadow_stack.as_vec())), - "state" => ?self.state, - "collection_id" => collection_id, - "original_size" => %self.collector.as_raw().allocated_size(), - ); - self.collector.as_raw().perform_raw_collection(&[ptr]); - assert_eq!( - self.state.replace(ContextState::Active), - ContextState::SafePoint { collection_id } - ); - assert!(self.collector.as_raw().manager().collecting.replace(false)); - } - - #[inline] - fn shadow_stack_ptr(&self) -> *mut ShadowStack { - self.shadow_stack.get() - } - - #[inline] - unsafe fn collector_ref(&self) -> &CollectorRef { - &self.collector - } - - #[inline] - fn state(&self) -> ContextState { - self.state.get() - } -} - -// Pending collections - -/// Keeps track of a pending collection (if any) -/// -/// This must be held under a write lock for a collection to happen. -/// This must be held under a read lock to prevent collections. -pub struct CollectorState { - next_pending_id: u64, -} -#[allow(clippy::new_without_default)] -impl CollectorState { - pub fn new() -> Self { - CollectorState { next_pending_id: 0 } - } - fn next_pending_id(&mut self) -> u64 { - let id = self.next_pending_id; - self.next_pending_id = id.checked_add(1).expect("Overflow"); - id - } -} diff --git a/libs/context/src/state/sync.rs b/libs/context/src/state/sync.rs deleted file mode 100644 index 3d588f4..0000000 --- a/libs/context/src/state/sync.rs +++ /dev/null @@ -1,667 +0,0 @@ -//! Thread safe state -//! -//! Note that this module has full permission to use -//! the standard library in all its glory. -use parking_lot::{ - Condvar, Mutex, RwLock, RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard, -}; -use std::cell::{Cell, UnsafeCell}; -use std::collections::HashSet; -use std::fmt::{Debug, Formatter}; -use std::mem::ManuallyDrop; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::{fmt, mem}; - -use slog::{o, trace, Drain, FnValue, Logger}; - -use super::{ContextState, ShadowStack}; -use crate::collector::SyncCollector; -use crate::utils::ThreadId; -use crate::{CollectorRef, RawCollectorImpl}; - -/// Manages coordination of garbage collections -/// -/// This is factored out of the main code mainly due to -/// differences from single-threaded collection -pub struct CollectionManager { - /// Lock on the internal state - state: RwLock>, - /// Simple flag on whether we're currently collecting - /// - /// This should be true whenever `self.state.pending` is `Some`. - /// However if you don't hold the lock you may not always - /// see a fully consistent state. - collecting: AtomicBool, - /// The condition variable for all contexts to be valid - /// - /// In order to be valid, a context must be either frozen - /// or paused at a safepoint. - /// - /// After a collection is marked as pending, threads must wait until - /// all contexts are valid before the actual work can begin. - valid_contexts_wait: Condvar, - /// Wait until a garbage collection is over. - /// - /// This must be separate from `valid_contexts_wait`. - /// A garbage collection can only begin once all contexts are valid, - /// so this condition logically depends on `valid_contexts_wait`. - collection_wait: Condvar, - /// The mutex used alongside `valid_contexts_wait` - /// - /// This doesn't actually protect any data. It's just - /// used because [parking_lot::RwLock] doesn't support condition vars. - /// This is the [officially suggested solution](https://github.com/Amanieu/parking_lot/issues/165) - // TODO: Should we replace this with `known_collections`? - valid_contexts_lock: Mutex<()>, - /// The mutex used alongside `collection_wait` - /// - /// Like `collection_wait`, his doesn't actually protect any data. - collection_wait_lock: Mutex<()>, -} -impl super::sealed::Sealed for CollectionManager {} -unsafe impl super::CollectionManager for CollectionManager -where - C: SyncCollector, - C: RawCollectorImpl>, -{ - type Context = RawContext; - - fn new() -> Self { - assert!(C::SYNC); - CollectionManager { - state: RwLock::new(CollectorState::new()), - valid_contexts_wait: Condvar::new(), - collection_wait: Condvar::new(), - valid_contexts_lock: Mutex::new(()), - collection_wait_lock: Mutex::new(()), - collecting: AtomicBool::new(false), - } - } - #[inline] - fn is_collecting(&self) -> bool { - self.collecting.load(Ordering::SeqCst) - } - #[inline] - fn should_trigger_collection(&self) -> bool { - /* - * Use relaxed ordering. Eventually consistency is correct - * enough for our use cases. It may delay some threads reaching - * a safepoint for a little bit, but it avoids an expensive - * memory fence on ARM and weakly ordered architectures. - */ - self.collecting.load(Ordering::Relaxed) - } - unsafe fn freeze_context(&self, context: &RawContext) { - assert_eq!(context.state.get(), ContextState::Active); - // TODO: Isn't this state read concurrently? - context.state.set(ContextState::Frozen); - /* - * We may need to notify others that we are frozen - * This means we are now "valid" for the purposes of - * collection ^_^ - */ - self.valid_contexts_wait.notify_all(); - } - unsafe fn unfreeze_context(&self, context: &RawContext) { - /* - * A pending collection might be relying in the validity of this - * context's shadow stack, so unfreezing it while in progress - * could trigger undefined behavior!!!!! - */ - context.collector.as_raw().prevent_collection(|_| { - assert_eq!(context.state.get(), ContextState::Frozen); - context.state.set(ContextState::Active); - }) - } - - #[inline] - fn prevent_collection(collector: &C, func: impl FnOnce() -> R) -> R { - collector.prevent_collection(|_state| func()) - } - - #[inline] - unsafe fn free_context(collector: &C, context: *mut Self::Context) { - collector.free_context(context) - } -} - -pub struct RawContext { - pub(crate) collector: CollectorRef, - original_thread: ThreadId, - // NOTE: We are Send, not Sync - pub(super) shadow_stack: UnsafeCell>, - // TODO: Does the collector access this async? - pub(super) state: Cell, - pub(super) logger: Logger, -} -impl Debug for RawContext { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - f.debug_struct("RawContext") - .field("collector", &format_args!("{:p}", &self.collector)) - .field( - "shadow_stacks", - // We're assuming this is valid.... - unsafe { &*self.shadow_stack.get() }, - ) - .field("state", &self.state.get()) - .finish() - } -} -/// Dummy impl -impl super::sealed::Sealed for RawContext {} -unsafe impl super::RawContext for RawContext -where - C: SyncCollector>, -{ - unsafe fn register_new(collector: &CollectorRef) -> ManuallyDrop> { - let original_thread = if collector.as_raw().logger().is_trace_enabled() { - ThreadId::current() - } else { - ThreadId::Nop - }; - let mut context = ManuallyDrop::new(Box::new(RawContext { - collector: collector.clone_internal(), - original_thread: original_thread.clone(), - logger: collector.as_raw().logger().new(o!( - "original_thread" => original_thread.clone() - )), - shadow_stack: UnsafeCell::new(ShadowStack { - last: std::ptr::null_mut(), - }), - state: Cell::new(ContextState::Active), - })); - let old_num_total = collector.as_raw().add_context(&mut **context); - trace!( - collector.as_raw().logger(), "Creating new context"; - "ptr" => format_args!("{:p}", &**context), - "old_num_total" => old_num_total, - "current_thread" => &original_thread - ); - context - } - #[cold] - #[inline(never)] - unsafe fn trigger_safepoint(&self) { - /* - * Collecting requires a *write* lock. - * We are higher priority than - * - * This assumes that parking_lot priorities pending writes - * over pending reads. The standard library doesn't guarantee this. - */ - let collector = self.collector.as_raw(); - let mut guard = collector.manager().state.write(); - let state = &mut *guard; - // If there is not a active `PendingCollection` - create one - if state.pending.is_none() { - assert_eq!( - collector.manager().collecting.compare_exchange( - false, - true, - Ordering::SeqCst, - Ordering::Relaxed, // Failure is a panic either way -_- - ), - Ok(false) - ); - let id = state.next_pending_id(); - #[allow(clippy::mutable_key_type)] // Used for debugging (see below) - let known_contexts = state.known_contexts.get_mut(); - state.pending = Some(PendingCollection::new( - id, - known_contexts - .iter() - .cloned() - .filter(|ctx| (**ctx).state.get().is_frozen()) - .collect(), - known_contexts.len(), - )); - trace!( - self.logger, - "Creating collector"; - "id" => id, - "ctx_ptr" => format_args!("{:?}", self), - "initially_valid_contexts" => ?state.pending.as_ref() - .unwrap().valid_contexts, - "known_contexts" => FnValue(|_| { - // TODO: Use nested-values/serde?!?! - #[allow(clippy::mutable_key_type)] // We only use this for debugging - let mut map = std::collections::HashMap::new(); - for &context in &*known_contexts { - map.insert(context, format!("{:?} @ {:?}: {:?}", - (*context).state.get(), - (*context).original_thread, - &*(*context).shadow_stack.get() - )); - } - format!("{:?}", map) - }) - ); - } - let shadow_stack = &*self.shadow_stack.get(); - let ptr = self as *const RawContext as *mut RawContext; - debug_assert!(state.known_contexts.get_mut().contains(&ptr)); - let pending: &mut _ = state.pending.as_mut().unwrap(); - // Change our state to mark we are now waiting at a safepoint - assert_eq!( - self.state.replace(ContextState::SafePoint { - collection_id: pending.id - }), - ContextState::Active - ); - debug_assert_eq!(state.known_contexts.get_mut().len(), pending.total_contexts); - pending.push_pending_context(ptr); - let expected_id = pending.id; - pending.waiting_contexts += 1; - trace!( - self.logger, "Awaiting collection"; - "ptr" => ?ptr, - "current_thread" => FnValue(|_| ThreadId::current()), - "shadow_stack" => FnValue(|_| format!("{:?}", shadow_stack.as_vec())), - "total_contexts" => pending.total_contexts, - "waiting_contexts" => pending.waiting_contexts, - "state" => ?pending.state, - "collector_id" => expected_id - ); - collector.await_collection(expected_id, ptr, guard, |state, contexts| { - let pending = state.pending.as_mut().unwrap(); - /* - * NOTE: We keep this trace since it actually dumps contents - * of stacks - */ - trace!( - self.logger, "Beginning simple collection"; - "current_thread" => FnValue(|_| ThreadId::current()), - "original_size" => %collector.allocated_size(), - "contexts" => ?contexts, - "total_contexts" => pending.total_contexts, - "state" => ?pending.state, - "collector_id" => pending.id, - ); - collector.perform_raw_collection(&contexts); - assert_eq!(pending.state, PendingState::InProgress); - pending.state = PendingState::Finished; - // Now acknowledge that we're finished - collector.acknowledge_finished_collection(&mut state.pending, ptr); - }); - trace!( - self.logger, "Finished waiting for collection"; - "current_thread" => FnValue(|_| ThreadId::current()), - "collector_id" => expected_id, - ); - } - - #[inline] - fn shadow_stack_ptr(&self) -> *mut ShadowStack { - self.shadow_stack.get() - } - - #[inline] - unsafe fn collector_ref(&self) -> &CollectorRef { - &self.collector - } - - #[inline] - fn state(&self) -> ContextState { - self.state.get() - } -} -// Pending collections - -/// Keeps track of a pending collection (if any) -/// -/// This must be held under a write lock for a collection to happen. -/// This must be held under a read lock to prevent collections. -pub struct CollectorState { - /// A pointer to the currently pending collection (if any) - /// - /// Once the number of the known roots in the pending collection - /// is equal to the number of `total_contexts`, - /// collection can safely begin. - pending: Option>, - /// A list of all the known contexts - /// - /// This persists across collections. Once a context - /// is freed it's assumed to be unused. - /// - /// NOTE: We need another layer of locking since "readers" - /// like add_context may want - known_contexts: Mutex>>, - next_pending_id: u64, -} -#[allow(clippy::new_without_default)] -impl CollectorState { - pub fn new() -> Self { - CollectorState { - pending: None, - known_contexts: Mutex::new(HashSet::new()), - next_pending_id: 0, - } - } - fn next_pending_id(&mut self) -> u64 { - let id = self.next_pending_id; - self.next_pending_id = id.checked_add(1).expect("Overflow"); - id - } -} - -/// Methods to control collector state. -/// -/// Because we're not defined in the same crate, -/// we must use an extension trait. -pub(crate) trait SyncCollectorImpl: - RawCollectorImpl> -{ - fn prevent_collection(&self, func: impl FnOnce(&CollectorState) -> R) -> R { - // Acquire the lock to ensure there's no collection in progress - let mut state = self.manager().state.read(); - while state.pending.is_some() { - RwLockReadGuard::unlocked(&mut state, || { - let mut lock = self.manager().collection_wait_lock.lock(); - self.manager().collection_wait.wait(&mut lock); - }) - } - assert!(!self.manager().collecting.load(Ordering::SeqCst)); - func(&*state) // NOTE: Lock will be released by RAII - } - unsafe fn add_context(&self, raw: *mut RawContext) -> usize { - /* - * It's unsafe to create a context - * while a collection is in progress. - */ - self.prevent_collection(|state| { - // Lock the list of contexts :p - let mut known_contexts = state.known_contexts.lock(); - let old_num_known = known_contexts.len(); - assert!(known_contexts.insert(raw), "Already inserted {:p}", raw); - old_num_known - }) - } - unsafe fn free_context(&self, raw: *mut RawContext) { - trace!( - self.logger(), "Freeing context"; - "ptr" => format_args!("{:p}", raw), - "state" => ?(*raw).state.get() - ); - /* - * TODO: Consider using regular read instead of `read_upgradable` - * This is only prevented because of the fact that we may - * need to mutate `valid_contexts` and `total_contexts` - */ - let ptr = raw; // TODO - blegh - let guard = self.manager().state.upgradable_read(); - match &guard.pending { - None => { - // No collection - // Still need to remove from list of known_contexts - assert!(guard.known_contexts.lock().remove(&ptr)); - drop(guard); - } - Some( - pending @ PendingCollection { - state: PendingState::Finished, - .. - }, - ) => { - /* - * We're assuming here that the finished collection - * has already been removed from the list - * of `pending_contexts` and has left the safepoint. - * Verify this assumption - */ - assert!(!pending.valid_contexts.contains(&ptr)); - assert!(guard.known_contexts.lock().remove(&ptr)); - drop(guard); - } - Some(PendingCollection { - state: PendingState::Waiting, - .. - }) => { - let mut guard = RwLockUpgradableReadGuard::upgrade(guard); - /* - * Freeing a context could actually benefit - * a waiting collection by allowing it to - * proceed. - * Verify that we're not actually in the `valid_contexts` - * list. A context in that list must not be freed. - * - * We need to upgrade the lock before we can mutate the state - */ - let pending = guard.pending.as_mut().unwrap(); - // We shouldn't be in the list of valid contexts! - assert_eq!( - pending - .valid_contexts - .iter() - .find(|&&ctx| std::ptr::eq(ctx, ptr)), - None, - "state = {:?}", - (*raw).state.get() - ); - pending.total_contexts -= 1; - assert!(guard.known_contexts.get_mut().remove(&ptr)); - drop(guard); - } - Some(PendingCollection { - state: PendingState::InProgress, - .. - }) => { - unreachable!("cant free while collection is in progress") - } - } - /* - * Notify all threads waiting for contexts to be valid. - * TODO: I think this is really only useful if we're waiting.... - */ - self.manager().valid_contexts_wait.notify_all(); - // Now drop the Box - drop(Box::from_raw(raw)); - } - /// Wait until the specified collection is finished - /// - /// This will implicitly begin collection as soon - /// as its ready - unsafe fn await_collection( - &self, - expected_id: u64, - context: *mut RawContext, - mut lock: RwLockWriteGuard>, - perform_collection: impl FnOnce(&mut CollectorState, Vec<*mut RawContext>), - ) { - loop { - match &mut lock.pending { - Some(ref mut pending) => { - /* - * We should never move onto a new collection - * till we're finished waiting.... - */ - assert_eq!(expected_id, pending.id); - // Since we're Some, this should be true - debug_assert!(self.manager().collecting.load(Ordering::SeqCst)); - match pending.state { - PendingState::Finished => { - self.acknowledge_finished_collection(&mut lock.pending, context); - drop(lock); - return; - } - PendingState::Waiting - if pending.valid_contexts.len() == pending.total_contexts => - { - /* - * We have all the roots. Trigger a collection - * Here we're assuming all the shadow stacks we've - * accumulated actually correspond to the shadow stacks - * of all the live contexts. - * We also assume that the shadow stacks correspond to - * the program's roots. - */ - assert_eq!( - std::mem::replace(&mut pending.state, PendingState::InProgress), - PendingState::Waiting - ); - /* - * In debug mode we keep using `valid_contexts` - * for sanity checks later on - */ - let contexts = if cfg!(debug_assertions) { - pending.valid_contexts.clone() - } else { - // Otherwise we might as well be done with it - mem::take(&mut pending.valid_contexts) - }; - perform_collection(&mut *lock, contexts); - drop(lock); - /* - * Notify all blocked threads we're finished. - * We can proceed immediately and everyone else - * will slowly begin to wakeup; - */ - self.manager().collection_wait.notify_all(); - return; - } - PendingState::Waiting => { - RwLockWriteGuard::unlocked(&mut lock, || { - let mut lock = self.manager().valid_contexts_lock.lock(); - /* - * Wait for all contexts to be valid. - * - * Typically we're waiting for them to reach - * a safepoint. - */ - self.manager().valid_contexts_wait.wait(&mut lock); - }) - } - PendingState::InProgress => { - RwLockWriteGuard::unlocked(&mut lock, || { - let mut lock = self.manager().collection_wait_lock.lock(); - /* - * Another thread has already started collecting. - * Wait for them to finish. - * - * Block until collection finishes - * Parking lot says there shouldn't be any "spurious" - * (accidental) wakeups. However I guess it's possible - * we're woken up somehow in the middle of collection. - */ - self.manager().collection_wait.wait(&mut lock); - }) - } - } - } - None => { - panic!("Unexpectedly finished collection: {}", expected_id) - } - } - } - } - unsafe fn acknowledge_finished_collection( - &self, - pending_ref: &mut Option>, - context: *mut RawContext, - ) { - let waiting_contexts = { - let pending = pending_ref.as_mut().unwrap(); - assert_eq!(pending.state, PendingState::Finished); - if cfg!(debug_assertions) { - // Remove ourselves to mark the fact we're done - match pending - .valid_contexts - .iter() - .position(|&ptr| std::ptr::eq(ptr, context)) - { - Some(index) => { - pending.valid_contexts.remove(index); - } - None => panic!("Unable to find context: {:p}", context), - } - } - // Mark ourselves as officially finished with the safepoint - assert_eq!( - (*context).state.replace(ContextState::Active), - ContextState::SafePoint { - collection_id: pending.id - } - ); - pending.waiting_contexts -= 1; - pending.waiting_contexts - }; - if waiting_contexts == 0 { - *pending_ref = None; - assert_eq!( - self.manager().collecting.compare_exchange( - true, - false, - Ordering::SeqCst, - Ordering::Relaxed, // Failure is catastrophic - ), - Ok(true) - ); - // Someone may be waiting for us to become `None` - self.manager().collection_wait.notify_all(); - } - } -} -/// Blanket implementation -impl SyncCollectorImpl for C -where - C: crate::collector::SyncCollector, - C: RawCollectorImpl>, -{ -} - -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -enum PendingState { - /// The desired collection was finished - Finished, - /// The collection is in progress - InProgress, - /// Waiting for all the threads to stop at a safepoint - /// - /// We need every thread's shadow stack to safely proceed. - Waiting, -} - -/// The state of a collector waiting for all its contexts -/// to reach a safepoint -#[derive(Debug)] -pub(crate) struct PendingCollection { - /// The state of the current collection - state: PendingState, - /// The total number of known contexts - /// - /// While a collection is in progress, - /// no new contexts will be added. - /// Freeing a context will update this as - /// appropriate. - total_contexts: usize, - /// The number of contexts that are waiting - waiting_contexts: usize, - /// The contexts that are ready to be collected. - valid_contexts: Vec<*mut RawContext>, - /// The unique id of this safepoint - /// - /// 64-bit integers pretty much never overflow (for like 100 years) - id: u64, -} -impl PendingCollection { - pub fn new(id: u64, valid_contexts: Vec<*mut RawContext>, total_contexts: usize) -> Self { - PendingCollection { - state: PendingState::Waiting, - total_contexts, - waiting_contexts: 0, - valid_contexts, - id, - } - } - /// Push a context that's pending collection - /// - /// Undefined behavior if the context roots are invalid in any way. - #[inline] - pub unsafe fn push_pending_context(&mut self, context: *mut RawContext) { - debug_assert_ne!((*context).state.get(), ContextState::Active); - assert_eq!(self.state, PendingState::Waiting); - debug_assert!(!self.valid_contexts.contains(&context)); - self.valid_contexts.push(context); - // We be waiting - assert_eq!(self.state, PendingState::Waiting); - } -} diff --git a/libs/context/src/utils.rs b/libs/context/src/utils.rs deleted file mode 100644 index ce59d66..0000000 --- a/libs/context/src/utils.rs +++ /dev/null @@ -1,164 +0,0 @@ -//! Utilities for the context library -//! -//! Also used by some collector implementations. -#[cfg(not(feature = "sync"))] -use core::cell::Cell; -use core::fmt::{self, Debug, Display, Formatter}; -use core::mem; - -/// Get the offset of the specified field within a structure -#[macro_export] -macro_rules! field_offset { - ($target:ty, $($field:ident).+) => {{ - const OFFSET: usize = { - let uninit = core::mem::MaybeUninit::<$target>::uninit(); - unsafe { ((core::ptr::addr_of!((*uninit.as_ptr())$(.$field)*)) as *const u8) - .offset_from(uninit.as_ptr() as *const u8) as usize } - }; - OFFSET - }}; -} - -/// Transmute between two types, -/// without verifying that there sizes are the same -/// -/// ## Safety -/// This function has undefined behavior if `T` and `U` -/// have different sizes. -/// -/// It also has undefined behavior whenever [mem::transmute] has -/// undefined behavior. -#[inline] -pub unsafe fn transmute_mismatched(src: T) -> U { - // NOTE: This assert has zero cost when monomorphized - assert_eq!(mem::size_of::(), mem::size_of::()); - let d = mem::ManuallyDrop::new(src); - mem::transmute_copy::(&*d) -} - -#[cfg(feature = "sync")] -pub type AtomicCell = ::crossbeam_utils::atomic::AtomicCell; -/// Fallback `AtomicCell` implementation when we actually -/// don't care about thread safety -#[cfg(not(feature = "sync"))] -#[derive(Default)] -pub struct AtomicCell(Cell); -#[cfg(not(feature = "sync"))] -impl AtomicCell { - pub const fn new(value: T) -> Self { - AtomicCell(Cell::new(value)) - } - pub fn store(&self, value: T) { - self.0.set(value) - } - pub fn load(&self) -> T { - self.0.get() - } - pub fn compare_exchange(&self, expected: T, updated: T) -> Result - where - T: PartialEq, - { - let existing = self.0.get(); - if existing == expected { - self.0.set(updated); - Ok(existing) - } else { - Err(existing) - } - } -} - -#[derive(Clone)] -pub enum ThreadId { - #[allow(unused)] - Nop, - #[cfg(feature = "std")] - Enabled { - id: std::thread::ThreadId, - name: Option, - }, -} -impl ThreadId { - #[cfg(feature = "std")] - pub fn current() -> ThreadId { - // NOTE: It's okay: `sync` requires std - let thread = std::thread::current(); - ThreadId::Enabled { - id: thread.id(), - name: thread.name().map(String::from), - } - } - #[cfg(not(feature = "std"))] - #[inline] - pub fn current() -> ThreadId { - ThreadId::Nop - } -} -impl slog::Value for ThreadId { - #[cfg(not(feature = "std"))] - fn serialize( - &self, - _record: &slog::Record, - _key: &'static str, - _serializer: &mut dyn slog::Serializer, - ) -> slog::Result<()> { - Ok(()) // Nop - } - #[cfg(feature = "std")] - fn serialize( - &self, - _record: &slog::Record, - key: &'static str, - serializer: &mut dyn slog::Serializer, - ) -> slog::Result<()> { - let (id, name) = match *self { - ThreadId::Nop => return Ok(()), - ThreadId::Enabled { ref id, ref name } => (id, name), - }; - match *name { - Some(ref name) => serializer.emit_arguments(key, &format_args!("{}: {:?}", *name, id)), - None => serializer.emit_arguments(key, &format_args!("{:?}", id)), - } - } -} -impl Debug for ThreadId { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match *self { - ThreadId::Nop => f.write_str("ThreadId(??)"), - #[cfg(feature = "std")] - ThreadId::Enabled { id, name: None } => { - write!(f, "{:?}", id) - } - #[cfg(feature = "std")] - ThreadId::Enabled { - id, - name: Some(ref name), - } => f.debug_tuple("ThreadId").field(&id).field(name).finish(), - } - } -} -/// The size of memory in bytes -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub struct MemorySize { - pub bytes: usize, -} -impl Display for MemorySize { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - if f.alternate() { - write!(f, "{}", self.bytes) - } else { - // Write approximation - let bytes = self.bytes; - let (amount, suffix) = if bytes > 1024 * 1024 * 1024 { - (1024 * 1024 * 1024, "GB") - } else if bytes > 1024 * 1024 { - (1024 * 1024, "MB") - } else if bytes > 1024 { - (1024, "KB") - } else { - (1, "") - }; - write!(f, "{:.2}{}", bytes as f64 / amount as f64, suffix) - } - } -} diff --git a/libs/simple/Cargo.toml b/libs/simple/Cargo.toml deleted file mode 100644 index abb40ad..0000000 --- a/libs/simple/Cargo.toml +++ /dev/null @@ -1,69 +0,0 @@ -[package] -name = "zerogc-simple" -description = "Lightweight mark/sweep collector for zerogc." -version.workspace = true -authors.workspace = true -repository.workspace =true -license.workspace = true -edition.workspace = true -readme = "../../README.md" - -[dependencies] -inherent = "1" -zerogc = { path = "../..", version = "0.2.0-alpha.6" } -once_cell = { version = "1.5", optional = true } -# Shared impl -zerogc-context = { path = "../context", version = "0.2.0-alpha.6", default-features = false } -zerogc-derive = { path = "../derive", version = "0.2.0-alpha.6" } -# Concurrency -parking_lot = { version = "0.11", optional = true } -# Logging -slog = "2.7" - -[features] -default = [ - "small-object-arenas", # Without this, allocating small objects is slow - "sync", # Thread-safety by default - "multiple-collectors", # By default, allow multiple collectors -] -# Use very fast dedicated arenas for small objects. -# This makes allocation much faster -# Time spent in malloc (even in heavy workloads) drops to near zero -# This can also improve memory significantly by avoiding per-object overheads -# -# However, it increases code complexity and is more -# agressive (memory wise) then delegating all work to std::alloc -# TODO: Return unused memory to the operating systems -# TODO: Thread-local caching (make arenas fast again) -small-object-arenas = ["once_cell"] -# Use recursion to implicitly track the grey stack -# This risks stack overflow at a possible performance gain -# See commit 9a9634d68a4933d -implicit-grey-stack = [] -# Allow multiple threads to access the garbage collector -# by creating a seperate context for each. -# -# This can increase overhead by requiring communication between threads. -sync = ["zerogc-context/sync", "parking_lot"] -# Allow multiple collectors to exist at once -# Otherwise, there's a single global collector (useful in VMs) -# -# Even if multiple collectors are enabled, pointers from -# one collector can't be safely mixed with other collectors. -multiple-collectors = [] - -[[test]] -name = "errors" -required-features = ["sync"] - -[dev-dependencies] -# Used for examples :) -zerogc-derive = { path = "../derive" } -# Used for binary_trees parallel example -rayon = "1.3" -slog-term = "2.6" -# Used to test the 'error' type -anyhow = "1" -thiserror = "1" -zerogc = { path = "../..", features = ["errors"] } - diff --git a/libs/simple/examples/binary_trees.rs b/libs/simple/examples/binary_trees.rs deleted file mode 100644 index 045549c..0000000 --- a/libs/simple/examples/binary_trees.rs +++ /dev/null @@ -1,97 +0,0 @@ -#![feature( - arbitrary_self_types, // Unfortunately this is required for methods on Gc refs -)] -use zerogc::prelude::*; -use zerogc_derive::Trace; -use zerogc_simple::{ - CollectorId as SimpleCollectorId, Gc, SimpleCollector, SimpleCollectorContext, -}; - -use slog::{o, Drain, Logger}; - -#[derive(Trace)] -#[zerogc(collector_ids(SimpleCollectorId))] -struct Tree<'gc> { - #[zerogc(mutable(public))] - children: GcCell>, Gc<'gc, Tree<'gc>>)>>, -} - -fn item_check(tree: &Tree) -> i32 { - if let Some((left, right)) = tree.children.get() { - 1 + item_check(&right) + item_check(&left) - } else { - 1 - } -} - -fn bottom_up_tree<'gc>(collector: &'gc SimpleCollectorContext, depth: i32) -> Gc<'gc, Tree<'gc>> { - let tree = collector.alloc(Tree { - children: GcCell::new(None), - }); - if depth > 0 { - let right = bottom_up_tree(collector, depth - 1); - let left = bottom_up_tree(collector, depth - 1); - tree.set_children(Some((left, right))); - } - tree -} - -fn inner(gc: &mut SimpleCollectorContext, depth: i32, iterations: u32) -> String { - let chk: i32 = (0..iterations) - .into_iter() - .map(|_| { - safepoint_recurse!(gc, |gc| { - let a = bottom_up_tree(&gc, depth); - item_check(&a) - }) - }) - .sum(); - format!("{}\t trees of depth {}\t check: {}", iterations, depth, chk) -} - -fn main() { - let n = std::env::args() - .nth(1) - .and_then(|n| n.parse().ok()) - .unwrap_or(10); - let min_depth = 4; - let max_depth = if min_depth + 2 > n { min_depth + 2 } else { n }; - - let plain = slog_term::PlainSyncDecorator::new(std::io::stdout()); - let logger = Logger::root( - slog_term::FullFormat::new(plain).build().fuse(), - o!("bench" => file!()), - ); - let collector = SimpleCollector::with_logger(logger); - let mut gc = collector.into_context(); - { - let depth = max_depth + 1; - let tree = bottom_up_tree(&gc, depth); - println!( - "stretch tree of depth {}\t check: {}", - depth, - item_check(&tree) - ); - } - safepoint!(gc, ()); - - let long_lived_tree = bottom_up_tree(&gc, max_depth); - - let (long_lived_tree, ()) = safepoint_recurse!(gc, long_lived_tree, |gc, _long_lived_tree| { - (min_depth / 2..max_depth / 2 + 1) - .into_iter() - .for_each(|half_depth| { - let depth = half_depth * 2; - let iterations = 1 << ((max_depth - depth + min_depth) as u32); - let message = - safepoint_recurse!(gc, |new_gc| { inner(&mut new_gc, depth, iterations) }); - println!("{}", message); - }) - }); - - println!( - "long lived tree of depth {}\t check: {}", - max_depth, - item_check(&long_lived_tree) - ); -} diff --git a/libs/simple/examples/binary_trees_parallel.rs b/libs/simple/examples/binary_trees_parallel.rs deleted file mode 100644 index a020757..0000000 --- a/libs/simple/examples/binary_trees_parallel.rs +++ /dev/null @@ -1,101 +0,0 @@ -#![feature( - arbitrary_self_types, // Unfortunately this is required for methods on Gc refs -)] -use zerogc::prelude::*; -use zerogc_derive::Trace; -use zerogc_simple::{ - CollectorId as SimpleCollectorId, Gc, SimpleCollector, SimpleCollectorContext, -}; - -use rayon::prelude::*; -use slog::{o, Drain, Logger}; - -#[derive(Trace)] -#[zerogc(collector_ids(SimpleCollectorId))] -struct Tree<'gc> { - #[zerogc(mutable(public))] - children: GcCell>, Gc<'gc, Tree<'gc>>)>>, -} - -fn item_check(tree: &Tree) -> i32 { - if let Some((left, right)) = tree.children.get() { - 1 + item_check(&right) + item_check(&left) - } else { - 1 - } -} - -fn bottom_up_tree<'gc>(collector: &'gc SimpleCollectorContext, depth: i32) -> Gc<'gc, Tree<'gc>> { - let tree = collector.alloc(Tree { - children: GcCell::new(None), - }); - if depth > 0 { - let right = bottom_up_tree(collector, depth - 1); - let left = bottom_up_tree(collector, depth - 1); - tree.set_children(Some((left, right))); - } - tree -} - -fn inner(collector: &SimpleCollector, depth: i32, iterations: u32) -> String { - let chk: i32 = (0..iterations) - .into_par_iter() - .map(|_| { - let mut gc = collector.create_context(); - safepoint_recurse!(gc, |gc| { - let a = bottom_up_tree(&gc, depth); - item_check(&a) - }) - }) - .sum(); - format!("{}\t trees of depth {}\t check: {}", iterations, depth, chk) -} - -fn main() { - let n = std::env::args() - .nth(1) - .and_then(|n| n.parse().ok()) - .unwrap_or(10); - let min_depth = 4; - let max_depth = if min_depth + 2 > n { min_depth + 2 } else { n }; - - let plain = slog_term::PlainSyncDecorator::new(std::io::stdout()); - let logger = Logger::root( - slog_term::FullFormat::new(plain).build().fuse(), - o!("bench" => file!()), - ); - let collector = SimpleCollector::with_logger(logger); - let mut gc = collector.create_context(); - { - let depth = max_depth + 1; - let tree = bottom_up_tree(&gc, depth); - println!( - "stretch tree of depth {}\t check: {}", - depth, - item_check(&tree) - ); - } - safepoint!(gc, ()); - - let long_lived_tree = bottom_up_tree(&gc, max_depth); - let long_lived_tree = long_lived_tree.create_handle(); - let frozen = freeze_context!(gc); - - (min_depth / 2..max_depth / 2 + 1) - .into_par_iter() - .for_each(|half_depth| { - let depth = half_depth * 2; - let iterations = 1 << ((max_depth - depth + min_depth) as u32); - // NOTE: We're relying on inner to do safe points internally - let message = inner(&collector, depth, iterations); - println!("{}", message); - }); - let new_context = unfreeze_context!(frozen); - let long_lived_tree = long_lived_tree.bind_to(&new_context); - - println!( - "long lived tree of depth {}\t check: {}", - max_depth, - item_check(&long_lived_tree) - ); -} diff --git a/libs/simple/src/alloc.rs b/libs/simple/src/alloc.rs deleted file mode 100644 index b68e954..0000000 --- a/libs/simple/src/alloc.rs +++ /dev/null @@ -1,433 +0,0 @@ -#![allow(clippy::vec_box)] // We must Box for a stable address -use std::alloc::Layout; -use std::mem; -use std::mem::MaybeUninit; -use std::ptr::NonNull; - -#[cfg(feature = "sync")] -use once_cell::sync::OnceCell; -#[cfg(not(feature = "sync"))] -use once_cell::unsync::OnceCell; -#[cfg(feature = "sync")] -use parking_lot::Mutex; -#[cfg(not(feature = "sync"))] -use std::cell::RefCell; - -use zerogc_context::utils::AtomicCell; - -const DEBUG_INTERNAL_ALLOCATOR: bool = cfg!(zerogc_simple_debug_alloc); -#[allow(clippy::assertions_on_constants)] // See rust-lang/clippy#7597 -mod debug { - pub const PADDING: u32 = 0xDEADBEAF; - pub const UNINIT: u32 = 0xCAFEBABE; - pub const PADDING_TIMES: usize = 16; - pub const PADDING_BYTES: usize = PADDING_TIMES * 4; - pub unsafe fn pad_memory_block(ptr: *mut u8, size: usize) { - assert!(super::DEBUG_INTERNAL_ALLOCATOR); - let start = ptr.sub(PADDING_BYTES); - for i in 0..PADDING_TIMES { - (start as *mut u32).add(i).write(PADDING); - } - let end = ptr.add(size); - for i in 0..PADDING_TIMES { - (end as *mut u32).add(i).write(PADDING); - } - } - pub unsafe fn mark_memory_uninit(ptr: *mut u8, size: usize) { - assert!(super::DEBUG_INTERNAL_ALLOCATOR); - let (blocks, leftover) = (size / 4, size % 4); - for i in 0..blocks { - (ptr as *mut u32).add(i).write(UNINIT); - } - let leftover_ptr = ptr.add(blocks * 4); - debug_assert_eq!(leftover_ptr.wrapping_add(leftover), ptr.add(size)); - for i in 0..leftover { - leftover_ptr.add(i).write(0xF0); - } - } - pub unsafe fn assert_padded(ptr: *mut u8, size: usize) { - assert!(super::DEBUG_INTERNAL_ALLOCATOR); - let start = ptr.sub(PADDING_BYTES); - let end = ptr.add(size); - let start_padding = - std::slice::from_raw_parts(start as *const u8 as *const u32, PADDING_TIMES); - let region = std::slice::from_raw_parts(ptr as *const u8, size); - let end_padding = std::slice::from_raw_parts(end as *const u8 as *const u32, PADDING_TIMES); - let print_memory_region = || { - use std::fmt::Write; - let mut res = String::new(); - for &val in start_padding { - write!(&mut res, "{:X}", val).unwrap(); - } - res.push_str("||"); - for &b in region { - write!(&mut res, "{:X}", b).unwrap(); - } - res.push_str("||"); - for &val in end_padding { - write!(&mut res, "{:X}", val).unwrap(); - } - res - }; - // Closest to farthest - for (idx, &block) in start_padding.iter().rev().enumerate() { - if block == PADDING { - continue; - } - assert_eq!( - block, - PADDING, - "Unexpected start padding (offset -{}) w/ {}", - idx * 4, - print_memory_region() - ); - } - for (idx, &block) in end_padding.iter().enumerate() { - if block == PADDING { - continue; - } - assert_eq!( - block, - PADDING, - "Unexpected end padding (offset {}) w/ {}", - idx * 4, - print_memory_region() - ) - } - } -} -/// The minimum size of supported memory (in words) -/// -/// Since the header takes at least one word, -/// its not really worth ever allocating less than this -pub const MINIMUM_WORDS: usize = 2; -/// The maximum words supported by small arenas -/// -/// Past this we have to fallback to the global allocator -pub const MAXIMUM_SMALL_WORDS: usize = 32; -/// The alignment of elements in the arena -pub const ARENA_ELEMENT_ALIGN: usize = std::mem::align_of::(); - -use crate::layout::{GcHeader, UnknownHeader}; - -#[inline] -pub const fn fits_small_object(layout: Layout) -> bool { - layout.size() <= MAXIMUM_SMALL_WORDS * std::mem::size_of::() - && layout.align() <= ARENA_ELEMENT_ALIGN -} - -pub(crate) struct Chunk { - pub start: *mut u8, - current: AtomicCell<*mut u8>, - pub end: *mut u8, -} -impl Chunk { - fn alloc(capacity: usize) -> Box { - assert!(capacity >= 1); - let mut result = Vec::::with_capacity(capacity); - let start = result.as_mut_ptr(); - std::mem::forget(result); - let current = AtomicCell::new(start); - Box::new(Chunk { - start, - current, - end: unsafe { start.add(capacity) }, - }) - } - - #[inline] - fn try_alloc(&self, amount: usize) -> Option> { - loop { - let old_current = self.current.load(); - let remaining = self.end as usize - old_current as usize; - if remaining >= amount { - unsafe { - let updated = old_current.add(amount); - if self.current.compare_exchange(old_current, updated).is_ok() { - return Some(NonNull::new_unchecked(old_current)); - } else { - continue; - } - } - } else { - return None; - } - } - } - #[inline] - fn capacity(&self) -> usize { - self.end as usize - self.start as usize - } -} -impl Drop for Chunk { - fn drop(&mut self) { - unsafe { drop(Vec::from_raw_parts(self.start, 0, self.capacity())) } - } -} - -/// A slot in the free list -#[repr(C)] -pub struct FreeSlot { - /// Pointer to the previous free slot - pub(crate) prev_free: Option>, -} -pub const NUM_SMALL_ARENAS: usize = 15; -const INITIAL_SIZE: usize = 512; - -/// The current state of the allocator. -/// -/// TODO: Support per-thread arena caching -struct ArenaState { - /// We have to Box the chunk so that it'll remain valid - /// even when we move it. - /// - /// This is required for thread safety. - /// One thread could still be seeing an old chunk's location - /// after it's been moved. - #[cfg(feature = "sync")] - chunks: Mutex>>, - /// List of chunks, not thread-safe - /// - /// We still box it however, as an extra check of safety. - #[cfg(not(feature = "sync"))] - chunks: RefCell>>, - /// Lockless access to the current chunk - /// - /// The pointers wont be invalidated, - /// since the references are internally boxed. - current_chunk: AtomicCell>, -} -impl ArenaState { - fn new(chunks: Vec>) -> Self { - assert!(!chunks.is_empty()); - let current_chunk = NonNull::from(&**chunks.last().unwrap()); - let chunk_lock; - #[cfg(feature = "sync")] - { - chunk_lock = Mutex::new(chunks); - } - #[cfg(not(feature = "sync"))] - { - chunk_lock = RefCell::new(chunks); - } - ArenaState { - chunks: chunk_lock, - current_chunk: AtomicCell::new(current_chunk), - } - } - #[inline] - #[cfg(feature = "sync")] - fn lock_chunks(&self) -> ::parking_lot::MutexGuard>> { - self.chunks.lock() - } - #[inline] - #[cfg(not(feature = "sync"))] - fn lock_chunks(&self) -> ::std::cell::RefMut>> { - self.chunks.borrow_mut() - } - #[inline] - fn current_chunk(&self) -> NonNull { - self.current_chunk.load() - } - #[inline] - unsafe fn force_current_chunk(&self, ptr: NonNull) { - self.current_chunk.store(ptr); - } - #[inline] - fn alloc(&self, element_size: usize) -> NonNull { - unsafe { - let chunk = &*self.current_chunk().as_ptr(); - match chunk.try_alloc(element_size) { - Some(header) => header.cast(), - None => self.alloc_fallback(element_size), - } - } - } - - #[cold] - #[inline(never)] - fn alloc_fallback(&self, element_size: usize) -> NonNull { - let mut chunks = self.lock_chunks(); - // Now that we hold the lock, check the current chunk again - unsafe { - if let Some(header) = self.current_chunk().as_ref().try_alloc(element_size) { - return header.cast(); - } - } - // Double capacity to amortize growth - let last_capacity = chunks.last().unwrap().capacity(); - chunks.push(Chunk::alloc(last_capacity * 2)); - unsafe { - self.force_current_chunk(NonNull::from(&**chunks.last().unwrap())); - self.current_chunk() - .as_ref() - .try_alloc(element_size) - .unwrap() - .cast::() - } - } -} - -/// The free list -/// -/// This is a lock-free linked list -#[derive(Default)] -pub(crate) struct FreeList { - next: AtomicCell>>, -} -impl FreeList { - unsafe fn add_free(&self, free: *mut UnknownHeader, size: usize) { - if DEBUG_INTERNAL_ALLOCATOR { - debug::assert_padded(free as *mut u8, size); - debug::mark_memory_uninit(free as *mut u8, size); - } - let new_slot = free as *mut FreeSlot; - let mut next = self.next.load(); - loop { - (*new_slot).prev_free = next; - match self - .next - .compare_exchange(next, Some(NonNull::new_unchecked(new_slot))) - { - Ok(_) => break, - Err(actual_next) => { - next = actual_next; - } - } - } - } - #[inline] - fn take_free(&self) -> Option> { - loop { - let next_free = match self.next.load() { - Some(free) => free, - None => return None, // Out of free space - }; - // Update free pointer - unsafe { - if self - .next - .compare_exchange(Some(next_free), next_free.as_ref().prev_free) - .is_err() - { - continue; /* retry */ - } - return Some(next_free.cast()); - } - } - } -} - -pub struct SmallArena { - pub(crate) element_size: usize, - state: ArenaState, - free: FreeList, -} - -impl SmallArena { - pub(crate) unsafe fn add_free(&self, obj: *mut UnknownHeader) { - self.free.add_free(obj, self.element_size) - } - #[cold] // Initialization is the slow path - fn with_words(num_words: usize) -> SmallArena { - assert!(num_words >= MINIMUM_WORDS); - let element_size = num_words * mem::size_of::(); - assert!(INITIAL_SIZE >= element_size * 2); - let chunks = vec![Chunk::alloc(INITIAL_SIZE)]; - SmallArena { - state: ArenaState::new(chunks), - element_size, - free: Default::default(), - } - } - #[inline] - pub(crate) fn alloc(&self) -> NonNull { - // Check the free list - if let Some(free) = self.free.take_free() { - free.cast() - } else if DEBUG_INTERNAL_ALLOCATOR { - let mem = self - .state - .alloc(self.element_size + debug::PADDING_BYTES * 2) - .as_ptr() as *mut u8; - unsafe { - let mem = mem.add(debug::PADDING_BYTES); - debug::pad_memory_block(mem, self.element_size); - debug::mark_memory_uninit(mem, self.element_size); - NonNull::new_unchecked(mem).cast() - } - } else { - self.state.alloc(self.element_size) - } - } -} -macro_rules! arena_match { - ($arenas:expr, $target:ident, max = $max:expr; $($size:pat => $num_words:literal @ $idx:expr),*) => { - Some(match $target { - $($size => $arenas[$idx].get_or_init(|| { - assert_eq!(SMALL_ARENA_SIZES[$idx], $num_words); - SmallArena::with_words($num_words) - }),)* - _ => { - assert!($target > $max); - return None - } - }) - }; -} -const SMALL_ARENA_SIZES: [usize; NUM_SMALL_ARENAS] = - [2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32]; -pub struct SmallArenaList { - // NOTE: Internally boxed to avoid bloating main struct - arenas: Box<[OnceCell; NUM_SMALL_ARENAS]>, -} -impl SmallArenaList { - pub fn new() -> Self { - // NOTE: Why does writing arrays have to be so difficult:? - unsafe { - let mut arenas: Box<[MaybeUninit>; NUM_SMALL_ARENAS]> = - Box::new_uninit().assume_init(); - for i in 0..NUM_SMALL_ARENAS { - arenas[i].as_mut_ptr().write(OnceCell::new()); - } - SmallArenaList { - // NOTE: This is done because I want to explicitly specify types - arenas: mem::transmute::< - Box<[MaybeUninit>; NUM_SMALL_ARENAS]>, - Box<[OnceCell; NUM_SMALL_ARENAS]>, - >(arenas), - } - } - } - #[inline] // This should hopefully be constant folded away (layout is const) - pub fn find(&self, layout: Layout) -> Option<&SmallArena> { - if !fits_small_object(layout) { - return None; - } - // Divide round up - let word_size = mem::size_of::(); - let num_words = (layout.size() + (word_size - 1)) / word_size; - self.find_raw(num_words) - } - #[inline] // We want this constant-folded away...... - fn find_raw(&self, num_words: usize) -> Option<&SmallArena> { - arena_match!( - self.arenas, num_words, max = 32; - 0..=2 => 2 @ 0, - 3 => 3 @ 1, - 4 => 4 @ 2, - 5 => 5 @ 3, - 6 => 6 @ 4, - 7 => 7 @ 5, - 8 => 8 @ 6, - 9..=10 => 10 @ 7, - 11..=12 => 12 @ 8, - 13..=14 => 14 @ 9, - 15..=16 => 16 @ 10, - 17..=20 => 20 @ 11, - 21..=24 => 24 @ 12, - 25..=28 => 28 @ 13, - 29..=32 => 32 @ 14 - ) - } -} diff --git a/libs/simple/src/layout.rs b/libs/simple/src/layout.rs deleted file mode 100644 index 1dc88bd..0000000 --- a/libs/simple/src/layout.rs +++ /dev/null @@ -1,704 +0,0 @@ -//! The in memory layout of objects using the simple collector. -//! -//! All objects are allocated with a [GcHeader], -//! that contains type information along with some internal mark bits. -//! -//! ## Safety -//! Relying on this internal layout is incredibly unsafe. -//! -//! However, `zerogc-simple` is committed to a stable ABI, so this should (hopefully) -//! be relatively well documented. -use std::alloc::Layout; -use std::cell::Cell; -use std::ffi::c_void; -use std::marker::PhantomData; -use std::mem::{self}; - -use zerogc::vec::raw::{GcRawVec, IGcVec}; -use zerogc::{GcSafe, GcSimpleAlloc, Trace}; - -use zerogc_context::field_offset; -use zerogc_derive::{unsafe_gc_impl, NullTrace}; - -use crate::{CollectorId, DynTrace, MarkVisitor, RawMarkState}; -use std::ptr::NonNull; - -/// Everything but the lower 2 bits of mark data are unused -/// for CollectorId. -/// -/// This is because we assume `align(usize) >= 4` -const STATE_MASK: usize = 0b11; -/// The internal 'marking data' used for the simple collector. -/// -/// The internals of this structure are private. -/// As of right now, the simple collector has no extra room -/// for any user-defined metadata in this type. -#[repr(transparent)] -#[derive(NullTrace)] -pub struct SimpleMarkData { - /// We're assuming that the collector is single threaded (which is true for now) - data: Cell, - #[zerogc(unsafe_skip_trace)] - fmt: PhantomData, -} - -/// Marker type for an unknown header -pub(crate) struct UnknownHeader(()); - -/// The layout of an object's header -#[derive(Debug)] -pub struct HeaderLayout { - /// The overall size of the header - pub header_size: usize, - /// The offset of the 'common' header, - /// starting from the start of the real header - pub common_header_offset: usize, - marker: PhantomData<*mut H>, -} -impl Copy for HeaderLayout {} -impl Clone for HeaderLayout { - #[inline] - fn clone(&self) -> Self { - *self - } -} - -impl HeaderLayout { - #[inline] - pub(crate) const fn value_offset_from_common_header(&self, align: usize) -> usize { - self.value_offset(align) - self.common_header_offset - } - #[inline] - pub(crate) const fn into_unknown(self) -> HeaderLayout { - HeaderLayout { - header_size: self.header_size, - common_header_offset: self.common_header_offset, - marker: PhantomData, - } - } - /// The alignment of the header - /// - /// NOTE: All headers have the same alignment - pub const ALIGN: usize = std::mem::align_of::(); - #[inline] - pub(crate) const unsafe fn common_header(self, ptr: *mut H) -> *mut GcHeader { - (ptr as *mut u8).add(self.common_header_offset).cast() - } - #[inline] - #[allow(clippy::wrong_self_convention)] - pub(crate) const unsafe fn from_common_header(self, ptr: *mut GcHeader) -> *mut H { - (ptr as *mut u8).sub(self.common_header_offset).cast() - } - /// Get the header from the specified value pointer - /// - /// ## Safety - /// Undefined behavior if the pointer doesn't point to a an object - /// allocated in this collector (ie. doesn't have the appropriate header). - #[inline] - pub const unsafe fn from_value_ptr(self, ptr: *mut T) -> *mut H { - let align = std::mem::align_of_val(&*ptr); - (ptr as *mut u8).sub(self.value_offset(align)).cast() - } - /// Get the in-memory layout of the header (doesn't include the value) - #[inline] - pub const fn layout(&self) -> Layout { - unsafe { Layout::from_size_align_unchecked(self.header_size, Self::ALIGN) } - } - /// Get the offset of the value from the start of the header, - /// given the alignment of its value - #[inline] - pub const fn value_offset(&self, align: usize) -> usize { - let padding = self.layout().padding_needed_for(align); - self.header_size + padding - } -} - -impl SimpleMarkData { - /// Create mark data from a specified snapshot - #[inline] - pub fn from_snapshot(snapshot: SimpleMarkDataSnapshot) -> Self { - SimpleMarkData { - data: Cell::new(snapshot.packed()), - fmt: PhantomData, - } - } - #[inline] - pub(crate) fn update_raw_state(&self, updated: RawMarkState) { - let mut snapshot = self.load_snapshot(); - snapshot.state = updated; - self.data.set(snapshot.packed()); - } - /// Load a snapshot of the object's current marking state - #[inline] - pub fn load_snapshot(&self) -> SimpleMarkDataSnapshot { - unsafe { SimpleMarkDataSnapshot::from_packed(self.data.get()) } - } -} - -/// A single snapshot of the state of the [SimpleMarkData] -/// -/// This is needed because mark data can be updated by several threads, -/// possibly atomically. -pub struct SimpleMarkDataSnapshot { - pub(crate) state: RawMarkState, - #[cfg(feature = "multiple-collectors")] - pub(crate) collector_id_ptr: *mut CollectorId, -} -impl SimpleMarkDataSnapshot { - pub(crate) fn new(state: RawMarkState, collector_id_ptr: *mut CollectorId) -> Self { - #[cfg(feature = "multiple-collectors")] - { - SimpleMarkDataSnapshot { - state, - collector_id_ptr, - } - } - #[cfg(not(feature = "multiple-collectors"))] - { - drop(collector_id_ptr); // avoid warnings - SimpleMarkDataSnapshot { state } - } - } - #[inline] - fn packed(&self) -> usize { - let base: usize; - #[cfg(feature = "multiple-collectors")] - { - base = self.collector_id_ptr as *const CollectorId as usize; - } - #[cfg(not(feature = "multiple-collectors"))] - { - base = 0; - } - debug_assert_eq!(base & STATE_MASK, 0); - (base & !STATE_MASK) | (self.state as u8 as usize & STATE_MASK) - } - #[inline] - unsafe fn from_packed(packed: usize) -> Self { - let state = RawMarkState::from_byte((packed & STATE_MASK) as u8); - let id_bytes: usize = packed & !STATE_MASK; - #[cfg(feature = "multiple-collectors")] - { - let collector_id_ptr = id_bytes as *mut CollectorId; - SimpleMarkDataSnapshot { - state, - collector_id_ptr, - } - } - #[cfg(not(feature = "multiple-collectors"))] - { - SimpleMarkDataSnapshot { state } - } - } -} - -/// Marker for an unknown GC object -#[repr(C)] -pub struct DynamicObj; -unsafe_gc_impl!( - target => DynamicObj, - params => [], - NEEDS_TRACE => true, - NEEDS_DROP => true, - null_trace => never, - trace_template => |self, visitor| { - unreachable!() - } -); - -#[repr(C)] -pub(crate) struct BigGcObject { - pub(crate) header: NonNull, -} -impl BigGcObject { - #[inline] - pub unsafe fn header(&self) -> &GcHeader { - self.header.as_ref() - } - #[inline] - pub unsafe fn from_ptr(header: *mut GcHeader) -> BigGcObject { - debug_assert!(!header.is_null()); - BigGcObject { - header: NonNull::new_unchecked(header), - } - } -} -impl Drop for BigGcObject { - fn drop(&mut self) { - unsafe { - let type_info = self.header().type_info; - if let Some(func) = type_info.drop_func { - func( - (self.header.as_ptr() as *const u8 as *mut u8) - .add(type_info.value_offset_from_common_header) - .cast(), - ); - } - let layout = type_info.determine_total_layout(self.header.as_ptr()); - let actual_header = type_info - .header_layout() - .from_common_header(self.header.as_ptr()); - std::alloc::dealloc(actual_header.cast(), layout); - } - } -} -/// A header for a GC array object -#[repr(C)] -pub struct GcArrayHeader { - pub(crate) len: usize, - pub(crate) common_header: GcHeader, -} -impl GcArrayHeader { - pub(crate) const LAYOUT: HeaderLayout = HeaderLayout { - header_size: std::mem::size_of::(), - common_header_offset: field_offset!(GcArrayHeader, common_header), - marker: PhantomData, - }; -} -/// A header for a Gc vector -#[repr(C)] -pub struct GcVecHeader { - pub(crate) capacity: usize, - /* - * NOTE: Suffix must be transmutable to `GcArrayHeader` - * in order for `steal_as_array_unchecked` to work - */ - pub(crate) len: Cell, - pub(crate) common_header: GcHeader, -} -impl GcVecHeader { - pub(crate) const LAYOUT: HeaderLayout = HeaderLayout { - common_header_offset: field_offset!(GcVecHeader, common_header), - header_size: std::mem::size_of::(), - marker: PhantomData, - }; -} - -/// The raw representation of a vector in the simple collector -/// -/// NOTE: Length and capacity are stored implicitly in the [GcVecHeader] -#[repr(C)] -pub struct SimpleVecRepr<'gc, T: Sized> { - marker: PhantomData>, - header: NonNull, - ctx: &'gc crate::SimpleCollectorContext, -} -impl<'gc, T> SimpleVecRepr<'gc, T> { - #[inline] - pub(crate) unsafe fn from_raw_parts( - header: NonNull, - ctx: &'gc crate::SimpleCollectorContext, - ) -> Self { - SimpleVecRepr { - header, - ctx, - marker: PhantomData, - } - } - #[inline] - pub(crate) fn header(&self) -> *const GcVecHeader { - self.header.as_ptr() as *const _ - } -} -impl<'gc, T: GcSafe<'gc, crate::CollectorId>> Copy for SimpleVecRepr<'gc, T> {} -impl<'gc, T: GcSafe<'gc, crate::CollectorId>> Clone for SimpleVecRepr<'gc, T> { - #[inline] - fn clone(&self) -> Self { - *self - } -} -impl<'gc, T: GcSafe<'gc, crate::CollectorId>> Extend for SimpleVecRepr<'gc, T> { - #[inline] - fn extend>(&mut self, iter: I) { - let iter = iter.into_iter(); - self.reserve(iter.size_hint().1.unwrap_or(0)); - for val in iter { - self.push(val); - } - } -} -#[inherent::inherent] -unsafe impl<'gc, T: GcSafe<'gc, crate::CollectorId>> GcRawVec<'gc, T> for SimpleVecRepr<'gc, T> { - #[allow(dead_code)] - unsafe fn steal_as_array_unchecked(mut self) -> zerogc::GcArray<'gc, T, crate::CollectorId> { - /* - * Invalidate capacity - * NOTE: This should never be relied upon. - * It is already undefined behavior to use this vector - * after calling this method. - * This is just an extra check - */ - self.header.as_mut().capacity = 0; - zerogc::GcArray::from_raw_ptr(NonNull::new_unchecked(self.as_mut_ptr()), self.len()) - } - pub fn iter(&self) -> zerogc::vec::raw::RawVecIter<'gc, T, Self> - where - T: Copy; -} -#[inherent::inherent] -unsafe impl<'gc, T: GcSafe<'gc, crate::CollectorId>> IGcVec<'gc, T> for SimpleVecRepr<'gc, T> { - type Id = crate::CollectorId; - - #[inline] - pub fn len(&self) -> usize { - unsafe { (*self.header()).len.get() } - } - - #[inline] - pub unsafe fn set_len(&mut self, len: usize) { - debug_assert!(len <= self.capacity()); - (*self.header()).len.set(len); - } - - #[inline] - pub fn capacity(&self) -> usize { - unsafe { (*self.header()).capacity } - } - - #[inline] - pub fn with_capacity_in(capacity: usize, ctx: &'gc crate::SimpleCollectorContext) -> Self { - ctx.alloc_raw_vec_with_capacity::(capacity) - } - - #[inline] - pub fn reserve_in_place( - &mut self, - _additional: usize, - ) -> Result<(), zerogc::vec::raw::ReallocFailedError> { - // TODO: Can we reasonably implement this? - Err(zerogc::vec::raw::ReallocFailedError::Unsupported) - } - - #[inline] - pub unsafe fn as_ptr(&self) -> *const T { - (self.header.as_ptr() as *mut u8) - .add(GcVecHeader::LAYOUT.value_offset(std::mem::align_of::())) - .cast() - } - - #[inline] - pub fn context(&self) -> &'gc crate::SimpleCollectorContext { - self.ctx - } - - // Default methods: - pub unsafe fn as_mut_ptr(&mut self) -> *mut T; - pub fn replace(&mut self, index: usize, val: T) -> T; - pub fn set(&mut self, index: usize, val: T); - pub fn extend_from_slice(&mut self, src: &[T]) - where - T: Copy; - pub fn push(&mut self, val: T); - pub fn pop(&mut self) -> Option; - pub fn swap_remove(&mut self, index: usize) -> T; - pub fn reserve(&mut self, additional: usize); - pub fn is_empty(&self) -> bool; - pub fn new_in(ctx: &'gc crate::SimpleCollectorContext) -> Self; - pub fn copy_from_slice(src: &[T], ctx: &'gc crate::SimpleCollectorContext) -> Self - where - T: Copy; - pub fn from_vec(src: Vec, ctx: &'gc crate::SimpleCollectorContext) -> Self; - pub fn get(&mut self, index: usize) -> Option - where - T: Copy; - pub unsafe fn as_slice_unchecked(&self) -> &[T]; -} -unsafe_gc_impl!( - target => SimpleVecRepr<'gc, T>, - params => ['gc, T: GcSafe<'gc, crate::CollectorId>], - bounds => { - GcRebrand => { where T: zerogc::GcRebrand<'new_gc, crate::CollectorId>, - T::Branded: Sized + zerogc::GcSafe<'new_gc, crate::CollectorId> }, - }, - NEEDS_TRACE => T::NEEDS_TRACE, - NEEDS_DROP => T::NEEDS_DROP, - null_trace => { where T: ::zerogc::NullTrace }, - trace_mut => |self, visitor| { - // Trace our innards - unsafe { - let start: *mut T = self.as_ptr() as *const T as *mut T; - for i in 0..self.len() { - visitor.trace(&mut *start.add(i))?; - } - } - Ok(()) - }, - trace_immutable => |self, visitor| { - // Trace our innards - unsafe { - let start: *mut T = self.as_ptr() as *const T as *mut T; - for i in 0..self.len() { - visitor.trace_immutable(&*start.add(i))?; - } - } - Ok(()) - }, - branded_type => SimpleVecRepr<'new_gc, T::Branded>, - collector_id => CollectorId -); - -/// A header for a GC object -/// -/// This is shared between both small arenas -/// and fallback alloc vis `BigGcObject` -#[repr(C)] -pub struct GcHeader { - /// The type information - pub type_info: &'static GcType, - /// The mark data - pub mark_data: SimpleMarkData, -} -impl GcHeader { - /// The layout of the header - pub const LAYOUT: HeaderLayout = HeaderLayout { - header_size: std::mem::size_of::(), - common_header_offset: 0, - marker: PhantomData, - }; - /// Get the collector id associated with this object. - #[inline] - pub fn collector_id(&self) -> &'_ crate::CollectorId { - #[cfg(feature = "multiple-collectors")] - { - unsafe { &*self.mark_data.load_snapshot().collector_id_ptr } - } - #[cfg(not(feature = "multiple-collectors"))] - { - const ID: CollectorId = unsafe { CollectorId::from_raw(PhantomData) }; - &ID - } - } - /// Create a new header - #[inline] - pub fn new(type_info: &'static GcType, mark_data: SimpleMarkData) -> Self { - GcHeader { - type_info, - mark_data, - } - } - /// A pointer to the header's value - #[inline] - pub fn value(&self) -> *mut c_void { - unsafe { - (self as *const GcHeader as *mut GcHeader as *mut u8) - // NOTE: This takes into account the possibility of `BigGcObject` - .add(self.type_info.value_offset_from_common_header) - .cast::() - } - } - /// Get the [GcHeader] for the specified value, assuming that its been allocated by this collector. - /// - /// ## Safety - /// Assumes the value was allocated in the simple collector. - #[inline] - pub unsafe fn from_value_ptr(ptr: *mut T) -> *mut GcHeader { - GcHeader::LAYOUT.from_value_ptr(ptr) - } - #[inline] - pub(crate) fn raw_mark_state(&self) -> RawMarkState { - // TODO: Is this safe? Couldn't it be accessed concurrently? - self.mark_data.load_snapshot().state - } - #[inline] - pub(crate) fn update_raw_mark_state(&self, raw_state: RawMarkState) { - self.mark_data.update_raw_state(raw_state); - } -} -/// Layout information on a [GcType] -pub enum GcTypeLayout { - /// A type with a fixed, statically-known layout - Fixed(Layout), - /// An array, whose size can vary at runtime - Array { - /// The fixed layout of elements in the array - /// - /// The overall alignment of the array is equal to the alignment of each element, - /// however the size may vary at runtime. - element_layout: Layout, - }, - /// A vector, whose capacity can vary from instance to instance - Vec { - /// The fixed layout of elements in the vector. - element_layout: Layout, - }, -} - -/// A type used by GC -#[repr(C)] -pub struct GcType { - /// Information on the type's layout - pub layout: GcTypeLayout, - /// The offset of the value from the start of the header - /// - /// This varies depending on the type's alignment - pub value_offset_from_common_header: usize, - /// The function to trace the type, or `None` if it doesn't need to be traced - pub trace_func: Option, - /// The function to drop the type, or `None` if it doesn't need to be dropped - pub drop_func: Option, -} -impl GcType { - #[inline] - fn align(&self) -> usize { - match self.layout { - GcTypeLayout::Fixed(fixed) => fixed.align(), - GcTypeLayout::Array { element_layout } | GcTypeLayout::Vec { element_layout } => { - element_layout.align() - } - } - } - pub(crate) fn header_layout(&self) -> HeaderLayout { - match self.layout { - GcTypeLayout::Fixed(_) => GcHeader::LAYOUT.into_unknown(), - GcTypeLayout::Array { .. } => GcArrayHeader::LAYOUT.into_unknown(), - GcTypeLayout::Vec { .. } => GcVecHeader::LAYOUT.into_unknown(), - } - } - #[inline] - unsafe fn determine_size(&self, header: *mut GcHeader) -> usize { - match self.layout { - GcTypeLayout::Fixed(layout) => layout.size(), - GcTypeLayout::Array { element_layout } => { - let header = GcArrayHeader::LAYOUT.from_common_header(header); - element_layout.repeat((*header).len).unwrap().0.size() - } - GcTypeLayout::Vec { element_layout } => { - let header = GcVecHeader::LAYOUT.from_common_header(header); - element_layout.repeat((*header).capacity).unwrap().0.size() - } - } - } - #[inline] - pub(crate) unsafe fn determine_total_size(&self, header: *mut GcHeader) -> usize { - self.determine_total_layout(header).size() - } - #[inline] - pub(crate) unsafe fn determine_total_layout(&self, header: *mut GcHeader) -> Layout { - self.header_layout() - .layout() - .extend(Layout::from_size_align_unchecked( - self.determine_size(header), - self.align(), - )) - .unwrap() - .0 - .pad_to_align() - } - /// Get the [GcType] for the specified `Sized` type - #[inline] - pub const fn for_regular<'gc, T: GcSafe<'gc, crate::CollectorId>>() -> &'static Self { - ::STATIC_TYPE - } -} - -pub(crate) trait StaticVecType { - const STATIC_VEC_TYPE: &'static GcType; -} -impl<'gc, T: GcSafe<'gc, crate::CollectorId>> StaticVecType for T { - const STATIC_VEC_TYPE: &'static GcType = &GcType { - layout: GcTypeLayout::Vec { - element_layout: Layout::new::(), - }, - value_offset_from_common_header: { - // We have same alignment as our members - let align = std::mem::align_of::(); - GcArrayHeader::LAYOUT.value_offset_from_common_header(align) - }, - trace_func: if ::NEEDS_TRACE { - Some({ - unsafe fn visit(val: *mut c_void, visitor: &mut MarkVisitor) { - let len = (*GcVecHeader::LAYOUT.from_value_ptr(val as *mut T)) - .len - .get(); - let slice = std::slice::from_raw_parts_mut(val as *mut T, len); - let Ok(()) = <[T] as Trace>::trace(slice, visitor); - } - visit:: as unsafe fn(*mut c_void, &mut MarkVisitor) - }) - } else { - None - }, - drop_func: if T::NEEDS_DROP { - Some({ - unsafe fn drop_gc_vec<'gc, T: GcSafe<'gc, crate::CollectorId>>(val: *mut c_void) { - let len = (*GcVecHeader::LAYOUT.from_value_ptr(val as *mut T)) - .len - .get(); - std::ptr::drop_in_place::<[T]>(std::ptr::slice_from_raw_parts_mut( - val as *mut T, - len, - )); - } - drop_gc_vec:: as unsafe fn(*mut c_void) - }) - } else { - None - }, - }; -} -pub(crate) trait StaticGcType { - const STATIC_TYPE: &'static GcType; -} -impl<'gc, T: GcSafe<'gc, crate::CollectorId>> StaticGcType for [T] { - const STATIC_TYPE: &'static GcType = &GcType { - layout: GcTypeLayout::Array { - element_layout: Layout::new::(), - }, - value_offset_from_common_header: { - GcArrayHeader::LAYOUT.value_offset_from_common_header(std::mem::align_of::()) - }, - trace_func: if ::NEEDS_TRACE { - Some({ - unsafe fn visit(val: *mut c_void, visitor: &mut MarkVisitor) { - let header = GcArrayHeader::LAYOUT.from_value_ptr(val as *mut T); - let len = (*header).len; - let slice = std::slice::from_raw_parts_mut(val as *mut T, len); - let Ok(()) = <[T] as Trace>::trace(slice, visitor); - } - visit:: as unsafe fn(*mut c_void, &mut MarkVisitor) - }) - } else { - None - }, - drop_func: if ::NEEDS_DROP { - Some({ - unsafe fn drop_gc_slice<'gc, T: GcSafe<'gc, crate::CollectorId>>(val: *mut c_void) { - let len = (*GcArrayHeader::LAYOUT.from_value_ptr(val as *mut T)).len; - std::ptr::drop_in_place::<[T]>(std::ptr::slice_from_raw_parts_mut( - val as *mut T, - len, - )); - } - drop_gc_slice:: as unsafe fn(*mut c_void) - }) - } else { - None - }, - }; -} -impl<'gc, T: GcSafe<'gc, crate::CollectorId>> StaticGcType for T { - const STATIC_TYPE: &'static GcType = &GcType { - layout: GcTypeLayout::Fixed(Layout::new::()), - value_offset_from_common_header: { - GcHeader::LAYOUT.value_offset_from_common_header(std::mem::align_of::()) - }, - trace_func: if ::NEEDS_TRACE { - Some(unsafe { - mem::transmute::<_, unsafe fn(*mut c_void, &mut MarkVisitor)>( - ::trace as fn(&mut T, &mut MarkVisitor), - ) - }) - } else { - None - }, - drop_func: if ::NEEDS_DROP { - unsafe { - Some(mem::transmute::<_, unsafe fn(*mut c_void)>( - std::ptr::drop_in_place:: as unsafe fn(*mut T), - )) - } - } else { - None - }, - }; -} diff --git a/libs/simple/src/lib.rs b/libs/simple/src/lib.rs deleted file mode 100644 index f004a17..0000000 --- a/libs/simple/src/lib.rs +++ /dev/null @@ -1,1282 +0,0 @@ -//! The simplest implementation of zerogc's garbage collection. -//! -//! Uses mark/sweep collection. This shares shadow stack code with the `zerogc-context` -//! crate, and thus [SimpleCollector] is actually a type alias for that crate. -//! -//! ## Internals -//! The internal layout information is public, -//! and available through the (layout module)[`self::layout`]. -//! -//! The garbage collector needs to store some dynamic type information at runtime, -//! to allow dynamic dispatch to trace and drop functions. -//! Each object's [GcHeader] has two fields: a [GcType] and some internal mark data. -//! -//! The mark data is implementation-internal. However, the header as a whole is `repr(C)` -//! and the type information -//! -//! Sometimes, users need to store their own type metadata for other purposes. -//! TODO: Allow users to do this. -#![deny( - missing_docs, // The 'simple' implementation needs to document its public API -)] -#![feature( - alloc_layout_extra, // Used for GcObject::from_raw - never_type, // Used for errors (which are currently impossible) - negative_impls, // impl !Send is much cleaner than PhantomData> - exhaustive_patterns, // Allow exhaustive matching against never - const_alloc_layout, // Used for StaticType - new_uninit, // Until Rust has const generics, this is how we init arrays.. - ptr_metadata, // Needed to abstract over Sized/unsized types - // Used for const layout computation: - const_mut_refs, - const_align_of_val, - // Needed for field_offset! - const_refs_to_cell, - // Used instead of drain_filter - extract_if, -)] -#![allow( - /* - * TODO: Should we be relying on vtable address stability? - * It seems safe as long as we reuse the same pointer.... - */ - clippy::vtable_address_comparisons, -)] -use std::alloc::Layout; -use std::any::TypeId; -#[cfg(not(feature = "multiple-collectors"))] -use std::marker::PhantomData; -use std::ops::{Deref, DerefMut}; -use std::ptr::{DynMetadata, NonNull, Pointee}; -#[cfg(not(feature = "multiple-collectors"))] -use std::sync::atomic::AtomicPtr; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::Arc; - -use slog::{debug, FnValue, Logger}; - -use zerogc::{GcSafe, GcVisitor, Trace}; - -use zerogc_context::utils::{MemorySize, ThreadId}; - -use crate::alloc::{SmallArena, SmallArenaList}; -use crate::layout::{ - BigGcObject, GcArrayHeader, GcHeader, GcType, GcTypeLayout, GcVecHeader, HeaderLayout, - SimpleMarkData, SimpleMarkDataSnapshot, SimpleVecRepr, StaticGcType, StaticVecType, -}; - -use std::cell::Cell; -use std::ffi::c_void; -use zerogc::vec::raw::GcRawVec; -use zerogc_context::collector::RawSimpleAlloc; -use zerogc_context::handle::{GcHandleList, RawHandleImpl}; -use zerogc_context::{ - CollectionManager as AbstractCollectionManager, CollectorContext, - RawContext as AbstractRawContext, -}; - -#[cfg(feature = "small-object-arenas")] -mod alloc; -#[cfg(not(feature = "small-object-arenas"))] -mod alloc { - use crate::layout::UnknownHeader; - use std::alloc::Layout; - - pub const fn fits_small_object(_layout: Layout) -> bool { - false - } - pub struct SmallArena; - impl SmallArena { - pub(crate) fn add_free(&self, _free: *mut UnknownHeader) { - unimplemented!() - } - pub(crate) fn alloc(&self) -> std::ptr::NonNull { - unimplemented!() - } - } - pub struct SmallArenaList; - impl SmallArenaList { - // Create dummy - pub fn new() -> Self { - SmallArenaList - } - pub fn find(&self, _layout: Layout) -> Option<&SmallArena> { - None - } - } -} -pub mod layout; - -/// The configuration for a garbage collection -pub struct GcConfig { - /// Whether to always force a collection at safepoints, - /// regardless of whether the heuristics say. - pub always_force_collect: bool, - /// The initial threshold to trigger garbage collection (in bytes) - pub initial_threshold: usize, -} -impl Default for GcConfig { - fn default() -> Self { - GcConfig { - always_force_collect: false, - initial_threshold: 2048, - } - } -} - -/// The alignment of the singleton empty vector -const EMPTY_VEC_ALIGNMENT: usize = std::mem::align_of::(); - -#[cfg(feature = "sync")] -type RawContext = zerogc_context::state::sync::RawContext; -#[cfg(feature = "sync")] -type CollectionManager = zerogc_context::state::sync::CollectionManager; -#[cfg(not(feature = "sync"))] -type RawContext = zerogc_context::state::nosync::RawContext; -#[cfg(not(feature = "sync"))] -type CollectionManager = zerogc_context::state::nosync::CollectionManager; - -/// A "simple" garbage collector -pub type SimpleCollector = ::zerogc_context::CollectorRef; -/// The context of a simple collector -pub type SimpleCollectorContext = ::zerogc_context::CollectorContext; -/// The id for a simple collector -pub type CollectorId = ::zerogc_context::CollectorId; -/// A garbage collected pointer, allocated in the "simple" collector -pub type Gc<'gc, T> = ::zerogc::Gc<'gc, T, CollectorId>; -/// A garbage collected array, allocated in the "simple" collector -pub type GcArray<'gc, T> = ::zerogc::array::GcArray<'gc, T, CollectorId>; -/// A garbage colelcted vector, allocated in the "simple" collector -pub type GcVec<'gc, T> = ::zerogc::vec::GcVec<'gc, T, CollectorId>; - -#[cfg(not(feature = "multiple-collectors"))] -static GLOBAL_COLLECTOR: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); - -unsafe impl RawSimpleAlloc for RawSimpleCollector { - #[inline] - unsafe fn alloc_uninit<'gc, T>(context: &'gc SimpleCollectorContext) -> *mut T - where - T: GcSafe<'gc, crate::CollectorId>, - { - let (_header, ptr) = context.collector().heap.allocator.alloc_layout( - GcHeader::LAYOUT, - Layout::new::(), - T::STATIC_TYPE, - ); - ptr as *mut T - } - - unsafe fn alloc_uninit_slice<'gc, T>(context: &'gc CollectorContext, len: usize) -> *mut T - where - T: GcSafe<'gc, crate::CollectorId>, - { - let (header, ptr) = context.collector().heap.allocator.alloc_layout( - GcArrayHeader::LAYOUT, - Layout::array::(len).unwrap(), - <[T] as StaticGcType>::STATIC_TYPE, - ); - (*header).len = len; - ptr.cast() - } - - fn alloc_raw_vec_with_capacity<'gc, T>( - context: &'gc CollectorContext, - capacity: usize, - ) -> Self::RawVec<'gc, T> - where - T: GcSafe<'gc, crate::CollectorId>, - { - if capacity == 0 && std::mem::align_of::() <= EMPTY_VEC_ALIGNMENT { - let header = context.collector().heap.empty_vec(); - // NOTE: Assuming header is already initialized - unsafe { - debug_assert_eq!((*header).len.get(), 0); - debug_assert_eq!((*header).capacity, 0); - return self::layout::SimpleVecRepr::from_raw_parts( - NonNull::new_unchecked(header), - context, - ); - } - } - let (header, value_ptr) = context.collector().heap.allocator.alloc_layout( - GcVecHeader::LAYOUT, - Layout::array::(capacity).unwrap(), - ::STATIC_VEC_TYPE, - ); - unsafe { - (*header).capacity = capacity; - (*header).len.set(0); - let res = self::layout::SimpleVecRepr::from_raw_parts( - NonNull::new_unchecked(header), - context, - ); - debug_assert_eq!(res.as_ptr(), value_ptr as *mut T as *const T,); - res - } - } -} - -#[doc(hidden)] // NOTE: Needs be public for RawCollectorImpl -pub unsafe trait DynTrace { - fn trace(&mut self, visitor: &mut MarkVisitor); -} -unsafe impl DynTrace for T { - fn trace(&mut self, visitor: &mut MarkVisitor) { - let Ok(()) = self.trace(visitor); - } -} - -unsafe impl RawHandleImpl for RawSimpleCollector { - type TypeInfo = GcType; - - #[inline] - fn type_info_of<'gc, T: GcSafe<'gc, CollectorId>>() -> &'static Self::TypeInfo { - ::STATIC_TYPE - } - - #[inline] - fn resolve_type_info<'gc, T: ?Sized + GcSafe<'gc, CollectorId>>( - gc: zerogc::Gc<'gc, T, CollectorId>, - ) -> &'static Self::TypeInfo { - unsafe { (*GcHeader::from_value_ptr(gc.as_raw_ptr())).type_info } - } - - #[inline] - fn handle_list(&self) -> &GcHandleList { - &self.handle_list - } -} - -/// A wrapper for [GcHandleList] that implements [DynTrace] -#[repr(transparent)] -struct GcHandleListWrapper(GcHandleList); -unsafe impl DynTrace for GcHandleListWrapper { - fn trace(&mut self, visitor: &mut MarkVisitor) { - unsafe { - let Ok(()) = self.0.trace::<_, !>(|raw_ptr, type_info| { - let header = &mut *GcHeader::from_value_ptr(raw_ptr); - // Mark grey - header.update_raw_mark_state(MarkState::Grey.to_raw(visitor.inverted_mark)); - // Visit innards - if let Some(func) = type_info.trace_func { - func(header.value(), visitor); - } - // Mark black - header.update_raw_mark_state(MarkState::Black.to_raw(visitor.inverted_mark)); - Ok(()) - }); - } - } -} - -struct GcHeap { - config: Arc, - threshold: AtomicUsize, - allocator: SimpleAlloc, - // NOTE: This is only public so it can be traced - cached_empty_vec: Cell>, -} -impl GcHeap { - fn new(config: Arc) -> GcHeap { - GcHeap { - threshold: AtomicUsize::new(if config.always_force_collect { - 0 - } else { - config.initial_threshold - }), - allocator: SimpleAlloc::new(), - config, - cached_empty_vec: Cell::new(None), - } - } - #[inline] - pub fn empty_vec(&self) -> *mut GcVecHeader { - match self.cached_empty_vec.get() { - Some(cached) => cached, - None => { - let res = self.create_empty_vec(); - self.cached_empty_vec.set(Some(self.create_empty_vec())); - res - } - } - } - #[cold] - fn create_empty_vec(&self) -> *mut GcVecHeader { - const DUMMY_LAYOUT: Layout = - unsafe { Layout::from_size_align_unchecked(0, EMPTY_VEC_ALIGNMENT) }; - const DUMMY_TYPE: GcType = GcType { - layout: GcTypeLayout::Vec { - element_layout: DUMMY_LAYOUT, - }, - value_offset_from_common_header: GcVecHeader::LAYOUT - .value_offset_from_common_header(EMPTY_VEC_ALIGNMENT), - drop_func: None, - trace_func: None, - }; - let (header, _) = - self.allocator - .alloc_layout(GcVecHeader::LAYOUT, DUMMY_LAYOUT, &DUMMY_TYPE); - unsafe { - (*header).capacity = 0; - (*header).len.set(0); - header - } - } - #[inline] - fn should_collect_relaxed(&self) -> bool { - /* - * Double check that 'config.always_force_collect' implies - * a threshold of zero. - * - * NOTE: This is not part of the ABI, it's only for internal debugging - */ - if cfg!(debug_assertions) && self.config.always_force_collect { - debug_assert_eq!(self.threshold.load(Ordering::Relaxed), 0); - } - /* - * Going with relaxed ordering because it's not essential - * that we see updates immediately. - * Eventual consistency should be enough to eventually - * trigger a collection. - * - * This is much cheaper on ARM (since it avoids a fence) - * and is much easier to use with a JIT. - * A JIT will insert these very often, so it's important to make - * them fast! - */ - self.allocator.allocated_size.load(Ordering::Relaxed) - >= self.threshold.load(Ordering::Relaxed) - } -} - -/// The abstract specification of a [Lock], -/// shared between thread-safe and thread-unsafe code -trait ILock<'a, T> { - type Guard: Sized + Deref + DerefMut + 'a; - fn lock(&'a self) -> Self::Guard; - fn get_mut(&'a mut self) -> &'a mut T; -} - -#[cfg(feature = "sync")] -struct Lock(::parking_lot::Mutex); -#[cfg(feature = "sync")] -impl From for Lock { - fn from(val: T) -> Self { - Lock(::parking_lot::Mutex::new(val)) - } -} -#[cfg(feature = "sync")] -impl<'a, T: 'a> ILock<'a, T> for Lock { - type Guard = ::parking_lot::MutexGuard<'a, T>; - - #[inline] - fn lock(&'a self) -> Self::Guard { - self.0.lock() - } - - #[inline] - fn get_mut(&'a mut self) -> &'a mut T { - self.0.get_mut() - } -} - -#[cfg(not(feature = "sync"))] -struct Lock(::std::cell::RefCell); -#[cfg(not(feature = "sync"))] -impl From for Lock { - fn from(val: T) -> Self { - Lock(::std::cell::RefCell::new(val)) - } -} -#[cfg(not(feature = "sync"))] -impl<'a, T: 'a> ILock<'a, T> for Lock { - type Guard = ::std::cell::RefMut<'a, T>; - - #[inline] - fn lock(&'a self) -> Self::Guard { - self.0.borrow_mut() - } - - #[inline] - fn get_mut(&'a mut self) -> &'a mut T { - self.0.get_mut() - } -} - -/// The thread-safe implementation of an allocator -/// -/// Most allocations should avoid locking. -pub(crate) struct SimpleAlloc { - collector_id: Option, - small_arenas: SmallArenaList, - big_objects: Lock>, - small_objects: Lock>, - /// Whether the meaning of the mark bit is currently inverted. - /// - /// This flips every collection - mark_inverted: AtomicBool, - allocated_size: AtomicUsize, -} -#[derive(Debug)] -struct TargetLayout { - header_layout: HeaderLayout, - value_offset: usize, - overall_layout: Layout, -} -impl SimpleAlloc { - fn new() -> SimpleAlloc { - SimpleAlloc { - collector_id: None, - allocated_size: AtomicUsize::new(0), - small_arenas: SmallArenaList::new(), - big_objects: Lock::from(Vec::new()), - small_objects: Lock::from(Vec::new()), - mark_inverted: AtomicBool::new(false), - } - } - #[inline] - fn allocated_size(&self) -> usize { - self.allocated_size.load(Ordering::Acquire) - } - #[inline] - fn add_allocated_size(&self, amount: usize) { - self.allocated_size.fetch_add(amount, Ordering::AcqRel); - } - #[inline] - fn set_allocated_size(&self, amount: usize) { - self.allocated_size.store(amount, Ordering::Release) - } - #[inline] - fn mark_inverted(&self) -> bool { - self.mark_inverted.load(Ordering::Acquire) - } - #[inline] - fn set_mark_inverted(&self, b: bool) { - self.mark_inverted.store(b, Ordering::Release) - } - - #[inline] - fn alloc_layout( - &self, - header_layout: HeaderLayout, - value_layout: Layout, - static_type: &'static GcType, - ) -> (*mut H, *mut u8) { - let collector_id_ptr = match self.collector_id { - Some(ref collector) => collector, - None => { - #[cfg(debug_assertions)] - { - unreachable!("Invalid collector id") - } - #[cfg(not(debug_assertions))] - { - unsafe { std::hint::unreachable_unchecked() } - } - } - }; - let (mut overall_layout, value_offset) = - header_layout.layout().extend(value_layout).unwrap(); - debug_assert_eq!( - header_layout.value_offset(value_layout.align()), - value_offset - ); - overall_layout = overall_layout.pad_to_align(); - debug_assert_eq!( - value_offset, - header_layout.value_offset(value_layout.align()) - ); - let target_layout = TargetLayout { - header_layout, - value_offset, - overall_layout, - }; - let (header, value_ptr) = - if let Some(arena) = self.small_arenas.find(target_layout.overall_layout) { - unsafe { self.alloc_layout_small(arena, target_layout) } - } else { - self.alloc_layout_big(target_layout) - }; - unsafe { - header_layout.common_header(header).write(GcHeader::new( - static_type, - SimpleMarkData::from_snapshot(SimpleMarkDataSnapshot::new( - MarkState::White.to_raw(self.mark_inverted()), - collector_id_ptr as *const _ as *mut _, - )), - )); - } - (header, value_ptr) - } - #[inline] - unsafe fn alloc_layout_small( - &self, - arena: &SmallArena, - target_layout: TargetLayout, - ) -> (*mut H, *mut u8) { - let ptr = arena.alloc(); - debug_assert_eq!( - ptr.as_ptr() as usize % target_layout.header_layout.layout().align(), - 0 - ); - self.add_allocated_size(target_layout.overall_layout.size()); - { - let mut lock = self.small_objects.lock(); - lock.push( - (ptr.as_ptr() as *mut u8) - .add(target_layout.header_layout.common_header_offset) - .cast(), - ); - } - ( - ptr.as_ptr().cast(), - (ptr.as_ptr() as *mut u8).add(target_layout.value_offset), - ) - } - fn alloc_layout_big(&self, target_layout: TargetLayout) -> (*mut H, *mut u8) { - let header: *mut H; - let value_ptr = unsafe { - header = std::alloc::alloc(target_layout.overall_layout).cast(); - (header as *mut u8).add(target_layout.value_offset) - }; - { - unsafe { - let mut objs = self.big_objects.lock(); - let common_header = (header as *mut u8) - .add(target_layout.header_layout.common_header_offset) - .cast(); - objs.push(BigGcObject::from_ptr(common_header)); - } - self.add_allocated_size(target_layout.overall_layout.size()); - } - (header, value_ptr) - } - unsafe fn sweep(&self) { - let mut expected_size = self.allocated_size(); - let mut actual_size = 0; - // Clear small arenas - let was_mark_inverted = self.mark_inverted.load(Ordering::SeqCst); - self.small_objects - .lock() - .extract_if(|&mut common_header| { - let total_size = (*common_header) - .type_info - .determine_total_size(common_header); - match (*common_header).raw_mark_state().resolve(was_mark_inverted) { - MarkState::White => { - // Free the object, dropping if necessary - expected_size -= total_size; - true // Drain, implicitly adding to free list - } - MarkState::Grey => panic!("All grey objects should've been processed"), - MarkState::Black => { - /* - * Retain the object - * State will be implicitly set to white - * by inverting mark the meaning of the mark bits. - */ - actual_size += total_size; - false // keep - } - } - }) - .for_each(|freed_common_header| { - let type_info = (*freed_common_header).type_info; - if let Some(func) = type_info.drop_func { - func((*freed_common_header).value()); - } - let overall_layout = type_info.determine_total_layout(freed_common_header); - let actual_start = type_info - .header_layout() - .from_common_header(freed_common_header); - self.small_arenas - .find(overall_layout) - .unwrap() - .add_free(actual_start) - }); - // Clear large objects - debug_assert_eq!(was_mark_inverted, self.mark_inverted()); - self.big_objects - .lock() - .extract_if(|big_item| { - let total_size = big_item - .header() - .type_info - .determine_total_size(big_item.header.as_ptr()); - match big_item - .header() - .mark_data - .load_snapshot() - .state - .resolve(was_mark_inverted) - { - MarkState::White => { - // Free the object - expected_size -= total_size; - true // remove from list - } - MarkState::Grey => panic!("All gray objects should've been processed"), - MarkState::Black => { - /* - * Retain the object - * State will be implicitly set to white - * by inverting mark the meaning of the mark bits. - */ - actual_size += total_size; - false // Keep - } - } - }) - .for_each(drop); - /* - * Flip the meaning of the mark bit, - * implicitly resetting all Black (reachable) objects - * to White. - */ - self.set_mark_inverted(!was_mark_inverted); - assert_eq!(expected_size, actual_size); - self.set_allocated_size(actual_size); - } -} -unsafe impl Send for SimpleAlloc {} -/// We're careful to be thread safe here -/// -/// This isn't auto implemented because of the -/// raw pointer to the collector (we only use it as an id) -unsafe impl Sync for SimpleAlloc {} - -/// We're careful - I swear :D -unsafe impl Send for RawSimpleCollector {} -unsafe impl Sync for RawSimpleCollector {} - -/// The internal data for a simple collector -#[doc(hidden)] -pub struct RawSimpleCollector { - logger: Logger, - heap: GcHeap, - manager: CollectionManager, - /// Tracks object handles - handle_list: GcHandleList, - config: Arc, -} - -unsafe impl ::zerogc_context::collector::RawCollectorImpl for RawSimpleCollector { - type DynTracePtr = NonNull; - type Config = GcConfig; - - #[cfg(feature = "multiple-collectors")] - type Ptr = NonNull; - - #[cfg(not(feature = "multiple-collectors"))] - type Ptr = PhantomData<&'static Self>; - - type Manager = CollectionManager; - - type RawContext = RawContext; - - type RawVec<'gc, T: GcSafe<'gc, CollectorId>> = SimpleVecRepr<'gc, T>; - - const SINGLETON: bool = cfg!(not(feature = "multiple-collectors")); - - const SYNC: bool = cfg!(feature = "sync"); - - #[inline] - fn id_for_gc<'a, 'gc, T>(gc: &'a Gc<'gc, T>) -> &'a CollectorId - where - 'gc: 'a, - T: ?Sized + 'gc, - { - unsafe { - let header = GcHeader::from_value_ptr(gc.as_raw_ptr()); - (*header).collector_id() - } - } - - #[inline] - fn id_for_array<'a, 'gc, T>(array: &'a GcArray<'gc, T>) -> &'a CollectorId - where - 'gc: 'a, - { - unsafe { - let header = GcArrayHeader::LAYOUT.from_value_ptr(array.as_raw_ptr()); - (*header).common_header.collector_id() - } - } - - #[inline] - fn resolve_array_len(gc: &GcArray) -> usize { - unsafe { - let header = GcArrayHeader::LAYOUT.from_value_ptr(gc.as_raw_ptr()); - (*header).len - } - } - - #[inline] - unsafe fn as_dyn_trace_pointer(value: *mut T) -> Self::DynTracePtr { - debug_assert!(!value.is_null()); - NonNull::new_unchecked(std::mem::transmute::< - *mut dyn DynTrace, - *mut (dyn DynTrace + 'static), - >(value as *mut dyn DynTrace)) - } - - #[cfg(not(feature = "multiple-collectors"))] - fn init(config: GcConfig, _logger: Logger) -> NonNull { - panic!("Not a singleton") - } - - #[cfg(feature = "multiple-collectors")] - fn init(config: GcConfig, logger: Logger) -> NonNull { - // We're assuming its safe to create multiple (as given by config) - let raw = unsafe { RawSimpleCollector::with_logger(config, logger) }; - let mut collector = Arc::new(raw); - let raw_ptr = - unsafe { NonNull::new_unchecked(Arc::as_ptr(&collector) as *mut RawSimpleCollector) }; - Arc::get_mut(&mut collector) - .unwrap() - .heap - .allocator - .collector_id = Some(unsafe { CollectorId::from_raw(raw_ptr) }); - std::mem::forget(collector); // We own it as a raw pointer... - raw_ptr - } - - #[inline(always)] - unsafe fn gc_write_barrier<'gc, T, V>( - _owner: &Gc<'gc, T>, - _value: &Gc<'gc, V>, - _field_offset: usize, - ) where - T: GcSafe<'gc, CollectorId> + ?Sized, - V: GcSafe<'gc, CollectorId> + ?Sized, - { - // Simple GC doesn't need write barriers - } - #[inline] - fn logger(&self) -> &Logger { - &self.logger - } - #[inline] - fn manager(&self) -> &CollectionManager { - &self.manager - } - #[inline] - fn should_collect(&self) -> bool { - /* - * Use relaxed ordering, just like `should_collect` - * - * This is faster on ARM since it avoids a memory fence. - * More importantly this is easier for a JIT to implement inline!!! - * As of this writing cranelift doesn't even seem to support fences :o - */ - self.heap.should_collect_relaxed() || self.manager.should_trigger_collection() - } - #[inline] - fn allocated_size(&self) -> MemorySize { - MemorySize { - bytes: self.heap.allocator.allocated_size(), - } - } - #[inline] - unsafe fn perform_raw_collection(&self, contexts: &[*mut RawContext]) { - self.perform_raw_collection(contexts) - } -} -#[cfg(feature = "sync")] -unsafe impl ::zerogc_context::collector::SyncCollector for RawSimpleCollector {} -#[cfg(not(feature = "multiple-collectors"))] -unsafe impl zerogc_context::collector::SingletonCollector for RawSimpleCollector { - #[inline] - fn global_ptr() -> *const Self { - GLOBAL_COLLECTOR.load(Ordering::Acquire) - } - - fn init_global(config: GcConfig, logger: Logger) { - let marker_ptr = NonNull::dangling().as_ptr(); - /* - * There can only be one collector (due to configuration). - * - * Exchange with marker pointer while we're initializing. - */ - assert_eq!( - GLOBAL_COLLECTOR.compare_exchange( - std::ptr::null_mut(), - marker_ptr, - Ordering::SeqCst, - Ordering::SeqCst - ), - Ok(std::ptr::null_mut()), - "Collector already exists" - ); - let mut raw = Box::new(unsafe { RawSimpleCollector::with_logger(config, logger) }); - const GLOBAL_ID: CollectorId = unsafe { CollectorId::from_raw(PhantomData) }; - raw.heap.allocator.collector_id = Some(GLOBAL_ID); - // It shall reign forever! - let raw = Box::leak(raw); - assert_eq!( - GLOBAL_COLLECTOR.compare_exchange( - marker_ptr, - raw as *mut RawSimpleCollector, - Ordering::SeqCst, - Ordering::SeqCst - ), - Ok(marker_ptr), - "Unexpected modification" - ); - } -} -impl RawSimpleCollector { - unsafe fn with_logger(config: GcConfig, logger: Logger) -> Self { - let config = Arc::new(config); - RawSimpleCollector { - logger, - manager: CollectionManager::new(), - heap: GcHeap::new(Arc::clone(&config)), - handle_list: GcHandleList::new(), - config, - } - } - #[cold] - #[inline(never)] - unsafe fn perform_raw_collection(&self, contexts: &[*mut RawContext]) { - debug_assert!(self.manager.is_collecting()); - let roots = { - let mut roots: Vec<*mut dyn DynTrace> = Vec::new(); - for ctx in contexts.iter() { - roots.extend( - (**ctx) - .assume_valid_shadow_stack() - .reverse_iter() - .map(NonNull::as_ptr), - ); - } - roots.push( - &self.handle_list - // Cast to wrapper type - as *const GcHandleList as *const GcHandleListWrapper - // Make into virtual pointer - as *const dyn DynTrace as *mut dyn DynTrace, - ); - #[repr(transparent)] - struct CachedEmptyVec(GcVecHeader); - unsafe impl DynTrace for CachedEmptyVec { - fn trace(&mut self, visitor: &mut MarkVisitor) { - let cached_vec_header = self as *mut Self as *mut GcVecHeader; - unsafe { - visitor._trace_own_rawvec::<()>(cached_vec_header); - } - } - } - if let Some(cached_vec_header) = self.heap.cached_empty_vec.get() { - roots.push(cached_vec_header as *mut CachedEmptyVec as *mut dyn DynTrace) - } - roots - }; - let num_roots = roots.len(); - let mut task = CollectionTask { - config: &*self.config, - expected_collector: self.heap.allocator.collector_id.unwrap(), - roots, - heap: &self.heap, - grey_stack: if cfg!(feature = "implicit-grey-stack") { - Vec::new() - } else { - Vec::with_capacity(64) - }, - }; - let original_size = self.heap.allocator.allocated_size(); - task.run(); - let updated_size = self.heap.allocator.allocated_size(); - debug!( - self.logger, "Finished simple GC"; - "current_thread" => FnValue(|_| ThreadId::current()), - "num_roots" => num_roots, - "original_size" => %MemorySize { bytes: original_size }, - "memory_freed" => %MemorySize { bytes: original_size - updated_size }, - ); - } -} -struct CollectionTask<'a> { - config: &'a GcConfig, - expected_collector: CollectorId, - roots: Vec<*mut dyn DynTrace>, - heap: &'a GcHeap, - #[cfg_attr(feature = "implicit-grey-stack", allow(dead_code))] - grey_stack: Vec<*mut GcHeader>, -} -impl<'a> CollectionTask<'a> { - fn run(&mut self) { - // Mark - for &root in &self.roots { - let mut visitor = MarkVisitor { - expected_collector: self.expected_collector, - grey_stack: &mut self.grey_stack, - inverted_mark: self.heap.allocator.mark_inverted(), - }; - // Dynamically dispatched - unsafe { - (*root).trace(&mut visitor); - } - } - #[cfg(not(feature = "implicit-grey-stack"))] - unsafe { - let was_inverted_mark = self.heap.allocator.mark_inverted(); - while let Some(obj) = self.grey_stack.pop() { - debug_assert_eq!( - (*obj).raw_mark_state().resolve(was_inverted_mark), - MarkState::Grey - ); - let mut visitor = MarkVisitor { - expected_collector: self.expected_collector, - grey_stack: &mut self.grey_stack, - inverted_mark: was_inverted_mark, - }; - if let Some(trace) = (*obj).type_info.trace_func { - (trace)((*obj).value(), &mut visitor); - } - // Mark the object black now it's innards have been traced - (*obj).update_raw_mark_state(MarkState::Black.to_raw(was_inverted_mark)); - } - } - // Sweep - unsafe { self.heap.allocator.sweep() }; - let updated_size = self.heap.allocator.allocated_size(); - if self.config.always_force_collect { - assert_eq!(self.heap.threshold.load(Ordering::SeqCst), 0); - } else { - // Update the threshold to be 150% of currently used size - self.heap - .threshold - .store(updated_size + (updated_size / 2), Ordering::SeqCst); - } - } -} - -#[doc(hidden)] // NOTE: Needs be public for RawCollectorImpl -pub struct MarkVisitor<'a> { - expected_collector: CollectorId, - #[cfg_attr(feature = "implicit-grey-stack", allow(dead_code))] - grey_stack: &'a mut Vec<*mut GcHeader>, - /// If this meaning of the mark bit is currently inverted - /// - /// This flips every collection - inverted_mark: bool, -} -impl<'a> MarkVisitor<'a> { - fn visit_raw_gc( - &mut self, - obj: &mut GcHeader, - trace_func: impl FnOnce(&mut GcHeader, &mut MarkVisitor<'a>), - ) { - match obj.raw_mark_state().resolve(self.inverted_mark) { - MarkState::White => trace_func(obj, self), - MarkState::Grey => { - /* - * We've already pushed this object onto the gray stack - * It will be traversed eventually, so we don't need to do anything. - */ - } - MarkState::Black => { - /* - * We've already traversed this object. - * It's already known to be reachable - */ - } - } - } -} -unsafe impl GcVisitor for MarkVisitor<'_> { - type Err = !; - - #[inline] - unsafe fn trace_gc<'gc, T, Id>( - &mut self, - gc: &mut ::zerogc::Gc<'gc, T, Id>, - ) -> Result<(), Self::Err> - where - T: GcSafe<'gc, Id>, - Id: ::zerogc::CollectorId, - { - if TypeId::of::() == TypeId::of::() { - /* - * Since the `TypeId`s match, we know the generic `Id` - * matches our own `crate::CollectorId`. - * Therefore its safe to specific the generic `Id` into the - * `Gc` into its more specific type. - */ - let gc = std::mem::transmute::< - &mut ::zerogc::Gc<'gc, T, Id>, - &mut ::zerogc::Gc<'gc, T, crate::CollectorId>, - >(gc); - /* - * Check the collectors match. Otherwise we're mutating - * other people's data. - */ - assert_eq!(*gc.collector_id(), self.expected_collector); - self._trace_own_gc(gc); - Ok(()) - } else { - // Just ignore - Ok(()) - } - } - - unsafe fn trace_trait_object<'gc, T, Id>( - &mut self, - gc: &mut zerogc::Gc<'gc, T, Id>, - ) -> Result<(), Self::Err> - where - T: ?Sized - + GcSafe<'gc, Id> - + Pointee> - + zerogc::DynTrace<'gc, Id>, - Id: zerogc::CollectorId, - { - if TypeId::of::() == TypeId::of::() { - /* - * The TypeIds match, so this cast is safe. See `visit_gc` for details - */ - let gc = std::mem::transmute::< - &mut ::zerogc::Gc<'gc, T, Id>, - &mut ::zerogc::Gc<'gc, T, crate::CollectorId>, - >(gc); - let header = GcHeader::from_value_ptr(gc.as_raw_ptr()); - self._trace_own_gc_with::<_>( - GcHeader::from_value_ptr(gc.as_raw_ptr()), - true, - |val, visitor| { - if let Some(func) = (*header).type_info.trace_func { - func(val, visitor) - } - Ok(()) - }, - ); - Ok(()) - } else { - Ok(()) - } - } - - #[inline] - unsafe fn trace_vec<'gc, T, V>(&mut self, raw: &mut V) -> Result<(), Self::Err> - where - T: GcSafe<'gc, V::Id>, - V: GcRawVec<'gc, T>, - { - if TypeId::of::() == TypeId::of::() { - let raw = &mut *(raw as *mut V as *mut self::layout::SimpleVecRepr<'gc, T>); - assert_eq!( - *(*raw.header()).common_header.collector_id(), - self.expected_collector - ); - self._trace_own_rawvec::(raw.header() as *mut _); - Ok(()) - } else { - unreachable!("Can't trace {}", std::any::type_name::()); - } - } - - #[inline] - unsafe fn trace_array<'gc, T, Id>( - &mut self, - array: &mut ::zerogc::array::GcArray<'gc, T, Id>, - ) -> Result<(), Self::Err> - where - T: GcSafe<'gc, Id>, - Id: ::zerogc::CollectorId, - { - if TypeId::of::() == TypeId::of::() { - /* - * See comment in 'visit_gc'. - * Essentially this is a checked cast - */ - let array = std::mem::transmute::< - &mut ::zerogc::array::GcArray<'gc, T, Id>, - &mut ::zerogc::array::GcArray<'gc, T, crate::CollectorId>, - >(array); - /* - * Check the collectors match. Otherwise we're mutating - * other people's data. - */ - assert_eq!(*array.collector_id(), self.expected_collector); - /* - * NOTE: Must transmute instead of using Gc::from_raw - * because we don't satisfy the 'GcSafe' bound. - */ - let mut gc = std::mem::transmute::, ::zerogc::Gc<'gc, [T], CollectorId>>( - NonNull::from(array.as_slice()), - ); - self._trace_own_gc(&mut gc); - /* - * NOTE: Must transmute instead of using GcArray::from_raw - * because we don't satisfy the 'GcSafe' bound. - */ - *array = std::mem::transmute::, GcArray<'gc, T>>(NonNull::new_unchecked( - gc.value().as_ptr() as *mut T, - )); - Ok(()) - } else { - Ok(()) - } - } -} -impl MarkVisitor<'_> { - /// Visit a GC type whose [::zerogc::CollectorId] matches our own, - /// tracing it with the specified closure - /// - /// The type should implement `GcSafe<'gc, crate::CollectorId>`, - /// although that can't be proven at compile time. - /// - /// The caller should only use `GcVisitor::visit_gc()` - unsafe fn _trace_own_gc<'gc, T: Trace + ?Sized>(&mut self, gc: &mut Gc<'gc, T>) { - self._trace_own_gc_with( - GcHeader::from_value_ptr(gc.as_raw_ptr()), - T::NEEDS_TRACE, - |_value, visitor| ::trace(&mut *gc.as_raw_ptr(), visitor), - ) - } - unsafe fn _trace_own_rawvec(&mut self, header: *mut GcVecHeader) { - self._trace_own_gc_with( - &mut (*header).common_header, - T::NEEDS_TRACE, - |ptr, visitor| { - let slice = std::slice::from_raw_parts_mut(ptr as *mut T, (*header).len.get()); - for val in slice { - T::trace(val, visitor)?; - } - Ok(()) - }, - ); - } - /// Visit a GC type whose [::zerogc::CollectorId] matches our own, - /// tracing it with the specified closure - /// - /// The caller should only use `GcVisitor::visit_gc()` - #[inline] - unsafe fn _trace_own_gc_with( - &mut self, - header: *mut GcHeader, - needs_trace: bool, - trace_func: F, - ) where - F: FnOnce(*mut c_void, &mut MarkVisitor) -> Result<(), !>, - { - // Verify this again (should be checked by caller) - debug_assert_eq!(*(*header).collector_id(), self.expected_collector); - self.visit_raw_gc(&mut *header, |actual_header, visitor| { - debug_assert_eq!(actual_header as *const _, header as *const _); - let inverted_mark = visitor.inverted_mark; - #[allow(unused_variables)] - let val = (*header).value(); - if !needs_trace { - /* - * We don't need to mark this grey - * It has no internals that need to be traced. - * We can directly move it directly to the black set - */ - (*header).update_raw_mark_state(MarkState::Black.to_raw(inverted_mark)); - } else { - /* - * We need to mark this object grey and push it onto the grey stack. - * It will be processed later - */ - (*header).update_raw_mark_state(MarkState::Grey.to_raw(inverted_mark)); - #[cfg(not(feature = "implicit-grey-stack"))] - { - visitor - .grey_stack - .push(header as *const GcHeader as *mut GcHeader); - drop(trace_func); - } - #[cfg(feature = "implicit-grey-stack")] - { - /* - * The user wants an implicit grey stack using - * recursion. This risks stack overflow but can - * boost performance (See 9a9634d68a4933d). - * On some workloads this is fine. - */ - let Ok(()) = trace_func(val, visitor); - /* - * Mark the object black now it's innards have been traced - * NOTE: We do **not** do this with an implicit stack. - */ - (*header).update_raw_mark_state(MarkState::Black.to_raw(inverted_mark)); - } - } - }); - } -} - -/// The raw mark state of an object -/// -/// Every cycle the meaning of the white/black states -/// flips. This allows us to implicitly mark objects -/// without actually touching their bits :) -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -#[repr(u8)] -enum RawMarkState { - /// Normally this marks the white state - /// - /// If we're inverted, this marks black - Red = 0, - /// This always marks the grey state - /// - /// Inverting the mark bit doesn't affect the - /// grey state - Grey = 1, - /// Normally this marks the blue state - /// - /// If we're inverted, this marks white - Blue = 2, -} -impl RawMarkState { - #[inline] - fn from_byte(b: u8) -> Self { - assert!(b < 3); - unsafe { std::mem::transmute::(b) } - } - #[inline] - fn resolve(self, inverted_mark: bool) -> MarkState { - match (self, inverted_mark) { - (RawMarkState::Red, false) => MarkState::White, - (RawMarkState::Red, true) => MarkState::Black, - (RawMarkState::Grey, _) => MarkState::Grey, - (RawMarkState::Blue, false) => MarkState::Black, - (RawMarkState::Blue, true) => MarkState::White, - } - } -} - -/// The current mark state of the object -/// -/// See [Tri Color Marking](https://en.wikipedia.org/wiki/Tracing_garbage_collection#Tri-color_marking) -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -#[repr(u8)] -enum MarkState { - /// The object is in the "white set" and is a candidate for having its memory freed. - /// - /// Once all the objects have been marked, - /// all remaining white objects will be freed. - White = 0, - /// The object is in the gray set and needs to be traversed to look for reachable memory - /// - /// After being scanned this object will end up in the black set. - Grey = 1, - /// The object is in the black set and is reachable from the roots. - /// - /// This object cannot be freed. - Black = 2, -} -impl MarkState { - #[inline] - fn to_raw(self, inverted_mark: bool) -> RawMarkState { - match (self, inverted_mark) { - (MarkState::White, false) => RawMarkState::Red, - (MarkState::White, true) => RawMarkState::Blue, - (MarkState::Grey, _) => RawMarkState::Grey, - (MarkState::Black, false) => RawMarkState::Blue, - (MarkState::Black, true) => RawMarkState::Red, - } - } -} diff --git a/libs/simple/tests/arrays.rs b/libs/simple/tests/arrays.rs deleted file mode 100644 index d654f16..0000000 --- a/libs/simple/tests/arrays.rs +++ /dev/null @@ -1,105 +0,0 @@ -use slog::Logger; - -use zerogc::safepoint; -use zerogc::GcSimpleAlloc; -use zerogc_derive::Trace; - -use zerogc_simple::{ - CollectorId as SimpleCollectorId, Gc, GcArray, GcConfig, GcVec, SimpleCollector, -}; - -fn test_collector() -> SimpleCollector { - let mut config = GcConfig::default(); - config.always_force_collect = true; // Force collections for predictability - SimpleCollector::with_config(config, Logger::root(::slog::Discard, ::slog::o!())) -} - -#[derive(Trace, Copy, Clone, Debug)] -#[zerogc(copy, collector_ids(SimpleCollectorId))] -struct Dummy<'gc> { - val: usize, - inner: Option>>, -} - -#[test] -fn array() { - let collector = test_collector(); - let mut context = collector.into_context(); - let array1 = context.alloc_slice_fill_copy(5, 12u32); - assert_eq!(*array1.as_slice(), *vec![12u32; 5]); - safepoint!(context, ()); - const TEXT: &[u8] = b"all cows eat grass"; - let array_text = context.alloc_slice_copy(TEXT); - let array_none: GcArray> = context.alloc_slice_none(12); - for val in array_none.as_slice() { - assert_eq!(*val, None); - } - let array_text = safepoint!(context, array_text); - assert_eq!(array_text.as_slice(), TEXT); - let mut nested_trace = Vec::new(); - let mut last = None; - for i in 0..16 { - let obj = context.alloc(Dummy { - val: i, - inner: last, - }); - nested_trace.push(obj); - last = Some(obj); - } - let nested_trace = context.alloc_slice_copy(nested_trace.as_slice()); - let nested_trace: GcArray> = safepoint!(context, nested_trace); - for (idx, val) in nested_trace.as_slice().iter().enumerate() { - assert_eq!(val.val, idx, "Invalid val: {:?}", val); - if let Some(last) = val.inner { - assert_eq!(last.val, idx - 1); - } - } -} - -#[test] -fn vec() { - let collector = test_collector(); - let mut context = collector.into_context(); - let mut vec1 = context.alloc_vec(); - for _ in 0..5 { - vec1.push(12u32); - } - assert_eq!(*vec1.as_slice(), *vec![12u32; 5]); - drop(vec1); - safepoint!(context, ()); - const TEXT: &[u8] = b"all cows eat grass"; - let mut vec_text = context.alloc_vec(); - vec_text.extend_from_slice(TEXT); - let mut vec_none: GcVec> = context.alloc_vec_with_capacity(12); - for _ in 0..12 { - vec_none.push(None); - } - for val in vec_none.iter() { - assert_eq!(*val, None); - } - drop(vec_none); - let vec_text: GcVec = - GcVec::copy_from_slice(safepoint!(context, vec_text).as_slice(), &context); - assert_eq!(vec_text.as_slice(), TEXT); - let mut nested_trace: GcVec> = context.alloc_vec_with_capacity(3); - let mut last = None; - for i in 0..16 { - let obj = context.alloc(Dummy { - val: i, - inner: last, - }); - nested_trace.push(obj); - last = Some(obj); - } - drop(vec_text); - let nested_trace: GcVec> = GcVec::from_vec( - safepoint!(context, nested_trace).into_iter().collect(), - &context, - ); - for (idx, val) in nested_trace.iter().enumerate() { - assert_eq!(val.val, idx, "Invalid val: {:?}", val); - if let Some(last) = val.inner { - assert_eq!(last.val, idx - 1); - } - } -} diff --git a/libs/simple/tests/errors.rs b/libs/simple/tests/errors.rs deleted file mode 100644 index d577bb2..0000000 --- a/libs/simple/tests/errors.rs +++ /dev/null @@ -1,57 +0,0 @@ -use std::fmt::Debug; - -use zerogc_derive::Trace; - -use zerogc::array::GcString; -use zerogc::prelude::*; -use zerogc_simple::{CollectorId, Gc, SimpleCollector, SimpleCollectorContext as GcContext}; - -#[derive(Debug, thiserror::Error, Trace)] -#[zerogc(collector_ids(CollectorId))] -pub enum OurError<'gc> { - #[error("Bad gc string: {0}")] - BadGcString(GcString<'gc, CollectorId>), - #[error("Bad gc int: {0}")] - BadGcInt(Gc<'gc, i32>), - #[error("Bad non-gc string: {0}")] - BadOtherString(String), -} - -fn implicitly_alloc<'gc>(ctx: &'gc GcContext, val: i32) -> Result> { - match val { - 0 => Err(OurError::BadGcString(ctx.alloc_str("gc foo"))), - 1 => Err(OurError::BadOtherString(String::from("boxed foo"))), - 2 => Err(OurError::BadGcInt(ctx.alloc(15))), - _ => Ok(String::from("sensible result")), - } -} - -fn into_anyhow<'gc>(ctx: &'gc GcContext, val: i32) -> Result { - let s = implicitly_alloc(ctx, val).map_err(|e| ctx.alloc_error(e))?; - Ok(format!("Result: {}", s)) -} - -#[test] -fn test_errors() { - let collector = SimpleCollector::create(); - let ctx = collector.create_context(); - fn display_anyhow(e: anyhow::Error) -> String { - format!("{}", e) - } - assert_eq!( - into_anyhow(&ctx, 0).map_err(display_anyhow), - Err("Bad gc string: gc foo".into()) - ); - assert_eq!( - into_anyhow(&ctx, 1).map_err(display_anyhow), - Err("Bad non-gc string: boxed foo".into()) - ); - assert_eq!( - into_anyhow(&ctx, 2).map_err(display_anyhow), - Err("Bad gc int: 15".into()) - ); - assert_eq!( - into_anyhow(&ctx, 3).map_err(display_anyhow), - Ok("Result: sensible result".into()) - ); -} diff --git a/libs/simple/tests/trait_objects.rs b/libs/simple/tests/trait_objects.rs deleted file mode 100644 index fccd785..0000000 --- a/libs/simple/tests/trait_objects.rs +++ /dev/null @@ -1,87 +0,0 @@ -use core::cell::Cell; - -use zerogc::{safepoint, trait_object_trace, DynTrace, GcSimpleAlloc, Trace}; - -use slog::Logger; -use zerogc_simple::{CollectorId as SimpleCollectorId, Gc, GcConfig, SimpleCollector}; - -fn test_collector() -> SimpleCollector { - let mut config = GcConfig::default(); - config.always_force_collect = true; // Force collections for predictability - SimpleCollector::with_config(config, Logger::root(::slog::Discard, ::slog::o!())) -} - -trait Foo<'gc>: DynTrace<'gc, SimpleCollectorId> { - fn method(&self) -> i32; - fn validate(&self); -} -trait_object_trace!( - impl<'gc,> Trace for dyn Foo<'gc>; - Branded<'new_gc> => (dyn Foo<'new_gc> + 'new_gc), - collector_id => SimpleCollectorId, - gc_lifetime => 'gc -); - -fn foo<'gc, T: ?Sized + Trace + Foo<'gc>>(t: &T) -> i32 { - assert_eq!(t.method(), 12); - t.method() * 2 -} -fn bar<'gc>(gc: Gc<'gc, dyn Foo<'gc> + 'gc>) -> i32 { - foo(gc.value()) -} -#[derive(Trace)] -#[zerogc(collector_ids(SimpleCollectorId), unsafe_skip_drop)] -struct Bar<'gc> { - inner: Option>>, - val: Gc<'gc, i32>, -} -impl<'gc> Foo<'gc> for Bar<'gc> { - fn method(&self) -> i32 { - *self.val - } - fn validate(&self) { - assert_eq!(*self.val, 12); - assert_eq!(*self.inner.unwrap().val, 4); - } -} -impl<'gc> Drop for Bar<'gc> { - fn drop(&mut self) { - BAR_DROP_COUNT.with(|val| { - val.set(val.get() + 1); - }) - } -} - -thread_local! { - static BAR_DROP_COUNT: Cell = const { Cell::new(0) }; -} -#[test] -fn foo_bar() { - let collector = test_collector(); - let mut context = collector.into_context(); - let val = context.alloc(12); - let inner = context.alloc(Bar { - inner: None, - val: context.alloc(4), - }); - let gc: Gc<'_, dyn Foo<'_>> = context.alloc(Bar { - inner: Some(inner), - val, - }); - assert_eq!(bar(gc), 24); - // Should be traced correctly - let gc = safepoint!(context, gc); - assert_eq!( - BAR_DROP_COUNT.with(Cell::get), - 0, - "Expected Bar to be retained" - ); - gc.validate(); - // Trace inner, should end up dropping Bar - safepoint!(context, ()); - assert_eq!( - BAR_DROP_COUNT.with(Cell::get), - 2, - "Expected Bar to be dropped" - ); -} From bc9bac5a9172dd94e59cde4309f208c934ee49cc Mon Sep 17 00:00:00 2001 From: Techcable Date: Tue, 7 May 2024 17:14:39 -0700 Subject: [PATCH 22/35] Drop features unsupported in v0.3 The new version drops support for most features in v0.2 --- Cargo.toml | 19 +- libs/derive/Cargo.toml | 6 +- src/allocator.rs | 67 ---- src/array.rs | 294 ----------------- src/array/repr.rs | 187 ----------- src/epsilon.rs | 487 ----------------------------- src/epsilon/alloc.rs | 46 --- src/epsilon/alloc/arena.rs | 21 -- src/epsilon/handle.rs | 77 ----- src/epsilon/layout.rs | 325 ------------------- src/errors.rs | 126 -------- src/hash_map.rs | 14 - src/hash_map/indexmap.rs | 365 --------------------- src/lib.rs | 213 +------------ src/macros.rs | 77 +---- src/prelude.rs | 5 +- src/serde.rs | 384 ----------------------- src/serde/hack.rs | 302 ------------------ src/vec.rs | 585 ---------------------------------- src/vec/cell.rs | 151 --------- src/vec/raw.rs | 626 ------------------------------------- 21 files changed, 6 insertions(+), 4371 deletions(-) delete mode 100644 src/allocator.rs delete mode 100644 src/array.rs delete mode 100644 src/array/repr.rs delete mode 100644 src/epsilon.rs delete mode 100644 src/epsilon/alloc.rs delete mode 100644 src/epsilon/alloc/arena.rs delete mode 100644 src/epsilon/handle.rs delete mode 100644 src/epsilon/layout.rs delete mode 100644 src/errors.rs delete mode 100644 src/hash_map.rs delete mode 100644 src/hash_map/indexmap.rs delete mode 100644 src/serde.rs delete mode 100644 src/serde/hack.rs delete mode 100644 src/vec.rs delete mode 100644 src/vec/cell.rs delete mode 100644 src/vec/raw.rs diff --git a/Cargo.toml b/Cargo.toml index b6f00bf..9441f90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,7 @@ license = "MIT" edition = "2018" [features] -default = ["std", "epsilon", "epsilon-arena-alloc"] +default = ["std"] # Depend on the standard library (optional) # # This implements tracing for most standard library types. @@ -62,20 +62,3 @@ std = ["alloc"] # # This implements `Trace` for `Box` and collections like `Vec` alloc = [] -# Emulate the `core::alloc::Allocator` api -# -# NOTE: This doesn't *necessarily* require the 'alloc' -# feature (because the API itself is in 'core') -allocator-api = [] -# Our custom hashmap implementation -hashmap-impl = ["allocator-api", "hashbrown", "ahash"] -# Support a 'GcError' type that implements 'std::error::Error' -# by wrapping a 'GcHandle' -errors = [] -# Serde support -serde1 = ["serde", "zerogc-derive/__serde-internal", "arrayvec/serde", "indexmap/serde-1", "hashbrown/serde"] -# Support the "epsilon" no-op collector -epsilon = [] -# Configure the "epsilon" collector use arena allocation -# (on by default) -epsilon-arena-alloc = ["epsilon", "bumpalo"] diff --git a/libs/derive/Cargo.toml b/libs/derive/Cargo.toml index a9746de..7f6be20 100644 --- a/libs/derive/Cargo.toml +++ b/libs/derive/Cargo.toml @@ -12,8 +12,7 @@ readme = "../../README.md" proc-macro = true [dev-dependencies] -zerogc = { version = "0.2.0-alpha.7", path = "../..", features = ["serde1"] } -serde = { version = "1" } +zerogc = { version = "0.2.0-alpha.7", path = "../.." } [dependencies] # Proc macros @@ -27,6 +26,3 @@ proc-macro-kwargs = "0.1.1" indexmap = "1" itertools = "0.10.1" -[features] -# Indicates that zerogc was compiled with support for serde, -__serde-internal = [] diff --git a/src/allocator.rs b/src/allocator.rs deleted file mode 100644 index c4a9642..0000000 --- a/src/allocator.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! Emulate the `core::alloc::Allocator` API -//! -//! Constructing a `GcAllocWrapper` is `unsafe`, -//! because it is the caller's responsibility to ensure -//! the returned pointers are appropriately traced. -//! -//! If there are any interior pointers, -//! those must also be traced as well. - -use core::alloc::{AllocError, Allocator, Layout}; -use core::ptr::NonNull; - -use crate::GcSimpleAlloc; - -/// A wrapper for a `GcContext` that implements [core::alloc::Allocator] -/// by allocating `GcArray` -/// -/// ## Safety -/// Using this allocator api comes with two major caveats: -/// 1. All pointers that are in-use must be traced by re-interpreting them as the relavent `GcArray` -/// 2. The `Trace` implementation must support relocating pointers. -/// -/// NOTE: Item number two may be considerably more difficult. -/// For example, the 'hashbrown::raw::RawTable' api supports accessing the raw pointers, -/// but doesn't support changing or reloacting it.....x -pub struct GcAllocWrapper<'gc, C: GcSimpleAlloc>(&'gc C); - -unsafe impl<'gc, C: GcSimpleAlloc> Allocator for GcAllocWrapper<'gc, C> { - fn allocate(&self, layout: Layout) -> Result, AllocError> { - unsafe { - let ptr: *mut u8 = match layout.align() { - 1 => self.0.alloc_uninit_slice::(layout.size()), - 2 => self - .0 - .alloc_uninit_slice::((layout.size() + 1) / 2) - .cast(), - 4 => self - .0 - .alloc_uninit_slice::((layout.size() + 3) / 4) - .cast(), - 8 => self - .0 - .alloc_uninit_slice::((layout.size() + 7) / 8) - .cast(), - _ => return Err(AllocError), - }; - Ok(NonNull::new_unchecked(core::ptr::slice_from_raw_parts_mut( - ptr, - layout.size(), - ))) - } - } - #[inline] - unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - /* - * with garbage collection, deallocation is a nop - * - * If we're in debug mode we will write - * 0xDEADBEAF to the memory to be extra sure. - */ - if cfg!(debug_assertions) { - const SRC: [u8; 4] = (0xDEAD_BEAFu32).to_ne_bytes(); - ptr.as_ptr() - .copy_from_nonoverlapping(SRC.as_ptr(), layout.size().min(4)); - } - } -} diff --git a/src/array.rs b/src/array.rs deleted file mode 100644 index f5f9cc0..0000000 --- a/src/array.rs +++ /dev/null @@ -1,294 +0,0 @@ -//! Defines the interface to garbage collected arrays. -use core::cmp::Ordering; -use core::fmt::{self, Debug, Display, Formatter}; -use core::hash::{Hash, Hasher}; -use core::marker::PhantomData; -use core::ops::{Deref, Index}; -use core::ptr::NonNull; -use core::slice::SliceIndex; -use core::str; - -use crate::{CollectorId, Gc, GcRebrand, GcSafe}; -use zerogc_derive::{unsafe_gc_impl, Trace}; - -use self::repr::GcArrayPtr; - -pub mod repr; - -/// A garbage collected string. -/// -/// This is a transparent wrapper around `GcArray`, -/// with the additional invariant that it's utf8 encoded. -/// -/// ## Safety -/// The bytes can be assumed to be UTF8 encoded, -/// just like with a `str`. -/// -/// Assuming the bytes are utf8 encoded, -/// this can be transmuted back and forth from `GcArray` -#[repr(transparent)] -#[derive(Trace, Eq, PartialEq, Hash, Clone, Copy)] -#[zerogc(copy, collector_ids(Id))] -pub struct GcString<'gc, Id: CollectorId> { - bytes: GcArray<'gc, u8, Id>, -} -impl<'gc, Id: CollectorId> GcString<'gc, Id> { - /// Convert an array of UTF8 bytes into a string. - /// - /// Returns an error if the bytes aren't valid UTF8, - /// just like [core::str::from_utf8]. - #[inline] - pub fn from_utf8(bytes: GcArray<'gc, u8, Id>) -> Result { - core::str::from_utf8(bytes.as_slice())?; - // SAFETY: Validated with from_utf8 call - Ok(unsafe { Self::from_utf8_unchecked(bytes) }) - } - /// Convert an array of UTF8 bytes into a string, - /// without checking for validity. - /// - /// ## Safety - /// Undefined behavior if the bytes aren't valid - /// UTF8, just like with [core::str::from_utf8_unchecked] - #[inline] - pub const unsafe fn from_utf8_unchecked(bytes: GcArray<'gc, u8, Id>) -> Self { - GcString { bytes } - } - /// Retrieve this string as a raw array of bytes - #[inline] - pub const fn as_bytes(&self) -> GcArray<'gc, u8, Id> { - self.bytes - } - /// Convert this string into a slice of bytes - #[inline] - pub fn as_str(&self) -> &'gc str { - unsafe { str::from_utf8_unchecked(self.as_bytes().as_slice()) } - } -} -impl<'gc, Id: CollectorId> Deref for GcString<'gc, Id> { - type Target = str; - #[inline] - fn deref(&self) -> &'_ str { - self.as_str() - } -} -impl<'gc, Id: CollectorId> Debug for GcString<'gc, Id> { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - Debug::fmt(self.as_str(), f) - } -} -impl<'gc, Id: CollectorId> Display for GcString<'gc, Id> { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - Display::fmt(self.as_str(), f) - } -} - -/// A garbage collected array. -/// -/// The length is immutable and cannot change -/// once it has been allocated. -/// -/// ## Safety -/// This is a 'repr(transparent)' wrapper arround -/// [GcArrayRepr]. -#[repr(transparent)] -pub struct GcArray<'gc, T, Id: CollectorId> { - ptr: Id::ArrayPtr, - marker: PhantomData>, -} -impl<'gc, T, Id: CollectorId> GcArray<'gc, T, Id> { - /// Convert this array into a slice - #[inline] - pub fn as_slice(&self) -> &'gc [T] { - unsafe { core::slice::from_raw_parts(self.as_raw_ptr(), self.len()) } - } - /// Load a raw pointer to the array's value - #[inline] - pub fn as_raw_ptr(&self) -> *mut T { - self.ptr.as_raw_ptr() as *mut T - } - /// Get the underlying 'Id::ArrayPtr' for this array - /// - /// ## Safety - /// Must not interpret the underlying pointer as the - /// incorrect type. - #[inline] - pub const unsafe fn as_internal_ptr_repr(&self) -> &'_ Id::ArrayPtr { - &self.ptr - } - /// Load the length of the array - #[inline] - pub fn len(&self) -> usize { - match self.ptr.len() { - Some(len) => len, - None => Id::resolve_array_len(self), - } - } - /// Check if the array is empty - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - /// Resolve the [CollectorId] - #[inline] - pub fn collector_id(&self) -> &'_ Id { - Id::resolve_array_id(self) - } - /// Create an array from the specified raw pointer and length - /// - /// ## Safety - /// Pointer and length must be valid, and point to a garbage collected - /// value allocated from the corresponding [CollectorId] - #[inline] - pub unsafe fn from_raw_ptr(ptr: NonNull, len: usize) -> Self { - GcArray { - ptr: Id::ArrayPtr::from_raw_parts(ptr, len), - marker: PhantomData, - } - } -} -/// If the underlying type is `Sync`, it's safe -/// to share garbage collected references between threads. -/// -/// The safety of the collector itself depends on whether [CollectorId] is Sync. -/// If it is, the whole garbage collection implementation should be as well. -unsafe impl<'gc, T, Id> Sync for GcArray<'gc, T, Id> -where - T: Sync, - Id: CollectorId + Sync, -{ -} -unsafe impl<'gc, T, Id> Send for GcArray<'gc, T, Id> -where - T: Sync, - Id: CollectorId + Sync, -{ -} -impl<'gc, T, I, Id: CollectorId> Index for GcArray<'gc, T, Id> -where - I: SliceIndex<[T]>, -{ - type Output = I::Output; - #[inline] - fn index(&self, idx: I) -> &I::Output { - &self.as_slice()[idx] - } -} -impl<'gc, T, Id: CollectorId> Deref for GcArray<'gc, T, Id> { - type Target = [T]; - - #[inline] - fn deref(&self) -> &Self::Target { - self.as_slice() - } -} -impl<'gc, T, Id: CollectorId> Copy for GcArray<'gc, T, Id> {} -impl<'gc, T, Id: CollectorId> Clone for GcArray<'gc, T, Id> { - #[inline] - fn clone(&self) -> Self { - *self - } -} -impl<'gc, T: Debug, Id: CollectorId> Debug for GcArray<'gc, T, Id> { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - f.debug_list().entries(self.iter()).finish() - } -} -impl<'gc, T: PartialEq, Id: CollectorId> PartialEq for GcArray<'gc, T, Id> { - #[inline] - fn eq(&self, other: &Self) -> bool { - self.as_slice() == other.as_slice() - } -} -impl<'gc, T: PartialEq, Id: CollectorId> PartialEq<[T]> for GcArray<'gc, T, Id> { - #[inline] - fn eq(&self, other: &[T]) -> bool { - self.as_slice() == other - } -} -impl<'gc, T: PartialOrd, Id: CollectorId> PartialOrd for GcArray<'gc, T, Id> { - #[inline] - fn partial_cmp(&self, other: &Self) -> Option { - self.as_slice().partial_cmp(other.as_slice()) - } -} -impl<'gc, T: PartialOrd, Id: CollectorId> PartialOrd<[T]> for GcArray<'gc, T, Id> { - #[inline] - fn partial_cmp(&self, other: &[T]) -> Option { - self.as_slice().partial_cmp(other) - } -} -impl<'gc, T: Ord, Id: CollectorId> Ord for GcArray<'gc, T, Id> { - #[inline] - fn cmp(&self, other: &Self) -> Ordering { - self.as_slice().cmp(other) - } -} -impl<'gc, T: Eq, Id: CollectorId> Eq for GcArray<'gc, T, Id> {} -impl<'gc, T: Hash, Id: CollectorId> Hash for GcArray<'gc, T, Id> { - #[inline] - fn hash(&self, hasher: &mut H) { - T::hash_slice(self.as_slice(), hasher) - } -} -impl<'gc, T, Id: CollectorId> IntoIterator for GcArray<'gc, T, Id> -where - T: 'gc, -{ - type Item = &'gc T; - - type IntoIter = core::slice::Iter<'gc, T>; - - #[inline] - fn into_iter(self) -> Self::IntoIter { - self.as_slice().iter() - } -} -impl<'array, 'gc, T, Id: CollectorId> IntoIterator for &'array GcArray<'gc, T, Id> -where - T: 'array, -{ - type Item = &'array T; - - type IntoIter = core::slice::Iter<'array, T>; - - #[inline] - fn into_iter(self) -> Self::IntoIter { - self.as_slice().iter() - } -} -// Need to implement by hand, because [T] is not GcRebrand -unsafe_gc_impl!( - target => GcArray<'gc, T, Id>, - params => ['gc, T: GcSafe<'gc, Id>, Id: CollectorId], - bounds => { - TraceImmutable => never, - GcRebrand => { where T: GcRebrand<'new_gc, Id>, >::Branded: Sized + GcSafe<'new_gc, Id> }, - }, - null_trace => never, - branded_type => GcArray<'new_gc, >::Branded, Id>, - NEEDS_TRACE => true, - NEEDS_DROP => false, - trace_mut => |self, visitor| { - unsafe { visitor.trace_array(self) } - }, - collector_id => Id, - visit_inside_gc => |gc, visitor| { - visitor.trace_gc(gc) - } -); - -#[cfg(test)] -mod test { - use crate::epsilon::{self}; - use crate::{CollectorId, GcArray}; - #[test] - fn test_covariance<'a>() { - fn covariant<'a, T, Id: CollectorId>(s: GcArray<'static, T, Id>) -> GcArray<'a, T, Id> { - s as _ - } - const SRC: &[u32] = &[1, 2, 5]; - let s: epsilon::GcArray<'static, u32> = epsilon::gc_array(SRC); - let k: epsilon::GcArray<'a, u32> = covariant(s); - assert_eq!(k.as_slice(), SRC); - } -} diff --git a/src/array/repr.rs b/src/array/repr.rs deleted file mode 100644 index b1495a1..0000000 --- a/src/array/repr.rs +++ /dev/null @@ -1,187 +0,0 @@ -//! Defines the underlying representation of a [GcArray](`crate::array::GcArray`) pointer. -//! -//! -//! Two possible implementations are also available: -//! 1. FatArrayPtr - Represents arrays as a fat pointer -//! 2. ThinArraPtr - Represents arrays as a thin pointer, -//! with the length stored indirectly in the object header. -#![allow( - clippy::len_without_is_empty, // This is really an internal interface... -)] -use core::ffi::c_void; -use core::marker::PhantomData; -use core::ptr::NonNull; - -use crate::CollectorId; - -/// The type of [GcArrayPtr] impl -#[derive(Copy, Clone, Debug)] -pub enum ArrayPtrKind { - /// A `FatArrayRepr`, which can be transmuted <-> to `&[T]` - Fat, - /// A `ThinArrayRepr`, which can be transmuted <-> to `NonNull` - Thin, -} - -/// The raw, untyped representation of a GcArray pointer. -/// -/// NOTE: This is only for customizing the *pointer* -/// representation. The in-memory layout of the array and its -/// header can be controlled separately from the pointer. -/// -/// This trait is sealed, and there are only two possible -/// implementations: -/// 1. fat pointers -/// 2. thin pointers -/// -/// This needs to be untyped, -/// because we expect the type to be [covariant](https://doc.rust-lang.org/nomicon/subtyping.html#variance). -/// If we were to use `Id::ArrayPtr` the borrow checker would infer the type -/// `T` to be invariant. Instead, we just treat it as a `NonNull`, -/// and add an extra `PhantomData`. Variance problem solved :p -/// -/// ## Safety -/// If the length is stored inline in the array (like a fat pointer), -/// then the length and never change. -/// -/// If the length is *not* stored inline, then it must -/// be retrieved from the corresponding [CollectorId]. -/// -/// The underlying 'repr' is responsible -/// for dropping memory as appropriate. -pub unsafe trait GcArrayPtr: Copy + sealed::Sealed { - /// The repr's collector - type Id: CollectorId; - /// The "kind" of the array pointer (whether fat or thin) - /// - /// This is necessary to correctly - /// transmute in a const-fn context - const UNCHECKED_KIND: ArrayPtrKind; - /// Construct an array representation from a combination - /// of a pointer and length. - /// - /// This is the garbage collected equivalent of [std::slice::from_raw_parts] - /// - /// ## Safety - /// The combination of pointer + length must be valid. - /// - /// The pointer must be the correct type (the details are erased at runtime). - unsafe fn from_raw_parts(ptr: NonNull, len: usize) -> Self; - /// Get a raw pointer to this array's elements. - /// - /// The pointer is untyped. - fn as_raw_ptr(&self) -> *mut c_void; - /// Get the length of this array, - /// or `None` if it's not stored in the pointer (it's a thin pointer). - /// - /// If this type is a fat pointer it will return - /// `Some`. - /// If this is a thin pointer, then it must return `None`. - /// - /// NOTE: Despite the fact that `as_raw_ptr` returns `c_void`, - /// the length is in terms of the (erased) runtime type `T`, - /// not in terms of bytes. - fn len(&self) -> Option; -} - -/// Represents an array as a fat pointer. -/// -/// ## Safety -/// This pointer is stored as a `NonNull<[c_void]>` -/// -/// Transmuting back and forth is safe if and only if -/// it is cast to a `T` first. -#[repr(transparent)] -pub struct FatArrayPtr { - /// NOTE: The length of this slice is in terms of `T`, - /// not in terms of bytes. - /// - /// It is (probably) an under-estimation - slice: NonNull<[c_void]>, - marker: PhantomData, -} -impl self::sealed::Sealed for FatArrayPtr {} -impl FatArrayPtr { - /// Get the length of this fat array (stored inline) - #[inline] - pub const fn len(&self) -> usize { - unsafe { (*self.slice.as_ptr()).len() } - } -} -impl Copy for FatArrayPtr {} -impl Clone for FatArrayPtr { - #[inline] - fn clone(&self) -> Self { - *self - } -} - -unsafe impl GcArrayPtr for FatArrayPtr { - type Id = Id; - const UNCHECKED_KIND: ArrayPtrKind = ArrayPtrKind::Fat; - - #[inline] - unsafe fn from_raw_parts(ptr: NonNull, len: usize) -> Self { - FatArrayPtr { - slice: NonNull::new_unchecked(core::ptr::slice_from_raw_parts( - ptr.as_ptr() as *const T, - len, - ) as *mut [T] as *mut [c_void]), - marker: PhantomData, - } - } - - #[inline] - fn as_raw_ptr(&self) -> *mut c_void { - self.slice.as_ptr() as *mut c_void - } - - #[inline] - fn len(&self) -> Option { - Some(self.len()) // delegates to inherent impl - } -} - -/// Represents an array as a thin pointer, -/// storing the length indirectly in the object's header. -/// -/// ## Safety -/// This type has the same layout as `NonNull`. -/// This representation can be relied upon if and only -/// if is cast to `NonNull` first. -#[repr(transparent)] -pub struct ThinArrayPtr { - elements: NonNull, - marker: PhantomData, -} -impl Copy for ThinArrayPtr {} -impl Clone for ThinArrayPtr { - #[inline] - fn clone(&self) -> Self { - *self - } -} -impl self::sealed::Sealed for ThinArrayPtr {} -unsafe impl GcArrayPtr for ThinArrayPtr { - type Id = Id; - const UNCHECKED_KIND: ArrayPtrKind = ArrayPtrKind::Thin; - #[inline] - unsafe fn from_raw_parts(ptr: NonNull, _len: usize) -> Self { - ThinArrayPtr { - elements: ptr.cast(), - marker: PhantomData, - } - } - #[inline] - fn as_raw_ptr(&self) -> *mut c_void { - self.elements.as_ptr() - } - #[inline] - fn len(&self) -> Option { - None - } -} - -mod sealed { - pub trait Sealed {} -} diff --git a/src/epsilon.rs b/src/epsilon.rs deleted file mode 100644 index 017ed62..0000000 --- a/src/epsilon.rs +++ /dev/null @@ -1,487 +0,0 @@ -//! An "epsilon" garbage collector, which never garbage collects or -//! frees memory until the garbage collector is dropped. -//! -//! Essentially, this is an arena allocator. -//! -//! Because it is backed by a simple arena allocator, -//! the [EpsilonSystem] is `!Sync`, and can't be used by multiple threads -//! at once (although references to it can be freely sent around once already allocated). -#![cfg(feature = "epsilon")] - -mod alloc; -mod handle; -mod layout; - -use crate::{CollectorId, GcContext, GcSafe, GcSimpleAlloc, GcSystem, Trace}; -use std::alloc::Layout; -use std::cell::{Cell, OnceCell}; -use std::ptr::NonNull; -use std::rc::Rc; - -use self::alloc::EpsilonAlloc; -use self::layout::EpsilonRawVec; - -/// The implementation of [EpsilonRawVec] -pub mod vec { - pub use super::layout::EpsilonRawVec; -} - -/// Coerce a reference into a [Gc] pointer. -/// -/// This is only supported on the epsilon collector. -/// Because the epsilon collector never allocates, -/// it doesn't need to make a distinction between `Gc` and `&T`. -/// -/// This will never actually be collected -/// and will always be valid -/// -/// TODO: Rename?? -#[inline] -pub const fn gc<'gc, T: ?Sized + GcSafe<'gc, EpsilonCollectorId> + 'gc>(ptr: &'gc T) -> Gc<'gc, T> { - /* - * SAFETY: Epsilon never collects unless explicitly added to - * the linked list of allocated objects. - * Therefore any reference can be assumed to be a Gc ptr. - */ - unsafe { std::mem::transmute::<&'gc T, crate::Gc<'gc, T, EpsilonCollectorId>>(ptr) } -} - -/// Coerce a slice into a `GcArray`. -/// -/// This is only supported on the epsilon collector. -/// Because the epsilon collector never collects, -/// it doesn't need to make a distinction between `GcArray` and `&[T]`. -/// -/// See also: [gc] for converting `&T` -> `Gc` -#[inline] -pub const fn gc_array<'gc, T: GcSafe<'gc, EpsilonCollectorId> + 'gc>( - slice: &'gc [T], -) -> GcArray<'gc, T> { - /* - * SAFETY: Epsilon uses the 'fat' representation for GcArrays. - * That means that repr(GcArray) == repr(&[T]). - * - * Since we never collect, we are free to transmute - * back and forth between them - */ - unsafe { std::mem::transmute::<&'gc [T], crate::GcArray<'gc, T, EpsilonCollectorId>>(slice) } -} - -/// Coerce a `&str` into a `GcString` -/// -/// This is only supported on the epsilon collector, -/// because the epsilon collector never collects. -/// -/// See also [gc_array] for converting `&[T]` -> `GcArray` -#[inline] -pub const fn gc_str<'gc>(s: &'gc str) -> GcString<'gc> { - /* - * SAFETY: Epsilon uses the 'fat' representation for GcArrays. - * This means that repr(GcArray) == repr(&[T]) - * - * Because we already know the string is UTF8 encoded, - * we can take advantage of the fact that repr(str) == repr(&[u8]) - * and repr(GcArray) == repr(GcString). - * Instead of going `str -> &[T] -> GcArray -> GcString` - * we can just go directly from `str -> GcString` - */ - unsafe { std::mem::transmute::<&'gc str, crate::array::GcString<'gc, EpsilonCollectorId>>(s) } -} - -/// Allocate a [(fake) Gc](Gc) that points to the specified -/// value and leak it. -/// -/// Since collection is unimplemented, -/// this intentionally leaks memory. -pub fn leaked<'gc, T: GcSafe<'gc, EpsilonCollectorId> + 'static>(value: T) -> Gc<'gc, T> { - gc(Box::leak(Box::new(value))) -} - -/// A [garbage collected pointer](`crate::Gc`) -/// that uses the [episolon collector](EpsilonSystem) -/// -/// **WARNING**: This never actually collects any garbage -pub type Gc<'gc, T> = crate::Gc<'gc, T, EpsilonCollectorId>; -/// A [garbage collected array](`crate::array::GcArray`) -/// that uses the [epsilon collector](EpsilonSystem) -/// -/// **WARNING**: This never actually collects any garbage. -pub type GcArray<'gc, T> = crate::array::GcArray<'gc, T, EpsilonCollectorId>; -/// A [garbage collected array](`crate::vec::GcVec`) -/// that uses the [epsilon collector](EpsilonSystem) -/// -/// **WARNING**: This never actually collects any garbage. -pub type GcVec<'gc, T> = crate::vec::GcVec<'gc, T, EpsilonContext>; -/// A [garbage collected string](`crate::array::GcString`) -/// that uses the epsilon collector. -/// -/// **WARNING**: This never actually collects any garbage -pub type GcString<'gc> = crate::array::GcString<'gc, EpsilonCollectorId>; - -/// A never-collecting garbage collector context. -/// -/// **WARNING**: This never actually collects any garbage. -pub struct EpsilonContext { - state: NonNull, - root: bool, -} -unsafe impl GcContext for EpsilonContext { - type System = EpsilonSystem; - type Id = EpsilonCollectorId; - - #[inline] - unsafe fn unchecked_safepoint(&self, _value: &mut &mut T) { - // safepoints are a nop in our system - } - - unsafe fn freeze(&mut self) { - unimplemented!() - } - - unsafe fn unfreeze(&mut self) { - unimplemented!() - } - - #[inline] - unsafe fn recurse_context(&self, value: &mut &mut T, func: F) -> R - where - T: Trace, - F: for<'gc> FnOnce(&'gc mut Self, &'gc mut T) -> R, - { - // safepoints are a nop since there is nothing to track - let mut child = EpsilonContext { - state: self.state, - root: false, - }; - func(&mut child, *value) - } - - #[inline] - fn system(&self) -> &'_ Self::System { - // Pointer to a pointer - unsafe { - NonNull::>::from(&self.state) - .cast::() - .as_ref() - } - } - - #[inline] - fn id(&self) -> Self::Id { - EpsilonCollectorId { _priv: () } - } -} -impl Drop for EpsilonContext { - #[inline] - fn drop(&mut self) { - unsafe { - if self.root { - drop(Rc::from_raw(self.state.as_ptr())) - } - } - } -} - -struct State { - alloc: alloc::Default, - /// The head of the linked-list of allocated objects. - head: Cell>>, - empty_vec: OnceCell>, -} -impl State { - #[inline] - unsafe fn push_state(&self, mut header: NonNull) { - header.as_mut().next = self.head.get(); - self.head.set(Some(header)); - } -} -impl Drop for State { - fn drop(&mut self) { - let mut ptr = self.head.get(); - unsafe { - while let Some(header) = ptr { - let header_layout = layout::EpsilonHeader::LAYOUT; - let desired_align = header.as_ref().type_info.layout.align(); - let padding = header_layout.padding_needed_for(desired_align); - let value_ptr = (header.as_ptr() as *const u8) - .add(header_layout.size()) - .add(padding); - if let Some(drop_func) = header.as_ref().type_info.drop_func { - (drop_func)(value_ptr as *const _ as *mut _); - } - let next = header.as_ref().next; - if self::alloc::Default::NEEDS_EXPLICIT_FREE { - let value_layout = header.as_ref().determine_layout(); - let original_header = NonNull::new_unchecked( - header - .cast::() - .as_ptr() - .sub(header.as_ref().type_info.layout.common_header_offset()), - ); - let header_size = - value_ptr.cast::().offset_from(original_header.as_ptr()) as usize; - let combined_layout = Layout::from_size_align_unchecked( - value_layout.size() + header_size, - value_layout - .align() - .max(layout::EpsilonHeader::LAYOUT.align()), - ); - self.alloc.free_alloc(original_header, combined_layout); - } - ptr = next; - } - } - } -} - -/// A dummy implementation of [GcSystem] -/// which is useful for testing -/// -/// **WARNING**: This never actually collects any memory. -pub struct EpsilonSystem { - /// The raw state of the system - state: NonNull, -} -impl EpsilonSystem { - #[inline] - fn from_state(state: Rc) -> EpsilonSystem { - EpsilonSystem { - state: unsafe { NonNull::new_unchecked(Rc::into_raw(state) as *mut _) }, - } - } - - #[inline] - fn clone_rc(&self) -> Rc { - unsafe { - Rc::increment_strong_count(self.state.as_ptr()); - Rc::from_raw(self.state.as_ptr()) - } - } - /// Create a new epsilon collector, which intentionally leaks memory - #[inline] - pub fn leak() -> Self { - EpsilonSystem::from_state(Rc::new(State { - alloc: self::alloc::Default::new(), - head: Cell::new(None), - empty_vec: OnceCell::new(), - })) - } - - #[inline] - fn state(&self) -> &'_ State { - unsafe { self.state.as_ref() } - } - - /// Create a new [EpsilonContext] - /// - /// There are few restrictions on this - /// because it doesn't actually do anything - #[inline] - pub fn new_context(&self) -> EpsilonContext { - EpsilonContext { - state: unsafe { NonNull::new_unchecked(Rc::into_raw(self.clone_rc()) as *mut _) }, - root: true, - } - } -} -impl Drop for EpsilonSystem { - #[inline] - fn drop(&mut self) { - unsafe { Rc::decrement_strong_count(self.state.as_ptr()) } - } -} -unsafe impl GcSystem for EpsilonSystem { - type Id = EpsilonCollectorId; - type Context = EpsilonContext; -} -unsafe impl GcSimpleAlloc for EpsilonContext { - #[inline] - unsafe fn alloc_uninit<'gc, T>(&'gc self) -> *mut T - where - T: GcSafe<'gc, EpsilonCollectorId>, - { - let tp = self::layout::TypeInfo::of::(); - let needs_header = self::alloc::Default::NEEDS_EXPLICIT_FREE || !tp.may_ignore(); - let ptr = if needs_header { - let (overall_layout, offset) = self::layout::EpsilonHeader::LAYOUT - .extend(Layout::new::()) - .unwrap(); - let mem = self.system().state().alloc.alloc_layout(overall_layout); - let header = mem.cast::(); - header.as_ptr().write(self::layout::EpsilonHeader { - type_info: tp, - next: None, - }); - self.system().state().push_state(header); - mem.as_ptr().add(offset) - } else { - self.system() - .state() - .alloc - .alloc_layout(Layout::new::()) - .as_ptr() - }; - ptr.cast() - } - - #[inline] - fn alloc<'gc, T>(&'gc self, value: T) -> crate::Gc<'gc, T, Self::Id> - where - T: GcSafe<'gc, Self::Id>, - { - unsafe { - let ptr = self.alloc_uninit::(); - ptr.write(value); - Gc::from_raw(NonNull::new_unchecked(ptr)) - } - } - - #[inline] - unsafe fn alloc_uninit_slice<'gc, T>(&'gc self, len: usize) -> *mut T - where - T: GcSafe<'gc, Self::Id>, - { - let type_info = self::layout::TypeInfo::of_array::(); - let (overall_layout, offset) = Layout::new::() - .extend(Layout::array::(len).unwrap()) - .unwrap(); - let mem = self.system().state().alloc.alloc_layout(overall_layout); - let header = mem.cast::(); - header.as_ptr().write(self::layout::EpsilonArrayHeader { - common_header: self::layout::EpsilonHeader { - type_info, - next: None, - }, - len, - }); - self.system() - .state() - .push_state(NonNull::from(&header.as_ref().common_header)); - mem.as_ptr().add(offset).cast() - } - - #[inline] - fn alloc_raw_vec_with_capacity<'gc, T>(&'gc self, capacity: usize) -> EpsilonRawVec<'gc, T> - where - T: GcSafe<'gc, Self::Id>, - { - if capacity == 0 { - if let Some(&empty_ptr) = self.system().state().empty_vec.get() { - return unsafe { self::layout::EpsilonRawVec::from_raw_parts(empty_ptr, self) }; - } - } - let type_info = layout::TypeInfo::of_vec::(); - let (overall_layout, offset) = Layout::new::() - .extend(Layout::array::(capacity).unwrap()) - .unwrap(); - let mem = self.system().state().alloc.alloc_layout(overall_layout); - unsafe { - let header = mem.cast::(); - header.as_ptr().write(self::layout::EpsilonVecHeader { - common_header: self::layout::EpsilonHeader { - type_info, - next: None, - }, - len: Cell::new(0), - capacity, - }); - self.system() - .state() - .push_state(NonNull::from(&header.as_ref().common_header)); - let value_ptr = mem.as_ptr().add(offset).cast::(); - let raw = self::layout::EpsilonRawVec::from_raw_parts(header, self); - debug_assert_eq!(raw.as_ptr(), value_ptr); - raw - } - } -} - -/// The id for an [EpsilonSystem] -/// -/// All epsilon collectors have the same id, -/// regardless of the system they were originally allocated from. -/// It is equivalent to [ -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] -pub struct EpsilonCollectorId { - _priv: (), -} -crate::impl_nulltrace_for_static!(EpsilonCollectorId); -unsafe impl CollectorId for EpsilonCollectorId { - type System = EpsilonSystem; - type Context = EpsilonContext; - type RawVec<'gc, T: GcSafe<'gc, Self>> = self::layout::EpsilonRawVec<'gc, T>; - /// We use fat-pointers for arrays, - /// so that we can transmute from `&'static [T]` -> `GcArray` - type ArrayPtr = zerogc::array::repr::FatArrayPtr; - - #[inline] - fn from_gc_ptr<'a, 'gc, T>(_gc: &'a Gc<'gc, T>) -> &'a Self - where - T: ?Sized, - 'gc: 'a, - { - const ID: EpsilonCollectorId = EpsilonCollectorId { _priv: () }; - &ID - } - - #[inline] - fn resolve_array_id<'a, 'gc, T>(_array: &'a GcArray<'gc, T>) -> &'a Self - where - 'gc: 'a, - { - const ID: EpsilonCollectorId = EpsilonCollectorId { _priv: () }; - &ID - } - - #[inline] - fn resolve_array_len(repr: &GcArray<'_, T>) -> usize { - repr.len() - } - - #[inline] - unsafe fn gc_write_barrier<'gc, T, V>( - _owner: &Gc<'gc, T>, - _value: &Gc<'gc, V>, - _field_offset: usize, - ) where - T: GcSafe<'gc, Self> + ?Sized, - V: GcSafe<'gc, Self> + ?Sized, - { - } - - unsafe fn assume_valid_system(&self) -> &Self::System { - /* - * NOTE: Supporting this would lose our ability to go from `&'static T` -> `Gc<'gc, T, EpsilonCollectorId> - * It would also necessitate a header for `Copy` objects. - */ - unimplemented!("Unable to convert EpsilonCollectorId -> EpsilonSystem") - } -} - -#[cfg(test)] -mod test { - use super::*; - use crate::Trace; - #[test] - fn lifetime_variance<'a>() { - #[derive(Trace, Copy, Clone)] - #[zerogc(copy, collector_ids(EpsilonCollectorId))] - enum ShouldBeVariant<'gc> { - First(Gc<'gc, ShouldBeVariant<'gc>>), - Second(u32), - #[allow(unused)] - Array(GcArray<'gc, ShouldBeVariant<'gc>>), - } - const STATIC: Gc<'static, u32> = gc(&32); - const SECOND: &ShouldBeVariant<'static> = &ShouldBeVariant::Second(32); - const FIRST_VAL: &ShouldBeVariant<'static> = &ShouldBeVariant::First(gc(SECOND)); - const FIRST: Gc<'static, ShouldBeVariant<'static>> = gc(FIRST_VAL); - fn covariant<'a, T>(s: Gc<'static, T>) -> Gc<'a, T> { - s as _ - } - let s: Gc<'a, u32> = covariant(STATIC); - assert_eq!(s.value(), &32); - let k: Gc<'a, ShouldBeVariant<'a>> = covariant::<'a, ShouldBeVariant<'static>>(FIRST) as _; - assert!(matches!(k.value(), ShouldBeVariant::First(_))); - } -} diff --git a/src/epsilon/alloc.rs b/src/epsilon/alloc.rs deleted file mode 100644 index a2eedf8..0000000 --- a/src/epsilon/alloc.rs +++ /dev/null @@ -1,46 +0,0 @@ -use std::alloc::Layout; -use std::ptr::NonNull; - -#[cfg(feature = "epsilon-arena-alloc")] -mod arena; - -pub trait EpsilonAlloc { - fn new() -> Self; - fn alloc_layout(&self, layout: Layout) -> NonNull; - unsafe fn free_alloc(&self, target: NonNull, layout: Layout); - const NEEDS_EXPLICIT_FREE: bool; -} - -#[cfg(feature = "epsilon-arena-alloc")] -pub type Default = arena::BumpEpsilonAlloc; -#[cfg(not(feature = "epsilon-arena-alloc"))] -pub type Default = StdEpsilonAlloc; - -pub struct StdEpsilonAlloc; -impl EpsilonAlloc for StdEpsilonAlloc { - #[inline] - fn new() -> Self { - StdEpsilonAlloc - } - - #[inline] - fn alloc_layout(&self, layout: Layout) -> NonNull { - const EMPTY: &[u8] = b""; - if layout.size() == 0 { - return NonNull::from(EMPTY).cast(); - } - // SAFETY: We checked for layout.size() == 0 - NonNull::new(unsafe { std::alloc::alloc(layout) }) - .unwrap_or_else(|| std::alloc::handle_alloc_error(layout)) - } - - #[inline] - unsafe fn free_alloc(&self, target: NonNull, layout: Layout) { - if layout.size() == 0 { - return; // We returned our dummy empty alloc - } - std::alloc::dealloc(target.as_ptr(), layout) - } - - const NEEDS_EXPLICIT_FREE: bool = true; -} diff --git a/src/epsilon/alloc/arena.rs b/src/epsilon/alloc/arena.rs deleted file mode 100644 index e673a1a..0000000 --- a/src/epsilon/alloc/arena.rs +++ /dev/null @@ -1,21 +0,0 @@ -use std::alloc::Layout; -use std::ptr::NonNull; - -use bumpalo::Bump; - -use super::EpsilonAlloc; - -pub struct BumpEpsilonAlloc(Bump); -impl EpsilonAlloc for BumpEpsilonAlloc { - #[inline] - fn new() -> Self { - BumpEpsilonAlloc(Bump::new()) - } - #[inline] - fn alloc_layout(&self, layout: Layout) -> NonNull { - self.0.alloc_layout(layout) - } - #[inline] - unsafe fn free_alloc(&self, _target: NonNull, _layout: Layout) {} - const NEEDS_EXPLICIT_FREE: bool = false; -} diff --git a/src/epsilon/handle.rs b/src/epsilon/handle.rs deleted file mode 100644 index bf89d8e..0000000 --- a/src/epsilon/handle.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::ptr::NonNull; -use std::rc::Rc; - -use crate::prelude::*; - -use super::{EpsilonCollectorId, EpsilonContext, EpsilonSystem, State}; - -pub struct GcHandle> { - /// The reference to the state, - /// which keeps our data alive - state: Rc, - ptr: *const T, -} -impl> Clone for GcHandle { - fn clone(&self) -> Self { - GcHandle { - state: Rc::clone(&self.state), - ptr: self.ptr, - } - } -} -unsafe impl> zerogc::GcHandle for GcHandle { - type System = EpsilonSystem; - type Id = EpsilonCollectorId; - - #[inline] - fn use_critical(&self, func: impl FnOnce(&T) -> R) -> R { - func(unsafe { &*self.ptr }) - } - - fn bind_to<'new_gc>( - &self, - context: &'new_gc EpsilonContext, - ) -> Gc<'new_gc, T::Branded, Self::Id> - where - T: GcRebrand<'new_gc, Self::Id>, - { - // TODO: Does the simple collector assert the ids are equal? - assert_eq!( - context.state.as_ptr() as *const State, - &*self.state as *const State - ); - unsafe { - Gc::from_raw(NonNull::new_unchecked(std::mem::transmute_copy::< - *const T, - *const T::Branded, - >(&self.ptr) - as *mut T::Branded)) - } - } -} -zerogc_derive::unsafe_gc_impl!( - target => GcHandle, - params => [T: GcSafe<'static, EpsilonCollectorId>], - bounds => { - Trace => { where T: ?Sized }, - TraceImmutable => { where T: ?Sized }, - TrustedDrop => { where T: ?Sized }, - GcSafe => { where T: ?Sized }, - }, - null_trace => { where T: ?Sized }, - NEEDS_DROP => true, - NEEDS_TRACE => false, - branded_type => Self, - trace_template => |self, visitor| { Ok(()) } -); - -unsafe impl HandleCollectorId for EpsilonCollectorId { - type Handle = GcHandle where T: GcSafe<'static, Self> + ?Sized; - - fn create_handle<'gc, T>(_gc: Gc<'gc, T, Self>) -> Self::Handle - where - T: GcSafe<'gc, Self> + GcRebrand<'static, Self> + ?Sized, - { - unimplemented!("epsilon collector can't convert Gc -> GcContext") - } -} diff --git a/src/epsilon/layout.rs b/src/epsilon/layout.rs deleted file mode 100644 index fc12ad4..0000000 --- a/src/epsilon/layout.rs +++ /dev/null @@ -1,325 +0,0 @@ -use std::alloc::Layout; -use std::cell::Cell; -use std::ffi::c_void; -use std::marker::PhantomData; -use std::ptr::NonNull; - -use crate::vec::raw::{GcRawVec, IGcVec}; -use crate::{GcArray, GcRebrand, GcSafe, GcSimpleAlloc}; - -use super::{EpsilonCollectorId, EpsilonContext}; - -/// The header of an object in the epsilon collector. -/// -/// Not all objects need headers. -/// If they are `Copy` and statically sized they can be elided. -/// They are also unnecessary for statically allocated objects. -pub struct EpsilonHeader { - /// This object's `TypeInfo`, or `None` if it doesn't need any. - pub type_info: &'static TypeInfo, - /// The next allocated object, or `None` if this is the final object. - pub next: Option>, -} -/* - * We are Send + Sync because once we are allocated - * `next` and `type_info` cannot change - */ -unsafe impl Send for EpsilonHeader {} -unsafe impl Sync for EpsilonHeader {} -impl EpsilonHeader { - pub const LAYOUT: Layout = Layout::new::(); - /// Assume the specified object has a header, - /// and retrieve it if so. - /// - /// ## Safety - /// Undefined behavior if the object doesn't have a header. - /// Undefined behavior if the object isn't allocated in the epsilon collector. - #[inline] - pub unsafe fn assume_header(header: *const T) -> *const EpsilonHeader { - let (_, offset) = Self::LAYOUT - .extend(Layout::for_value(&*header)) - .unwrap_unchecked(); - (header as *const c_void).sub(offset).cast() - } - #[inline] - #[track_caller] - pub unsafe fn determine_layout(&self) -> Layout { - let tp = self.type_info; - match tp.layout { - LayoutInfo::Fixed(fixed) => fixed, - LayoutInfo::Array { element_layout } | LayoutInfo::Vec { element_layout } => { - let array_header = EpsilonArrayHeader::from_common_header(self); - let len = (*array_header).len; - element_layout.repeat(len).unwrap_unchecked().0 - } - } - } -} -#[repr(C)] -pub struct EpsilonArrayHeader { - pub len: usize, - pub common_header: EpsilonHeader, -} -impl EpsilonArrayHeader { - const COMMON_OFFSET: usize = std::mem::size_of::() - std::mem::size_of::(); - #[inline] - pub unsafe fn from_common_header(header: *const EpsilonHeader) -> *const Self { - (header as *const c_void).sub(Self::COMMON_OFFSET).cast() - } -} -#[repr(C)] -pub struct EpsilonVecHeader { - pub capacity: usize, - // NOTE: Suffix must be transmutable to `EpsilonArrayHeader` - pub len: Cell, - pub common_header: EpsilonHeader, -} -impl EpsilonVecHeader { - const COMMON_OFFSET: usize = std::mem::size_of::() - std::mem::size_of::(); -} -pub enum LayoutInfo { - Fixed(Layout), - /// A variable sized array - Array { - element_layout: Layout, - }, - /// A variable sized vector - Vec { - element_layout: Layout, - }, -} -impl LayoutInfo { - #[inline] - pub const fn align(&self) -> usize { - match *self { - LayoutInfo::Fixed(layout) - | LayoutInfo::Array { - element_layout: layout, - } - | LayoutInfo::Vec { - element_layout: layout, - } => layout.align(), - } - } - #[inline] - pub fn common_header_offset(&self) -> usize { - match *self { - LayoutInfo::Fixed(_) => 0, - LayoutInfo::Array { .. } => EpsilonArrayHeader::COMMON_OFFSET, - LayoutInfo::Vec { .. } => EpsilonVecHeader::COMMON_OFFSET, - } - } -} -pub struct TypeInfo { - /// The function to drop this object, or `None` if the object doesn't need to be dropped - pub drop_func: Option, - pub layout: LayoutInfo, -} -impl TypeInfo { - #[inline] - pub const fn may_ignore(&self) -> bool { - // NOTE: We don't care about `size` - self.drop_func.is_none() && self.layout.align() <= std::mem::align_of::() - } - #[inline] - pub const fn of() -> &'static TypeInfo { - ::TYPE_INFO - } - #[inline] - pub const fn of_array() -> &'static TypeInfo { - <[T] as StaticTypeInfo>::TYPE_INFO - } - #[inline] - pub const fn of_vec() -> &'static TypeInfo { - // For now, vectors and arrays share type info - ::VEC_INFO.as_ref().unwrap() - } -} -trait StaticTypeInfo { - const TYPE_INFO: &'static TypeInfo; - const VEC_INFO: &'static Option; -} -impl StaticTypeInfo for T { - const TYPE_INFO: &'static TypeInfo = &TypeInfo { - drop_func: if std::mem::needs_drop::() { - Some(unsafe { - std::mem::transmute::( - std::ptr::drop_in_place::, - ) - }) - } else { - None - }, - layout: LayoutInfo::Fixed(Layout::new::()), - }; - const VEC_INFO: &'static Option = &Some(TypeInfo { - drop_func: if std::mem::needs_drop::() { - Some(drop_array::) - } else { - None - }, - layout: LayoutInfo::Vec { - element_layout: Layout::new::(), - }, - }); -} -impl StaticTypeInfo for [T] { - const TYPE_INFO: &'static TypeInfo = &TypeInfo { - drop_func: if std::mem::needs_drop::() { - Some(drop_array::) - } else { - None - }, - layout: LayoutInfo::Array { - element_layout: Layout::new::(), - }, - }; - const VEC_INFO: &'static Option = &None; -} -/// Drop an array or vector of the specified type -unsafe fn drop_array(ptr: *mut c_void) { - let header = EpsilonArrayHeader::from_common_header(EpsilonHeader::assume_header( - ptr as *const _ as *const T, - )); - let len = (*header).len; - std::ptr::drop_in_place(std::ptr::slice_from_raw_parts_mut(ptr as *mut T, len)); -} - -/// The raw representation of a vector in the "epsilon" collector -/* - * Implementation note: Length and capacity are stored implicitly in the [`EpsilonVecHeader`] - */ -pub struct EpsilonRawVec<'gc, T> { - header: NonNull, - context: &'gc EpsilonContext, - marker: PhantomData>, -} -impl<'gc, T> Copy for EpsilonRawVec<'gc, T> {} -impl<'gc, T> Clone for EpsilonRawVec<'gc, T> { - #[inline] - fn clone(&self) -> Self { - *self - } -} -impl<'gc, T> EpsilonRawVec<'gc, T> { - #[inline] - pub(super) unsafe fn from_raw_parts( - header: NonNull, - context: &'gc EpsilonContext, - ) -> Self { - EpsilonRawVec { - header, - context, - marker: PhantomData, - } - } - #[inline] - fn header(&self) -> *const EpsilonVecHeader { - self.header.as_ptr() as *const EpsilonVecHeader - } -} -zerogc_derive::unsafe_gc_impl!( - target => EpsilonRawVec<'gc, T>, - params => ['gc, T: GcSafe<'gc, EpsilonCollectorId>], - bounds => { - TraceImmutable => never, - GcRebrand => { where T: GcRebrand<'new_gc, EpsilonCollectorId>, T::Branded: Sized } - }, - branded_type => EpsilonRawVec<'new_gc, T::Branded>, - collector_id => EpsilonCollectorId, - NEEDS_TRACE => true, // meh - NEEDS_DROP => T::NEEDS_DROP, - null_trace => never, - trace_mut => |self, visitor| { - unsafe { visitor.trace_vec(self) } - }, -); -#[inherent::inherent] -unsafe impl<'gc, T: GcSafe<'gc, EpsilonCollectorId>> GcRawVec<'gc, T> for EpsilonRawVec<'gc, T> { - #[inline] - #[allow(dead_code)] - unsafe fn steal_as_array_unchecked(mut self) -> GcArray<'gc, T, EpsilonCollectorId> { - // Set capacity to zero, so no one else gets any funny ideas! - self.header.as_mut().capacity = 0; - GcArray::from_raw_ptr(NonNull::new_unchecked(self.as_mut_ptr()), self.len()) - } - pub fn iter(&self) -> zerogc::vec::raw::RawVecIter<'gc, T, Self> - where - T: Copy; -} -#[inherent::inherent] -unsafe impl<'gc, T: GcSafe<'gc, EpsilonCollectorId>> IGcVec<'gc, T> for EpsilonRawVec<'gc, T> { - type Id = EpsilonCollectorId; - - #[inline] - pub fn with_capacity_in(capacity: usize, ctx: &'gc EpsilonContext) -> Self { - ctx.alloc_raw_vec_with_capacity::(capacity) - } - - #[inline] - pub fn len(&self) -> usize { - unsafe { (*self.header()).len.get() } - } - - #[inline] - pub unsafe fn set_len(&mut self, len: usize) { - (*self.header()).len.set(len) - } - - #[inline] - pub fn capacity(&self) -> usize { - unsafe { (*self.header()).capacity } - } - - #[inline] - pub fn reserve_in_place( - &mut self, - _additional: usize, - ) -> Result<(), crate::vec::raw::ReallocFailedError> { - Err(crate::vec::raw::ReallocFailedError::Unsupported) - } - - #[inline] - pub unsafe fn as_ptr(&self) -> *const T { - const LAYOUT: Layout = Layout::new::(); - let offset = LAYOUT.size() + LAYOUT.padding_needed_for(core::mem::align_of::()); - (self.header() as *const u8).add(offset) as *const T - } - - #[inline] - pub fn context(&self) -> &'gc EpsilonContext { - self.context - } - - // Default methods: - pub unsafe fn as_mut_ptr(&mut self) -> *mut T; - pub fn replace(&mut self, index: usize, val: T) -> T; - pub fn set(&mut self, index: usize, val: T); - pub fn extend_from_slice(&mut self, src: &[T]) - where - T: Copy; - pub fn push(&mut self, val: T); - pub fn pop(&mut self) -> Option; - pub fn swap_remove(&mut self, index: usize) -> T; - pub fn reserve(&mut self, additional: usize); - pub fn is_empty(&self) -> bool; - pub fn new_in(ctx: &'gc EpsilonContext) -> Self; - pub fn copy_from_slice(src: &[T], ctx: &'gc EpsilonContext) -> Self - where - T: Copy; - pub fn from_vec(src: Vec, ctx: &'gc EpsilonContext) -> Self; - pub fn get(&mut self, index: usize) -> Option - where - T: Copy; - pub unsafe fn as_slice_unchecked(&self) -> &[T]; -} -impl<'gc, T: GcSafe<'gc, EpsilonCollectorId>> Extend for EpsilonRawVec<'gc, T> { - #[inline] - fn extend>(&mut self, iter: E) { - let iter = iter.into_iter(); - self.reserve(iter.size_hint().1.unwrap_or(0)); - for val in iter { - self.push(val); - } - } -} diff --git a/src/errors.rs b/src/errors.rs deleted file mode 100644 index c495cd3..0000000 --- a/src/errors.rs +++ /dev/null @@ -1,126 +0,0 @@ -//! Adds a `GcError` type, -//! which implements `std::error::Error + 'static` -//! for garbage collected error types. -//! -//! This allows for easy compatibility with [`anyhow`](https://docs.rs/anyhow/1.0.43/anyhow/) -//! even if you want to use garbage collected data in your errors -//! (which would otherwise require a `'gc` lifetime). -//! -//! The implementation doesn't require any unsafe code. -//! It's simply a thin wrapper around [GcHandle] that uses -//! [GcHandle::use_critical] section to block -//! garbage collection during formatting calls... - -use std::error::Error as StdError; -use std::fmt::{self, Debug, Display, Formatter}; - -use crate::prelude::*; -use crate::DynTrace; - -/// A garbage collected [`std::error::Error`] type -/// -/// This is automatically implemented for all types that -/// 1. Implement [`std::error::Error`] -/// 2. Implement [GcSafe] -/// 3. Implement [`GcRebrand<'static, Id>`](`crate::GcRebrand`) -/// 4. It must be [Sync] -/// 5. Have no other lifetimes besides `'gc` -/// -/// The fifth point is rather subtle. -/// Another way of saying it is that `T: 'gc` and `T::Branded: 'static`. -pub trait GcErrorType<'gc, Id: CollectorId>: - StdError + Sync + GcSafe<'gc, Id> + 'gc + GcRebrand<'static, Id> + self::sealed::Sealed -where - >::Branded: 'static, -{ -} -impl<'gc, Id: CollectorId, T: StdError + 'gc + Sync + GcSafe<'gc, Id> + GcRebrand<'static, Id>> - GcErrorType<'gc, Id> for T -where - >::Branded: 'static, -{ -} -impl<'gc, Id: CollectorId, T: StdError + 'gc + Sync + GcSafe<'gc, Id> + GcRebrand<'static, Id>> - self::sealed::Sealed for T -where - >::Branded: 'static, -{ -} - -/// A super-trait of [GcErrorType] -/// that is suitable for dynamic dispatch. -/// -/// This can only actually implemented -/// if the type is a `GcErrorType` at runtime. -/// -/// This is an implementation detail -#[doc(hidden)] -pub trait DynGcErrorType<'gc, Id: CollectorId>: - Sync + StdError + DynTrace<'gc, Id> + self::sealed::Sealed -{ -} -impl<'gc, T: GcErrorType<'gc, Id>, Id: CollectorId> DynGcErrorType<'gc, Id> for T where - >::Branded: 'static -{ -} - -crate::trait_object_trace!( - impl<'gc, Id> Trace for dyn DynGcErrorType<'gc, Id> { where Id: CollectorId }; - Branded<'new_gc> => (dyn DynGcErrorType<'new_gc, Id> + 'new_gc), - collector_id => Id, - gc_lifetime => 'gc -); - -/// A wrapper around a dynamically dispatched, -/// [std::error::Error] with garbage collected data. -/// -/// The internal `'gc` lifetime has been erased, -/// by wrapping it in a [GcHandle]. -/// Because the internal lifetime has been erased, -/// the type can be safely -/// -/// This is analogous to [`anyhow::Error`](https://docs.rs/anyhow/1.0.43/anyhow/struct.Error.html) -/// but only for garbage collected . -pub struct GcError { - handle: Box>>, -} -impl GcError { - /// Allocate a new dynamically dispatched [GcError] - /// by converting from a specified `Gc` object. - /// - /// A easier, simpler and recommended alternative - /// is [GcSimpleAlloc::alloc_error]. - #[cold] - pub fn from_gc_allocated<'gc, T: GcErrorType<'gc, Id> + 'gc>(gc: Gc<'gc, T, Id>) -> Self - where - >::Branded: 'static, - { - let dynamic = gc as Gc<'gc, dyn DynGcErrorType<'gc, Id>, Id>; - GcError { - handle: Box::new(Gc::create_handle(&dynamic)), - } - } -} -impl Display for GcError { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - self.handle.use_critical(|err| Display::fmt(&err, f)) - } -} -impl Debug for GcError { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - self.handle.use_critical(|err| Debug::fmt(&err, f)) - } -} -impl StdError for GcError { - /* - * TODO: We can't give 'source' and 'backtrace' - * because they borrow for `'self` - * and it is possible a moving garbage - * collector could relocate - * internal data - */ -} - -mod sealed { - pub trait Sealed {} -} diff --git a/src/hash_map.rs b/src/hash_map.rs deleted file mode 100644 index 9be9553..0000000 --- a/src/hash_map.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! A garbage collected HashMap implementation -//! -//! Right now, the only implementation -//! is [GcIndexMap]. It is a garbage collected -//! version of [indexmap::IndexMap](https://docs.rs/indexmap/1.7.0/indexmap/map/struct.IndexMap.html). -//! -//! In the future, unordered maps may be possible -//! (although they'll likely require much more work). -pub mod indexmap; - -/// The default hasher for garbage collected maps. -pub type DefaultHasher = ahash::RandomState; - -pub use self::indexmap::GcIndexMap; diff --git a/src/hash_map/indexmap.rs b/src/hash_map/indexmap.rs deleted file mode 100644 index e061487..0000000 --- a/src/hash_map/indexmap.rs +++ /dev/null @@ -1,365 +0,0 @@ -//! Contains the implementation of [GcIndexMap] - -use core::borrow::Borrow; -use core::hash::{BuildHasher, Hash, Hasher}; -use core::mem; - -use hashbrown::raw::RawTable; - -use zerogc_derive::{unsafe_gc_impl, NullTrace, Trace}; - -use crate::prelude::*; -use crate::SimpleAllocCollectorId; - -/// A garbage collected hashmap that preserves insertion order. -/// -/// This is based off [indexmap::IndexMap](https://docs.rs/indexmap/1.7.0/indexmap/map/struct.IndexMap.html), -/// both in API design and in implementation. -/// -/// Like a [GcVec], there can only be one owner at a time, -/// simplifying mutability checking. -pub struct GcIndexMap< - 'gc, - K: GcSafe<'gc, Id>, - V: GcSafe<'gc, Id>, - Id: SimpleAllocCollectorId, - S: BuildHasher = super::DefaultHasher, -> { - /// indices mapping from the entry hash to its index - /// - /// NOTE: This uses `std::alloc` instead of the garbage collector to allocate memory. - /// This is necessary because of the possibility of relocating pointers..... - /// - /// The unfortunate downside is that allocating from `std::alloc` is slightly - /// slower than allocating from a typical gc (which often uses bump-pointer allocation). - indices: RawTable, - /// an ordered, garbage collected vector of entries, - /// in the original insertion order. - entries: GcVec<'gc, Bucket, Id>, - /// The hasher used to hash elements - hasher: S, -} -unsafe impl<'gc, K: GcSafe<'gc, Id>, V: GcSafe<'gc, Id>, Id: SimpleAllocCollectorId, S: BuildHasher> - crate::ImplicitWriteBarrier for GcIndexMap<'gc, K, V, Id, S> -{ -} -impl<'gc, K: GcSafe<'gc, Id>, V: GcSafe<'gc, Id>, Id: SimpleAllocCollectorId, S: BuildHasher> - GcIndexMap<'gc, K, V, Id, S> -{ - /// Allocate a new hashmap inside the specified collector - #[inline] - pub fn new_in(ctx: &'gc Id::Context) -> Self - where - S: Default, - { - Self::with_capacity_in(0, ctx) - } - /// Allocate a new hashmap with the specified capacity, - /// inside of the specified collector - #[inline] - pub fn with_capacity_in(capacity: usize, ctx: &'gc Id::Context) -> Self - where - S: Default, - { - Self::with_capacity_and_hasher_in(capacity, Default::default(), ctx) - } - /// Allocate a new hashmap with the specified capacity and hasher, - /// inside of the specified collector - #[inline] - pub fn with_capacity_and_hasher_in(capacity: usize, hasher: S, ctx: &'gc Id::Context) -> Self { - GcIndexMap { - indices: RawTable::with_capacity(capacity), - entries: GcVec::with_capacity_in(capacity, ctx), - hasher, - } - } - - /// Allocate a new hashmap with the specified hasher, - /// inside the specified collector - #[inline] - pub fn with_hasher_in(hasher: S, ctx: &'gc Id::Context) -> Self { - Self::with_capacity_and_hasher_in(0, hasher, ctx) - } - /// Return the number of entries in the map - #[inline] - pub fn len(&self) -> usize { - self.entries.len() - } - /// Check if the map is empty. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - /// Return a reference to the value associated with the specified key, - /// or `None` if it isn't present in the map. - pub fn get(&self, key: &Q) -> Option<&V> - where - K: Borrow, - Q: Hash + Eq, - { - self.get_index_of(key) - .map(|index| &self.entries[index].value) - } - /// Return a mutable reference to the value associated with the specified key, - /// or `None` if it isn't present in the map. - pub fn get_mut(&mut self, key: &Q) -> Option<&mut V> - where - K: Borrow, - Q: Hash + Eq, - { - self.get_index_of(key) - .map(move |index| &mut self.entries[index].value) - } - /// Remove the entry associated with 'key' and return its value. - /// - /// NOTE: This is equivalent to `swap_remove` and does *not* preserver ordering. - pub fn remove(&mut self, key: &Q) -> Option - where - K: Borrow, - Q: Hash + Eq, - { - self.swap_remove(key) - } - - /// Remove the value associated with the specified key. - /// - /// This does **not** preserve ordering. - /// It is similar to [Vec::swap_remove], - /// or more specifically [IndexMap::swap_remove](https://docs.rs/indexmap/1.7.0/indexmap/map/struct.IndexMap.html#method.swap_remove). - pub fn swap_remove(&mut self, key: &Q) -> Option - where - K: Borrow, - Q: Hash + Eq, - { - let hash = self.hash(key); - self.indices - .remove_entry(hash.get(), equivalent(key, &self.entries)) - .map(|index| { - let entry = self.entries.swap_remove(index); - /*hash_builder - * correct the index that points to the moved entry - * It was at 'self.len()', now it's at - */ - if let Some(entry) = self.entries.get(index) { - let last = self.entries.len(); - *self - .indices - .get_mut(entry.hash.get(), move |&i| i == last) - .expect("index not found") = index; - } - entry.value - }) - } - /// Returns - /// Insert a key value pair into the map, returning the previous value (if any( - /// - /// If the key already exists, this replaces the existing pair - /// and returns the previous value. - #[inline] - pub fn insert(&mut self, key: K, value: V) -> Option - where - K: Hash + Eq, - { - self.insert_full(key, value).1 - } - /// Insert a key value pair into the map, implicitly looking up its index. - /// - /// If the key already exists, this replaces the existing pair - /// and returns the previous value. - /// - /// If the key doesn't already exist, - /// this returns a new entry. - pub fn insert_full(&mut self, key: K, value: V) -> (usize, Option) - where - K: Hash + Eq, - { - let hash = self.hash(&key); - match self - .indices - .get(hash.get(), equivalent(&key, &*self.entries)) - { - Some(&i) => (i, Some(mem::replace(&mut self.entries[i].value, value))), - None => (self.push(hash, key, value), None), - } - } - /// Return the index of the item with the specified key. - pub fn get_index_of(&self, key: &Q) -> Option - where - Q: Hash + Eq, - K: Borrow, - { - if self.is_empty() { - None - } else { - let hash = self.hash(key); - self.indices - .get(hash.get(), equivalent(key, &*self.entries)) - .copied() - } - } - fn hash(&self, value: &Q) -> HashValue { - let mut h = self.hasher.build_hasher(); - value.hash(&mut h); - HashValue(h.finish() as usize) - } - /// Append a new key-value pair, *without* checking whether it already exists. - /// - /// Return the pair's new index - fn push(&mut self, hash: HashValue, key: K, value: V) -> usize { - let i = self.entries.len(); - self.indices.insert(hash.get(), i, get_hash(&self.entries)); - self.entries.push(Bucket { key, value, hash }); - i - } - /// Iterate over the entries in the map (in order) - #[inline] - pub fn iter(&self) -> Iter<'_, K, V> { - Iter(self.entries.iter()) - } - /// Mutably iterate over the entries in the map (in order) - #[inline] - pub fn iter_mut(&mut self) -> IterMut<'_, K, V> { - IterMut(self.entries.iter_mut()) - } - /// Iterate over tke keys in the map (in order) - #[inline] - pub fn keys(&self) -> Keys<'_, K, V> { - Keys(self.entries.iter()) - } - /// Iterate over the values in the map (in order) - #[inline] - pub fn values(&self) -> Values<'_, K, V> { - Values(self.entries.iter()) - } - /// Mutably iterate over the values in the map (in order) - #[inline] - pub fn values_mut(&mut self) -> ValuesMut<'_, K, V> { - ValuesMut(self.entries.iter_mut()) - } - /// Return the context implicitly associated with this map - /// - /// See also: [GcVec::context] - #[inline] - pub fn context(&self) -> &'gc Id::Context { - self.entries.context() - } -} -macro_rules! define_iterator { - (struct $name:ident { - const NAME = $item_name:literal; - type Item = $item:ty; - type Wrapped = $wrapped:ident; - map => |$bucket:ident| $map:expr - }) => { - #[doc = concat!("An iterator over the ", $item_name, " of a [GcIndexMap]")] - pub struct $name<'a, K: 'a, V: 'a>(core::slice::$wrapped<'a, Bucket>); - impl<'a, K: 'a, V: 'a> Iterator for $name<'a, K, V> { - type Item = $item; - #[inline] - fn next(&mut self) -> Option { - self.0.next().map(|$bucket| $map) - } - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.0.size_hint() - } - } - impl<'a, K, V> DoubleEndedIterator for $name<'a, K, V> { - #[inline] - fn next_back(&mut self) -> Option { - self.0.next_back().map(|$bucket| $map) - } - } - impl<'a, K, V> core::iter::ExactSizeIterator for $name<'a, K, V> {} - impl<'a, K, V> core::iter::FusedIterator for $name<'a, K, V> {} - }; -} - -define_iterator!(struct Iter { - const NAME = "entries"; - type Item = (&'a K, &'a V); - type Wrapped = Iter; - map => |bucket| (&bucket.key, &bucket.value) -}); -define_iterator!(struct Keys { - const NAME = "keys"; - type Item = &'a K; - type Wrapped = Iter; - map => |bucket| &bucket.key -}); -define_iterator!(struct Values { - const NAME = "valuesj"; - type Item = &'a V; - type Wrapped = Iter; - map => |bucket| &bucket.value -}); -define_iterator!(struct ValuesMut { - const NAME = "mutable values"; - type Item = &'a mut V; - type Wrapped = IterMut; - map => |bucket| &mut bucket.value -}); -define_iterator!(struct IterMut { - const NAME = "mutable entries"; - type Item = (&'a K, &'a mut V); - type Wrapped = IterMut; - map => |bucket| (&bucket.key, &mut bucket.value) -}); - -unsafe_gc_impl!( - target => GcIndexMap<'gc, K, V, Id, S>, - params => ['gc, K: GcSafe<'gc, Id>, V: GcSafe<'gc, Id>, Id: SimpleAllocCollectorId, S: BuildHasher], - bounds => { - GcSafe => { where S: 'static }, - Trace => { where S: 'static }, - TraceImmutable => never, - TrustedDrop => { where K: TrustedDrop, V: TrustedDrop, S: 'static }, - GcRebrand => { - where K: GcRebrand<'new_gc, Id>, V: GcRebrand<'new_gc, Id>, S: 'static, K::Branded: Sized, V::Branded: Sized } - }, - branded_type => GcIndexMap<'new_gc, K::Branded, V::Branded, Id, S>, - NEEDS_TRACE => true, - NEEDS_DROP => core::mem::needs_drop::(), - null_trace => never, - trace_template => |self, visitor| { - for entry in self.entries.#iter() { - visitor.#trace_func(entry)?; - } - Ok(()) - }, - collector_id => Id -); - -#[inline] -fn equivalent<'a, K, V, Q: ?Sized>( - key: &'a Q, - entries: &'a [Bucket], -) -> impl Fn(&usize) -> bool + 'a -where - Q: Hash + Eq, - K: Borrow, -{ - move |&other_index| entries[other_index].key.borrow() == key -} - -#[inline] -fn get_hash(entries: &[Bucket]) -> impl Fn(&usize) -> u64 + '_ { - move |&i| entries[i].hash.get() -} - -#[derive(Copy, Clone, Debug, PartialEq, NullTrace)] -struct HashValue(usize); -impl HashValue { - #[inline(always)] - fn get(self) -> u64 { - self.0 as u64 - } -} - -#[derive(Copy, Clone, Debug, Trace)] -#[zerogc(unsafe_skip_drop)] -struct Bucket { - hash: HashValue, - key: K, - value: V, -} diff --git a/src/lib.rs b/src/lib.rs index 6731e39..8060c8d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,33 +56,17 @@ use core::hash::{Hash, Hasher}; use core::marker::{PhantomData, Unsize}; use core::mem::{self}; use core::ops::{CoerceUnsized, Deref, DerefMut}; -use core::ptr::{DynMetadata, NonNull, Pointee}; +use core::ptr::NonNull; -use vec::raw::GcRawVec; use zerogc_derive::unsafe_gc_impl; pub use zerogc_derive::{NullTrace, Trace}; -pub use crate::array::GcArray; -use crate::vec::GcVec; - -#[macro_use] // We have macros in here! -#[cfg(feature = "serde1")] -pub mod serde; #[macro_use] mod manually_traced; #[macro_use] mod macros; -#[cfg(feature = "allocator-api")] -pub mod allocator; -pub mod array; pub mod cell; -pub mod epsilon; -#[cfg(feature = "errors")] -pub mod errors; -#[cfg(feature = "hashmap-impl")] -pub mod hash_map; pub mod prelude; -pub mod vec; /// Invoke the closure with a temporary [GcContext], /// then perform a safepoint afterwards. @@ -456,6 +440,7 @@ pub unsafe trait GcSimpleAlloc: GcContext { unsafe fn alloc_uninit<'gc, T>(&'gc self) -> *mut T where T: GcSafe<'gc, Self::Id>; + /// Allocate the specified object in this garbage collector, /// binding it to the lifetime of this collector. /// @@ -479,150 +464,6 @@ pub unsafe trait GcSimpleAlloc: GcContext { Gc::from_raw(NonNull::new_unchecked(ptr)) } } - /// Allocate a [GcString](`crate::array::GcString`), copied from the specified source - #[inline] - fn alloc_str<'gc>(&'gc self, src: &str) -> array::GcString<'gc, Self::Id> { - let bytes = self.alloc_slice_copy(src.as_bytes()); - // SAFETY: Guaranteed by the original `src` - unsafe { array::GcString::from_utf8_unchecked(bytes) } - } - - /// Wrap the specified error in a dynamically dispatched [GcError](`self::errors::GcError`) - /// - /// Uses a [GcHandle] to ensure the result lives for `'static`, - /// and thus can implement `std::error::Error` and conversion into `anyhow::Error`. - #[cfg(feature = "errors")] - #[cold] - fn alloc_error<'gc, T>(&'gc self, src: T) -> crate::errors::GcError - where - T: crate::errors::GcErrorType<'gc, Self::Id>, - >::Branded: 'static, - Self::Id: HandleCollectorId, - { - crate::errors::GcError::from_gc_allocated(self.alloc(src)) - } - - /// Allocate a slice with the specified length, - /// whose memory is uninitialized - /// - /// ## Safety - /// The slice **MUST** be initialized by the next safepoint. - /// By the time the next collection rolls around, - /// the collector will assume its entire contents are initialized. - /// - /// In particular, the traditional way to implement a [Vec] is wrong. - /// It is unsafe to leave the range of memory `[len, capacity)` uninitialized. - unsafe fn alloc_uninit_slice<'gc, T>(&'gc self, len: usize) -> *mut T - where - T: GcSafe<'gc, Self::Id>; - /// Allocate a slice, copied from the specified input - fn alloc_slice_copy<'gc, T>(&'gc self, src: &[T]) -> GcArray<'gc, T, Self::Id> - where - T: GcSafe<'gc, Self::Id> + Copy, - { - unsafe { - let res_ptr = self.alloc_uninit_slice::(src.len()); - res_ptr.copy_from_nonoverlapping(src.as_ptr(), src.len()); - GcArray::from_raw_ptr(NonNull::new_unchecked(res_ptr), src.len()) - } - } - /// Allocate an array, taking ownership of the values in - /// the specified vec. - #[inline] - #[cfg(any(feature = "alloc", feature = "std"))] - fn alloc_array_from_vec<'gc, T>(&'gc self, mut src: Vec) -> GcArray<'gc, T, Self::Id> - where - T: GcSafe<'gc, Self::Id>, - { - unsafe { - let ptr = src.as_ptr(); - let len = src.len(); - /* - * NOTE: Don't steal ownership of the - * source Vec until *after* we allocate. - * - * It is possible allocation panics in - * which case we want to free the source elements. - */ - let dest = self.alloc_uninit_slice::(len); - /* - * NOTE: From here to the end, - * we should be infallible. - */ - src.set_len(0); - dest.copy_from_nonoverlapping(ptr, len); - let res = GcArray::from_raw_ptr(NonNull::new_unchecked(ptr as *mut _), len); - drop(src); - res - } - } - /// Allocate a slice by filling it with results from the specified closure. - /// - /// The closure receives the target index as its only argument. - /// - /// ## Safety - /// The closure must always succeed and never panic. - /// - /// Otherwise, the gc may end up tracing the values even though they are uninitialized. - #[inline] - unsafe fn alloc_slice_fill_with<'gc, T, F>( - &'gc self, - len: usize, - mut func: F, - ) -> GcArray<'gc, T, Self::Id> - where - T: GcSafe<'gc, Self::Id>, - F: FnMut(usize) -> T, - { - let res_ptr = self.alloc_uninit_slice::(len); - for i in 0..len { - res_ptr.add(i).write(func(i)); - } - GcArray::from_raw_ptr(NonNull::new_unchecked(res_ptr), len) - } - /// Allocate a slice of the specified length, - /// initializing everything to `None` - #[inline] - fn alloc_slice_none<'gc, T>(&'gc self, len: usize) -> GcArray<'gc, Option, Self::Id> - where - T: GcSafe<'gc, Self::Id>, - { - unsafe { self.alloc_slice_fill_with(len, |_idx| None) } - } - /// Allocate a slice by repeatedly copying a single value. - #[inline] - fn alloc_slice_fill_copy<'gc, T>(&'gc self, len: usize, val: T) -> GcArray<'gc, T, Self::Id> - where - T: GcSafe<'gc, Self::Id> + Copy, - { - unsafe { self.alloc_slice_fill_with(len, |_idx| val) } - } - /// Create a new [GcRawVec] with the specified capacity - /// and an implicit reference to this [GcContext]. - fn alloc_raw_vec_with_capacity<'gc, T>( - &'gc self, - capacity: usize, - ) -> ::RawVec<'gc, T> - where - T: GcSafe<'gc, Self::Id>; - /// Create a new [GcVec] with zero initial length, - /// with an implicit reference to this [GcContext]. - #[inline] - fn alloc_vec<'gc, T>(&'gc self) -> GcVec<'gc, T, Self::Id> - where - T: GcSafe<'gc, Self::Id>, - { - unsafe { crate::vec::GcVec::from_raw(self.alloc_raw_vec_with_capacity::(0)) } - } - /// Allocate a new [GcVec] with the specified capacity - /// and an implicit reference to this [GcContext] - #[inline] - fn alloc_vec_with_capacity<'gc, T>(&'gc self, capacity: usize) -> GcVec<'gc, T, Self::Id> - where - T: GcSafe<'gc, Self::Id>, - { - unsafe { crate::vec::GcVec::from_raw(self.alloc_raw_vec_with_capacity::(capacity)) } - } } /// The internal representation of a frozen context /// @@ -694,13 +535,6 @@ pub unsafe trait CollectorId: type System: GcSystem; /// The type of [GcContext] associated with this id. type Context: GcContext; - /// The implementation of [GcRawVec] for this type. - /// - /// May be [crate::vec::raw::Unsupported] if vectors are unsupported. - type RawVec<'gc, T: GcSafe<'gc, Self>>: crate::vec::raw::GcRawVec<'gc, T, Id = Self>; - /// The raw representation of `GcArray` pointers - /// in this collector. - type ArrayPtr: crate::array::repr::GcArrayPtr; /// Get the runtime id of the collector that allocated the [Gc] /// @@ -711,16 +545,6 @@ pub unsafe trait CollectorId: T: ?Sized, 'gc: 'a; - /// Resolve the CollectorId for the specified [GcArray]'s representation. - /// - /// This is the [GcArray] counterpart of `from_gc_ptr` - fn resolve_array_id<'a, 'gc, T>(array: &'a GcArray<'gc, T, Self>) -> &'a Self - where - 'gc: 'a; - - /// Resolve the length of the specified array - fn resolve_array_len(repr: &GcArray<'_, T, Self>) -> usize; - /// Perform a write barrier before writing to a garbage collected field /// /// ## Safety @@ -1515,37 +1339,4 @@ pub unsafe trait GcVisitor: Sized { where T: GcSafe<'gc, Id>, Id: CollectorId; - - /// Visit a garbage collected trait object. - /// - /// ## Safety - /// The trait object must point to a garbage collected object. - unsafe fn trace_trait_object<'gc, T, Id>( - &mut self, - gc: &mut Gc<'gc, T, Id>, - ) -> Result<(), Self::Err> - where - T: ?Sized + GcSafe<'gc, Id> + Pointee> + DynTrace<'gc, Id>, - Id: CollectorId; - - /// Visit a garbage collected vector. - /// - /// ## Safety - /// Undefined behavior if the vector is invalid. - unsafe fn trace_vec<'gc, T, V>(&mut self, raw: &mut V) -> Result<(), Self::Err> - where - T: GcSafe<'gc, V::Id>, - V: GcRawVec<'gc, T>; - - /// Visit a garbage collected array. - /// - /// ## Safety - /// Undefined behavior if the array is invalid. - unsafe fn trace_array<'gc, T, Id>( - &mut self, - array: &mut GcArray<'gc, T, Id>, - ) -> Result<(), Self::Err> - where - T: GcSafe<'gc, Id>, - Id: CollectorId; } diff --git a/src/macros.rs b/src/macros.rs index 681c947..d17f810 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -1,79 +1,4 @@ -/// Implement [Trace](`crate::Trace`) for a dynamically dispatched trait object -/// -/// This requires that the trait object extends [DynTrace](`crate::DynTrace`). -/// -/// ## Example -/// ``` -/// # use zerogc::{Trace, DynTrace, trait_object_trace}; -/// # use zerogc::epsilon::{self, EpsilonCollectorId, Gc}; -/// # type OurSpecificId = EpsilonCollectorId; -/// trait Foo<'gc>: DynTrace<'gc, OurSpecificId> { -/// fn method(&self) -> i32; -/// } -/// trait_object_trace!( -/// impl<'gc,> Trace for dyn Foo<'gc>; -/// Branded<'new_gc> => (dyn Foo<'new_gc> + 'new_gc), -/// collector_id => OurSpecificId, -/// gc_lifetime => 'gc -/// ); -/// fn foo<'gc, T: ?Sized + Trace + Foo<'gc>>(t: &T) -> i32 { -/// assert_eq!(t.method(), 12); -/// t.method() * 2 -/// } -/// fn bar<'gc>(gc: Gc<'gc, dyn Foo<'gc> + 'gc>) -> i32 { -/// foo(gc.value()) -/// } -/// #[derive(Trace)] -/// # #[zerogc(collector_ids(EpsilonCollectorId))] -/// struct Bar<'gc> { -/// val: Gc<'gc, i32> -/// } -/// impl<'gc> Foo<'gc> for Bar<'gc> { -/// fn method(&self) -> i32 { -/// *self.val -/// } -/// } -/// let val = epsilon::leaked(12); -/// let gc: Gc<'_, Bar<'_>> = epsilon::leaked(Bar { val }); -/// assert_eq!(bar(gc as Gc<'_, dyn Foo>), 24); -/// ``` -/// -/// ## Safety -/// This macro is completely safe. -#[macro_export] -macro_rules! trait_object_trace { - (impl $(<$($lt:lifetime,)* $($param:ident),*>)? Trace for dyn $target:path $({ where $($where_clause:tt)* })?; - Branded<$branded_lt:lifetime> => $branded:ty, - collector_id => $collector_id:path, - gc_lifetime => $gc_lt:lifetime) => { - unsafe impl$(<$($lt,)* $($param),*>)? $crate::TrustedDrop for (dyn $target + $gc_lt) where Self: $crate::DynTrace<$gc_lt, $collector_id>, $($($where_clause)*)? {} - unsafe impl$(<$($lt,)* $($param),*>)? $crate::GcSafe<$gc_lt, $collector_id> for (dyn $target + $gc_lt) where Self: $crate::DynTrace<$gc_lt, $collector_id>, $($($where_clause)*)? { - #[inline] - unsafe fn trace_inside_gc(gc: &mut $crate::Gc<$gc_lt, Self, $collector_id>, visitor: &mut V) -> Result<(), V::Err> - where V: $crate::GcVisitor { - visitor.trace_trait_object(gc) - } - - } - unsafe impl$(<$($lt,)* $($param),*>)? $crate::Trace for (dyn $target + $gc_lt) where Self: $crate::DynTrace::<$gc_lt, $collector_id>, $($($where_clause)*)? { - /* - * Insufficient compile-time information to know whether we need to be traced. - * - * Therefore, we make a conservative estimate - */ - const NEEDS_TRACE: bool = true; - // Likewise for `NEEDS_DROP` - const NEEDS_DROP: bool = true; - - fn trace(&mut self, _visitor: &mut V) -> Result<(), V::Err> { - unimplemented!("Unable to use DynTrace outside of a Gc") - } - } - unsafe impl<$branded_lt, $($($lt,)* $($param,)*)?> $crate::GcRebrand<$branded_lt, $collector_id> for (dyn $target + $gc_lt) $(where $($where_clause)*)? { - type Branded = $branded; - } - } -} +//! Miscellaneous macros for `zerogc` /// Implement [Trace](`crate::Trace`) and [TraceImmutable](`crate::TraceImmutable`) as a no-op, /// based on the fact that a type implements [NullTrace](crate::NullTrace) diff --git a/src/prelude.rs b/src/prelude.rs index 7197d0c..0453c71 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -10,10 +10,7 @@ pub use crate::{freeze_context, safepoint, safepoint_recurse, unfreeze_context}; // Basic collector types pub use crate::{Gc, GcContext, GcHandle, GcSimpleAlloc, GcSystem, GcVisitor, HandleCollectorId}; // Traits for user code to implement -pub use crate::{GcRebrand, GcSafe, NullTrace, Trace, TraceImmutable, TrustedDrop}; -// TODO: Should this trait be auto-imported??? -pub use crate::array::{GcArray, GcString}; pub use crate::cell::GcCell; -pub use crate::vec::GcVec; pub use crate::AssumeNotTraced; pub use crate::CollectorId; +pub use crate::{GcRebrand, GcSafe, NullTrace, Trace, TraceImmutable, TrustedDrop}; diff --git a/src/serde.rs b/src/serde.rs deleted file mode 100644 index 0bd3aa8..0000000 --- a/src/serde.rs +++ /dev/null @@ -1,384 +0,0 @@ -//! Support for deserializing garbage collected types -//! -//! As long as you aren't worried about cycles, serialization is easy. -//! Just do `#[derive(Serialize)]` on your type. -//! -//! Deserialization is much harder, because allocating a [Gc] requires -//! access to [GcSimpleAlloc] and [serde::de::DeserializeSeed] can't be automatically derived. -//! -//! As a workaround, zerogc introduces a `GcDeserialize` type, -//! indicating an implementation of [serde::Deserialize] -//! that requires a [GcContext]. -use std::collections::{HashMap, HashSet}; -use std::hash::{BuildHasher, Hash}; -use std::marker::PhantomData; - -use serde::de::{self, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; -use serde::ser::SerializeSeq; -use serde::Serialize; - -#[cfg(feature = "indexmap")] -use indexmap::{IndexMap, IndexSet}; - -use crate::array::{GcArray, GcString}; -use crate::prelude::*; - -#[doc(hidden)] -#[macro_use] -pub mod hack; - -/// An implementation of [serde::Deserialize] that requires a [GcContext] for allocation. -/// -/// The type must be [GcSafe], so that it can actually be allocated. -pub trait GcDeserialize<'gc, 'de, Id: CollectorId>: GcSafe<'gc, Id> + Sized { - /// Deserialize the value given the specified context - fn deserialize_gc>( - ctx: &'gc Id::Context, - deserializer: D, - ) -> Result; -} - -/// A garbage collected type that can be deserialized without borrowing any data. -/// -/// [GcDeserialize] is to [`serde::de::Deserialize`] -/// as [GcDeserializeOwned] is to [`serde::de::DeserializeOwned`] -pub trait GcDeserializeOwned<'gc, Id: CollectorId>: for<'de> GcDeserialize<'gc, 'de, Id> {} -impl<'gc, Id, T> GcDeserializeOwned<'gc, Id> for T -where - Id: CollectorId, - T: for<'de> GcDeserialize<'gc, 'de, Id>, -{ -} - -impl<'gc, 'de, Id: CollectorId, T: GcDeserialize<'gc, 'de, Id>> GcDeserialize<'gc, 'de, Id> - for Gc<'gc, T, Id> -where - Id::Context: GcSimpleAlloc, -{ - #[inline] - fn deserialize_gc>( - ctx: &'gc Id::Context, - deserializer: D, - ) -> Result { - Ok(ctx.alloc(T::deserialize_gc(ctx, deserializer)?)) - } -} - -impl<'gc, 'de, Id: CollectorId, T: GcDeserialize<'gc, 'de, Id>> GcDeserialize<'gc, 'de, Id> - for GcArray<'gc, T, Id> -where - Id::Context: GcSimpleAlloc, -{ - fn deserialize_gc>( - ctx: &'gc Id::Context, - deserializer: D, - ) -> Result { - Ok(ctx.alloc_array_from_vec(Vec::::deserialize_gc(ctx, deserializer)?)) - } -} - -impl<'gc, 'de, Id: CollectorId> GcDeserialize<'gc, 'de, Id> for GcString<'gc, Id> -where - Id::Context: GcSimpleAlloc, -{ - fn deserialize_gc>( - ctx: &'gc Id::Context, - deserializer: D, - ) -> Result { - struct GcStrVisitor<'gc, A: GcSimpleAlloc> { - ctx: &'gc A, - } - impl<'de, 'gc, A: GcSimpleAlloc> de::Visitor<'de> for GcStrVisitor<'gc, A> { - type Value = GcString<'gc, A::Id>; - fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str("a string") - } - fn visit_str(self, v: &str) -> Result - where - E: de::Error, - { - Ok(self.ctx.alloc_str(v)) - } - } - deserializer.deserialize_str(GcStrVisitor { ctx }) - } -} - -impl<'gc, T: Serialize, Id: CollectorId> Serialize for Gc<'gc, T, Id> { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - self.value().serialize(serializer) - } -} - -impl<'gc, T: Serialize, Id: CollectorId> Serialize for GcArray<'gc, T, Id> { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let mut seq = serializer.serialize_seq(Some(self.len()))?; - for val in self.as_slice().iter() { - seq.serialize_element(val)?; - } - seq.end() - } -} - -impl<'gc, Id: CollectorId> Serialize for GcString<'gc, Id> { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'gc, 'de, T, Id: CollectorId> GcDeserialize<'gc, 'de, Id> for PhantomData { - fn deserialize_gc>( - _ctx: &'gc Id::Context, - _deserializer: D, - ) -> Result { - Ok(PhantomData) - } -} - -impl Serialize for GcCell { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - self.get().serialize(serializer) - } -} - -impl<'gc, 'de, T, Id> GcDeserialize<'gc, 'de, Id> for GcCell -where - T: Copy + GcDeserialize<'gc, 'de, Id>, - Id: CollectorId, -{ - fn deserialize_gc>( - ctx: &'gc Id::Context, - deser: D, - ) -> Result { - Ok(GcCell::new(T::deserialize_gc(ctx, deser)?)) - } -} - -impl<'gc, 'de, Id: CollectorId> GcDeserialize<'gc, 'de, Id> for () { - fn deserialize_gc>( - _ctx: &'gc Id::Context, - deserializer: D, - ) -> Result { - struct UnitVisitor; - impl<'de> Visitor<'de> for UnitVisitor { - type Value = (); - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("a unit tuple") - } - fn visit_unit(self) -> Result - where - E: de::Error, - { - Ok(()) - } - } - deserializer.deserialize_unit(UnitVisitor) - } -} - -/// Implement [GcDeserialize] for a type by delegating to its [serde::Deserialize] implementation. -/// -/// This should only be used for types that can never have gc pointers inside of them (or if you don't care to support that). -#[macro_export] -macro_rules! impl_delegating_deserialize { - (impl GcDeserialize for $target:path) => ( - $crate::impl_delegating_deserialize!(impl <'gc, 'de, Id> GcDeserialize<'gc, 'de, Id> for $target where Id: zerogc::CollectorId); - ); - (impl $(<$($lt:lifetime,)* $($param:ident),*>)? GcDeserialize<$gc:lifetime, $de:lifetime, $id:ident> for $target:path $(where $($where_clause:tt)*)?) => { - impl$(<$($lt,)* $($param),*>)? $crate::serde::GcDeserialize<$gc, $de, $id> for $target - where Self: Deserialize<$de> + $(, $($where_clause)*)?{ - fn deserialize_gc>(_ctx: &$gc <$id as $crate::CollectorId>::Context, deserializer: D) -> Result>::Error> { - >::deserialize(deserializer) - } - } - }; -} - -/// An implementation of [serde::de::DeserializeSeed] that wraps [GcDeserialize] -pub struct GcDeserializeSeed<'gc, 'de, Id: CollectorId, T: GcDeserialize<'gc, 'de, Id>> { - context: &'gc Id::Context, - marker: PhantomData T>, -} -impl<'de, 'gc, Id: CollectorId, T: GcDeserialize<'gc, 'de, Id>> GcDeserializeSeed<'gc, 'de, Id, T> { - /// Create a new wrapper for the specified context - #[inline] - pub fn new(context: &'gc Id::Context) -> Self { - GcDeserializeSeed { - context, - marker: PhantomData, - } - } -} -impl<'de, 'gc, Id: CollectorId, T: GcDeserialize<'gc, 'de, Id>> DeserializeSeed<'de> - for GcDeserializeSeed<'gc, 'de, Id, T> -{ - type Value = T; - - fn deserialize(self, deserializer: D) -> Result - where - D: Deserializer<'de>, - { - T::deserialize_gc(self.context, deserializer) - } -} - -macro_rules! impl_for_map { - ($target:ident $(where $($bounds:tt)*)?) => { - impl<'gc, 'de, Id: CollectorId, - K: Eq + Hash + GcDeserialize<'gc, 'de, Id>, - V: GcDeserialize<'gc, 'de, Id>, - S: BuildHasher + Default - > GcDeserialize<'gc, 'de, Id> for $target $(where $($bounds)*)* { - fn deserialize_gc>(ctx: &'gc Id::Context, deserializer: D) -> Result { - struct MapVisitor< - 'gc, 'de, Id: CollectorId, - K: GcDeserialize<'gc, 'de, Id>, - V: GcDeserialize<'gc, 'de, Id>, - S: BuildHasher + Default - > { - ctx: &'gc Id::Context, - marker: PhantomData<(&'de S, K, V)> - } - impl<'gc, 'de, Id: CollectorId, - K: Eq + Hash + GcDeserialize<'gc, 'de, Id>, - V: GcDeserialize<'gc, 'de, Id>, - S: BuildHasher + Default - > Visitor<'de> for MapVisitor<'gc, 'de, Id, K, V, S> { - type Value = $target; - fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str(concat!("a ", stringify!($target))) - } - #[inline] - fn visit_map(self, mut access: A) -> Result - where A: MapAccess<'de>, { - let mut values = $target::::with_capacity_and_hasher( - access.size_hint().unwrap_or(0).min(1024), - S::default() - ); - while let Some((key, value)) = access.next_entry_seed( - GcDeserializeSeed::new(self.ctx), - GcDeserializeSeed::new(self.ctx) - )? { - values.insert(key, value); - } - - Ok(values) - } - } - let visitor: MapVisitor = MapVisitor { ctx, marker: PhantomData }; - deserializer.deserialize_map(visitor) - } - } - }; -} - -macro_rules! impl_for_set { - ($target:ident $(where $($bounds:tt)*)?) => { - impl<'gc, 'de, Id: CollectorId, - T: Eq + Hash + GcDeserialize<'gc, 'de, Id>, - S: BuildHasher + Default - > GcDeserialize<'gc, 'de, Id> for $target $(where $($bounds)*)* { - fn deserialize_gc>(ctx: &'gc Id::Context, deserializer: D) -> Result { - struct SetVisitor< - 'gc, 'de, Id: CollectorId, - T: GcDeserialize<'gc, 'de, Id>, - S: BuildHasher + Default - > { - ctx: &'gc Id::Context, - marker: PhantomData T> - } - impl<'gc, 'de, Id: CollectorId, - T: Eq + Hash + GcDeserialize<'gc, 'de, Id>, - S: BuildHasher + Default - > Visitor<'de> for SetVisitor<'gc, 'de, Id, T, S> { - type Value = $target; - fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str(concat!("a ", stringify!($target))) - } - #[inline] - fn visit_seq(self, mut access: A) -> Result - where A: SeqAccess<'de>, { - let mut values = $target::::with_capacity_and_hasher( - access.size_hint().unwrap_or(0).min(1024), - S::default() - ); - while let Some(value) = access.next_element_seed( - GcDeserializeSeed::new(self.ctx) - )? { - values.insert(value); - } - - Ok(values) - } - } - let visitor: SetVisitor = SetVisitor { ctx, marker: PhantomData }; - deserializer.deserialize_seq(visitor) - } - } - }; -} - -impl_for_map!(HashMap where K: TraceImmutable, S: 'static); -impl_for_set!(HashSet where T: TraceImmutable, S: 'static); -#[cfg(feature = "indexmap")] -impl_for_map!(IndexMap where K: GcSafe<'gc, Id>, S: 'static); -#[cfg(feature = "indexmap")] -impl_for_set!(IndexSet where T: TraceImmutable, S: 'static); - -#[cfg(test)] -mod test { - use super::*; - use crate::epsilon::{EpsilonCollectorId, EpsilonSystem}; - #[test] - #[cfg(feature = "indexmap")] - fn indexmap() { - let system = EpsilonSystem::leak(); - let ctx = system.new_context(); - const INPUT: &str = r##"{"foo": "bar", "eats": "turds"}"##; - let mut deser = serde_json::Deserializer::from_str(INPUT); - let s = |s: &'static str| String::from(s); - assert_eq!( - > as GcDeserialize< - EpsilonCollectorId, - >>::deserialize_gc(&ctx, &mut deser) - .unwrap(), - indexmap::indexmap!( - s("foo") => ctx.alloc(s("bar")), - s("eats") => ctx.alloc(s("turds")) - ) - ); - let mut deser = serde_json::Deserializer::from_str(INPUT); - assert_eq!( - , fnv::FnvBuildHasher> as GcDeserialize>::deserialize_gc(&ctx, &mut deser).unwrap(), - indexmap::indexmap!( - s("foo") => ctx.alloc(s("bar")), - s("eats") => ctx.alloc(s("turds")) - ) - ); - } - #[test] - fn gc() { - let system = EpsilonSystem::leak(); - let ctx = system.new_context(); - let mut deser = serde_json::Deserializer::from_str(r#"128"#); - assert_eq!( - as GcDeserialize>::deserialize_gc( - &ctx, &mut deser - ) - .unwrap(), - ctx.alloc(128) - ); - } -} diff --git a/src/serde/hack.rs b/src/serde/hack.rs deleted file mode 100644 index 0a3c407..0000000 --- a/src/serde/hack.rs +++ /dev/null @@ -1,302 +0,0 @@ -//! A horrible hack to pass `GcContext` back and forth to serde using thread locals. -use std::any::TypeId; -use std::cell::{Cell, RefCell, UnsafeCell}; -use std::collections::hash_map::Entry; -use std::collections::HashMap; -use std::ffi::c_void; -use std::marker::PhantomData; -use std::ptr::NonNull; - -use crate::prelude::*; -use crate::serde::GcDeserialize; -use serde::{Deserialize, Deserializer}; - -struct ContextHackState { - current_ctx: UnsafeCell>, - /// The number of active references to the context. - /// - /// If this is zero, then `state` should be `None`, - /// otherwise it should be `Some` - active_refs: Cell, -} -impl ContextHackState { - const fn uninit() -> ContextHackState { - ContextHackState { - current_ctx: UnsafeCell::new(None), - active_refs: Cell::new(0), - } - } - #[inline] - unsafe fn get_unchecked(&self) -> Option<&ContextHack> { - if self.active_refs.get() == 0 { - None - } else { - Some((&*self.current_ctx.get()).as_ref().unwrap_unchecked()) - } - } - unsafe fn lock_unchecked<'gc, C: GcContext>(&self) -> ContextHackGuard<'gc, C> { - self.active_refs.set(self.active_refs.get() + 1); - debug_assert_eq!( - TypeId::of::(), - self.get_unchecked().unwrap().collector_type_id - ); - ContextHackGuard { - state: NonNull::from(self), - marker: PhantomData, - } - } - #[inline] - unsafe fn release_lock(&self) -> bool { - debug_assert!(self.active_refs.get() > 0); - match self.active_refs.get() { - 1 => { - self::unregister_context(self); - true - } - 0 => std::hint::unreachable_unchecked(), - _ => { - self.active_refs.set(self.active_refs.get() - 1); - false - } - } - } -} -/// A hack to store a dynamically typed [GcContext] in a thread-local. -struct ContextHack { - collector_type_id: TypeId, - ptr: NonNull, -} -impl ContextHack { - /// Cast this context into the specified type. - /// - /// Returns `None` if the id doesn't match - #[inline] - pub fn cast_as(&self) -> Option<&'_ Ctx> { - if TypeId::of::() == self.collector_type_id { - Some(unsafe { &*(self.ptr.as_ptr() as *mut Ctx) }) - } else { - None - } - } -} -thread_local! { - static PRIMARY_DE_CONTEXT: ContextHackState = ContextHackState::uninit(); - static OTHER_DE_CONTEXT: RefCell>> = RefCell::new(HashMap::new()); -} - -pub struct ContextHackGuard<'gc, C: GcContext> { - state: NonNull, - marker: PhantomData<&'gc C>, -} -impl<'gc, C: GcContext> ContextHackGuard<'gc, C> { - /// Get the guard's underlying context. - /// - /// ## Safety - /// Undefined behavior if this guard is dropped - /// and then subsequently mutated. - /// - /// Undefined behavior if a safepoint occurs (although the immutable reference should prevent that) - #[inline] - pub unsafe fn get_unchecked(&self) -> &'gc C { - &*self - .state - .as_ref() - .get_unchecked() - .unwrap_unchecked() - .ptr - .cast::() - .as_ptr() - } -} -impl<'gc, C: GcContext> Drop for ContextHackGuard<'gc, C> { - #[inline] - fn drop(&mut self) { - unsafe { - self.state.as_ref().release_lock(); - } - } -} -/// Temporarily places the specified gc context -/// in a thread-local, to allow deserialization with serde. -/// -/// Panics if another context of the same type is -/// already in the process of being deserialized. -/// -/// ## Safety -/// Undefined behavior if the context is mutated or ever used with the wrong -/// lifetime (although this is technically part of the [current_ctx] contract) -#[track_caller] -pub unsafe fn set_context(ctx: &C) -> ContextHackGuard<'_, C> { - let guard = PRIMARY_DE_CONTEXT.with(|state| match state.get_unchecked() { - Some(other) => { - if let Some(other) = other.cast_as::() { - assert_eq!(other.id(), ctx.id(), "Multiple collectors of the same type"); - Some(state.lock_unchecked()) - } else { - None - } - } - None => { - state.current_ctx.get().write(Some(ContextHack { - collector_type_id: TypeId::of::(), - ptr: NonNull::from(ctx).cast(), - })); - Some(state.lock_unchecked()) - } - }); - guard.unwrap_or_else(|| _fallback_set_context(ctx)) -} -#[cold] -unsafe fn _fallback_set_context(ctx: &C) -> ContextHackGuard<'_, C> { - OTHER_DE_CONTEXT.with(|map| { - let mut map = map.borrow_mut(); - match map.entry(TypeId::of::()) { - Entry::Occupied(occupied) => { - let other = occupied.get(); - assert_eq!( - other.get_unchecked().unwrap().cast_as::().unwrap().id(), - ctx.id(), - "Multiple collectors of the same type" - ); - other.lock_unchecked() - } - Entry::Vacant(entry) => { - let res = entry.insert(Box::new(ContextHackState { - active_refs: Cell::new(0), - current_ctx: UnsafeCell::new(Some(ContextHack { - collector_type_id: TypeId::of::(), - ptr: NonNull::from(ctx).cast(), - })), - })); - res.lock_unchecked() - } - } - }) -} -#[cold] -unsafe fn unregister_context(state: &ContextHackState) { - let expected_ctx = state.get_unchecked().unwrap_unchecked(); - assert_eq!(state.active_refs.get(), 1); - let needs_fallback_free = PRIMARY_DE_CONTEXT.with(|ctx| { - if let Some(actual) = ctx.get_unchecked() { - if actual.collector_type_id == expected_ctx.collector_type_id { - debug_assert_eq!(actual.ptr.as_ptr(), expected_ctx as *const _ as *mut _); - ctx.active_refs.set(0); - ctx.current_ctx.get().write(None); - return false; // don't search the fallback HashMap. We're freed the old fashioned way - } - } - true // need to fallback to search HashMap - }); - if needs_fallback_free { - OTHER_DE_CONTEXT.with(|map| { - let mut map = map.borrow_mut(); - let actual_state = map - .remove(&expected_ctx.collector_type_id) - .unwrap_or_else(|| unreachable!("Can't find collector in serde::hack state")); - debug_assert_eq!( - expected_ctx.collector_type_id, - actual_state.get_unchecked().unwrap().collector_type_id - ); - debug_assert_eq!( - actual_state.get_unchecked().unwrap() as *const _, - expected_ctx as *const _ - ); - actual_state.as_ref().active_refs.set(0); - drop(actual_state); - }) - } -} - -/// Get the current context for deserialization -/// -/// ## Safety -/// The inferred lifetime must be correct. -#[track_caller] -pub unsafe fn current_ctx<'gc, Id: CollectorId>() -> ContextHackGuard<'gc, Id::Context> { - PRIMARY_DE_CONTEXT - .with(|state| match state.get_unchecked() { - Some(hack) if hack.collector_type_id == TypeId::of::() => { - Some(state.lock_unchecked()) - } - _ => None, - }) - .unwrap_or_else(|| _fallback_current_ctx::<'gc, Id>()) -} -#[cold] -#[track_caller] -unsafe fn _fallback_current_ctx<'gc, Id: CollectorId>() -> ContextHackGuard<'gc, Id::Context> { - OTHER_DE_CONTEXT.with(|map| { - let map = map.borrow(); - let state = map.get(&TypeId::of::()).unwrap_or_else(|| { - unreachable!( - "Can't find collector for {} in serde::hack state", - std::any::type_name::() - ) - }); - state.lock_unchecked() - }) -} - -/// Wrapper function to deserialize the specified value via the "hack", -/// getting the current context via [current_ctx] -/// -/// ## Safety -/// The contract of [current_de_ctx] must be upheld. -/// In other words, the current context must've been set by [set_context] and have the appropriate lifetime -#[track_caller] -pub unsafe fn unchecked_deserialize_hack< - 'gc, - 'de, - D: Deserializer<'de>, - Id: CollectorId, - T: GcDeserialize<'gc, 'de, Id>, ->( - deserializer: D, -) -> Result { - let guard = current_ctx::<'gc, Id>(); - T::deserialize_gc(guard.get_unchecked(), deserializer) -} - -#[repr(transparent)] -#[derive(Eq, Hash, Debug, PartialEq, Clone)] -pub struct DeserializeHackWrapper(T, PhantomData); - -impl<'gc, 'de, Id: CollectorId, T: GcDeserialize<'gc, 'de, Id>> Deserialize<'de> - for DeserializeHackWrapper -{ - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let guard = unsafe { current_ctx::<'gc, Id>() }; - Ok(DeserializeHackWrapper( - T::deserialize_gc(unsafe { guard.get_unchecked() }, deserializer)?, - PhantomData, - )) - } -} - -/// Transmute between two types whose sizes may not be equal at compile time -/// -/// ## Safety -/// All the usual cavets of [std::mem::transmute] apply. -/// -/// However, the sizes aren't verified -/// to be the same size at compile time. -/// -/// It is undefined behavior to invoke this function with types whose sizes -/// don't match at runtime. -/// However, in the current implementation, this will cause a panic. -#[inline] -pub unsafe fn transmute_mismatched(src: T) -> U { - assert_eq!( - std::mem::size_of::(), - std::mem::size_of::(), - "UB: Mismatched sizes for {} and {}", - std::any::type_name::(), - std::any::type_name::() - ); - let src = std::mem::ManuallyDrop::new(src); - std::mem::transmute_copy::(&*src) -} diff --git a/src/vec.rs b/src/vec.rs deleted file mode 100644 index 06c3228..0000000 --- a/src/vec.rs +++ /dev/null @@ -1,585 +0,0 @@ -//! Garbage collected vectors. -//! -//! There are three different vector types in zerogc -//! 1. `GcVec` - Requires unique ownership `!Copy`, -//! but coerces directly to a slice and implements `Index` -//! 2. `GcVecCell` - Is `Copy`, but uses a [RefCell] to guard coercion to a slice along with all other accesses. -//! - Since every operation implicitly calls `borrow`, *any* operation may panic. -//! 3. `GcRawVec` - Is `Copy`, but coercion to slice is `unsafe`. -//! - Iteration implicitly panics if the length changes. -//! -//! Here's a table comparing them: -//! -//! | Supported Features | [`GcVec`] | [`GcRawVec`] | [`GcVecCell`] | -//! |----------------------|--------------|-------------------|----------------| -//! | Shared ownership? | No - `!Copy` | Yes, is `Copy` | Yes, is `Copy` | -//! | Can coerce to slice? | Easiy `Deref` | No (is `unsafe`) | Needs `borrow()` -> `cell::Ref<[T]>` | -//! | impl `std::ops::Index` | Yes. | No. | No (but can `borrow` as `GcVec` which does) | -//! | Runtime borrow checks? | No. | Only for `iter` | Yes | -//! | Best stdlib analogy | `Vec` | `Rc>>` | `Rc>>` | -//! -//! All vector types are `!Send`, because all garbage collected vectors -//! have an [implicit reference to the GcContext](`crate::vec::IGcVec::context`). -//! -//! ## The shared mutability problem -//! Because a vector's length is mutable, garbage collected vectors are significantly more complicated -//! than [garbage collected arrays](`crate::array::GcArray`). -//! Rust's memory safety is based around the distinction between mutable and immutable references -//! `&mut T` and `&T`.Mutable references `&mut T` must be uniquely owned and cannot be duplicated, -//! while immutable references `&T` can be freely duplicated and shared, -//! but cannot be duplicated. This has been called "aliasing XOR mutability". -//! -//! This becomes a significant problem with garbage collected references like [`Gc`](`crate::Gc`), -//! because they can be copied freely (they implement `Copy`). -//! Just like [`Rc`](`std::rc::Rc`), this means they can only give out immutable references `&T` -//! by difference. With vectors this makes it impossible to mutate the length (ie. call `push`) -//! without interior mutability like RefCell. -//! -//! This is a problem for languages beyond just rust, because of issues like iterator invalidation. -//! If we have two references to a vector, a `push` on one vector may mutate the length while an iteration -//! on the other vector is still in progress. -//! -//! To put it another way, it's impossible for a garbage collected vector/list to implement all of the following properties: -//! 1. Coerce the vector directly into a slice or pointer to the underlying memory. -//! 2. Allow multiple references to the vector (implement ~Copy`) -//! 3. The ability to mutate the length and potentially reallocate. -//! -//! ### What Java/Python do -//! Most languages sidestep the problem by forbidding option 1 -//! and refusing to give direct pointer-access to garbage-collected arrays. -//! Usually, forbidding option 1 is not a problem, since arrays in Java -//! and lists in Python are builtin into the language and replace most of the users of pointers. -//! -//! There are some exceptions though. Java's JNI allows raw access to an arrays's memory with `GetPrimitiveArrayCritical`. -//! However, there is no need to worry about mutating length because java arrays have a fixed -//! size (although it does temporarily prevent garbage collection). -//! Python's native CAPI exposes access to a list's raw memory with `PyListObject->ob_items`, -//! although native code must be careful of any user code changing the length. -//! -//! This problem can also come up in unexpected places. -//! For example, the C implementation of Python's `list.sort` coerces the list into a raw pointer, -//! then sorts the memory directly using the pointers. The original code for that function -//! continued to use the original pointer even if a user comparison, leading to bugs. -//! The current implementation temporarily sets the length to zero and carefully guards against any mutations. -//! -//! ### What Rust/C++ do (in their stdlib) -//! Rust and C++ tend to restrict the second option, allowing only a single -//! reference to a vector by default. -//! -//! Rust's ownership system and borrow checker ensures that `Vec` only has a single owner. -//! Changing the length requires a a uniquely owned `&mut T` reference. -//! Mutating the length while iterating is statically impossible -//! -//! In C++, you *should* only have one owner, but this can't be statically verified. -//! Mutating the length while iterating is undefined behavior. -//! -//! In Rust, you can have duplicate ownership by wrapping in a `Rc`, -//! and you can allow runtime-checked mutation by wrapping in a `RefCell`. -//! -//! ### The zerogc solution -//! `zerogc` allows the user to pick their guarantees. -//! -//! If you are fine with unique ownership, you can use [`GcVec`]. -//! Unlike most garbage collected types, this allows mutable access -//! to the contents. It can even coerce to a `&mut [T]` reference. -//! -//! If you want multiple ownership, you are left with two options: -//! 1. `GcVecCell` which is essentially `Gc>>`. It has some runtime overhead, -//! but allows coercion to `&[T]` (and even `&mut [T]`) -//! 2. `GcRawVec` which *doesn't* allow coercion to `&[T]`, but avoids runtime borrow checks. -//! -//! `GcRawVec` is probably the most niche of all the vector types and it doesn't really -//! have a direct analogue with any standard library types. It is probably most similar to -//! a `Cell< Gc< Vec> >>`. -//! -//! Calls to `get` yield `T` instead of `&T` require `T: Copy` (just like [`Cell`](`core::cell::Cell`)), -//! because there is no way to guarantee the length wont be mutated or the element won't be `set` -//! while the reference is in use. -//! -//! A `GcRawVec` may never give out references or slices directly to its contents, -//! because other references may trigger reallocation at any time (via a `push`). -//! -//! NOTE: A similar problem occurs with asynchronous, concurrent collectors. -//! It has been called the [stretchy vector problem](https://www.ravenbrook.com/project/mps/master/manual/html/guide/vector.html) -//! by some. This is less of a problem for `zerogc`, because collections can only -//! happen at explicit safepoints. -#[cfg(all(not(feature = "std"), feature = "alloc"))] -use alloc::vec::Vec; -use core::cell::UnsafeCell; -use core::fmt::{self, Debug, Formatter}; -use core::marker::PhantomData; -use core::ops::{Deref, DerefMut, Index, IndexMut, RangeBounds}; -use core::ptr::NonNull; -use core::slice::SliceIndex; - -use inherent::inherent; -use zerogc_derive::unsafe_gc_impl; - -use crate::vec::raw::ReallocFailedError; -use crate::{CollectorId, GcRebrand, GcSafe, Trace}; - -pub mod cell; -pub mod raw; - -pub use self::cell::GcVecCell; -pub use self::raw::{GcRawVec, IGcVec}; - -/// A uniquely owned [Vec] for use with garbage collectors. -/// -/// See the [module docs](`zerogc::vec`) for details -/// on why this can not be `Copy`. -/// -/// On the other hand, `Clone` can be implemented -/// although this does a *deep copy* (just like `Vec::clone`). -pub struct GcVec<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> { - /* - * NOTE: This must be `UnsafeCell` - * so that we can implement `TraceImmutable` - */ - raw: UnsafeCell>, -} -unsafe impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> crate::ImplicitWriteBarrier - for GcVec<'gc, T, Id> -{ -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> GcVec<'gc, T, Id> { - /// Create a [GcVec] from a [GcRawVec]. - /// - /// ## Safety - /// There must be no other references to the specified raw vector. - #[inline] - pub unsafe fn from_raw(raw: Id::RawVec<'gc, T>) -> Self { - GcVec { - raw: UnsafeCell::new(raw), - } - } - /// Consume ownership of this vector, converting it into its corresponding [`GcArray`](`zerogc::array::GcArray`) - /// - /// Depending on the collector, this may need to copy the values into a new allocation. - /// Although it should reuse the existing memory wherever possible. - #[inline] - pub fn into_array(self) -> crate::GcArray<'gc, T, Id> { - unsafe { self.into_raw().steal_as_array_unchecked() } - } - /// Convert this vector into its underlying [GcRawVec](`zerogc::vec::raw::GcRawVec`) - /// - /// ## Safety - /// Because this consumes ownership, - /// it is safe. - #[inline] - pub fn into_raw(self) -> Id::RawVec<'gc, T> { - self.raw.into_inner() - } - /// Slice this vector's elements - /// - /// NOTE: The borrow is bound to the lifetime - /// of this vector, and not `&'gc [T]` - /// because of the possibility of calling `as_mut_slice` - /// - /// ## Safety - /// Because this vector is uniquely owned, - /// its length cannot be mutated while it is still borrowed. - /// - /// This avoids the pitfalls of calling [IGcVec::as_slice_unchecked] - /// on a vector with shared references. - #[inline] - pub fn as_slice(&self) -> &'_ [T] { - // SAFETY: We are the only owner - unsafe { self.as_raw().as_slice_unchecked() } - } - /// Get a mutable slice of this vector's elements. - /// - /// ## Safety - /// Because this vector is uniquely owned, - /// the underlying contents cannot be modified - /// while another reference is in use (because there are no other references) - #[inline] - pub fn as_mut_slice(&mut self) -> &'_ mut [T] { - unsafe { core::slice::from_raw_parts_mut(self.as_mut_raw().as_mut_ptr(), self.len()) } - } - /// Get a reference to the underlying [GcRawVec](`zerogc::vec::raw::GcRawVec`), - /// bypassing restrictions on unique ownership. - /// - /// ## Safety - /// It is undefined behavior to change the length of the - /// raw vector while this vector is in use. - #[inline] - pub unsafe fn as_raw(&self) -> &Id::RawVec<'gc, T> { - &*self.raw.get() - } - /// Get a mutable reference to the underlying raw vector, - /// bypassing restrictions on unique ownership. - /// - /// ## Safety - /// It is undefined behavior to change the length of the - /// raw vector while this vector is in use. - /// - /// Although calling this method requires unique ownership - /// of this vector, the raw vector is `Copy` and could - /// be duplicated. - #[inline] - pub unsafe fn as_mut_raw(&mut self) -> &mut Id::RawVec<'gc, T> { - self.raw.get_mut() - } - /// Iterate over the vector's contents. - /// - /// ## Safety - /// This is safe for the same reason [GcVec::as_mut_slice] is. - #[inline] - pub fn iter(&self) -> core::slice::Iter<'_, T> { - self.as_slice().iter() - } - - /// Mutably iterate over the vector's contents. - /// - /// ## Safety - /// This is safe for the same reason [GcVec::as_mut_slice] is. - #[inline] - pub fn iter_mut(&mut self) -> core::slice::IterMut<'_, T> { - self.as_mut_slice().iter_mut() - } - /// Creates a draining iterator that removes the specified range in the vector - /// and yields the removed items. - /// - /// See [Vec::drain] for more details on this operation. - /// - /// Whether or not leaking the iterator causes the underlying elements to be - /// leaked (if it does "leak amplification") depends on the implementation. - /// - /// Some implementations may need to allocate intermediate memory, - /// but all should be able to complete in at most `O(n)` time. - /// In other words, this should always avoid the quadratic behavior - /// of repeated `remove` calls. - pub fn drain(&mut self, range: impl RangeBounds) -> Drain<'_, 'gc, T, Id> { - /* - * See `Vec::drain` - */ - let old_len = self.len(); - let range = core::slice::range(range, ..old_len); - /* - * Preemptively set length to `range.start` in case the resulting `Drain` - * is leaked. - */ - unsafe { - self.set_len(range.start); - let r = core::slice::from_raw_parts(self.as_ptr().add(range.start), range.len()); - Drain { - tail_start: range.end, - tail_len: old_len - range.end, - iter: r.iter(), - vec: NonNull::from(self), - } - } - } -} -impl<'gc, T: GcSafe<'gc, Id>, I, Id: CollectorId> Index for GcVec<'gc, T, Id> -where - I: SliceIndex<[T]>, -{ - type Output = I::Output; - #[inline] - fn index(&self, idx: I) -> &I::Output { - &self.as_slice()[idx] - } -} -impl<'gc, T: GcSafe<'gc, Id>, I, Id: CollectorId> IndexMut for GcVec<'gc, T, Id> -where - I: SliceIndex<[T]>, -{ - #[inline] - fn index_mut(&mut self, index: I) -> &mut Self::Output { - &mut self.as_mut_slice()[index] - } -} -/// Because `GcVec` is uniquely owned (`!Copy`), -/// it can safely dereference to a slice -/// without risk of another reference mutating it. -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Deref for GcVec<'gc, T, Id> { - type Target = [T]; - #[inline] - fn deref(&self) -> &[T] { - self.as_slice() - } -} -/// Because `GcVec` is uniquely owned (`!Copy`), -/// it can safely de-reference to a mutable slice -/// without risk of another reference mutating its contents -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> DerefMut for GcVec<'gc, T, Id> { - #[inline] - fn deref_mut(&mut self) -> &mut [T] { - self.as_mut_slice() - } -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Debug for GcVec<'gc, T, Id> -where - T: Debug, -{ - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_list().entries(self.iter()).finish() - } -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> AsRef<[T]> for GcVec<'gc, T, Id> { - #[inline] - fn as_ref(&self) -> &[T] { - self.as_slice() - } -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> AsMut<[T]> for GcVec<'gc, T, Id> { - #[inline] - fn as_mut(&mut self) -> &mut [T] { - self.as_mut_slice() - } -} -#[inherent] -unsafe impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> IGcVec<'gc, T> for GcVec<'gc, T, Id> { - type Id = Id; - - #[inline] - pub fn with_capacity_in(capacity: usize, ctx: &'gc Id::Context) -> Self { - unsafe { Self::from_raw(Id::RawVec::<'gc, T>::with_capacity_in(capacity, ctx)) } - } - - #[inline] - pub fn len(&self) -> usize { - unsafe { self.as_raw() }.len() - } - - #[inline] - pub unsafe fn set_len(&mut self, len: usize) { - self.as_mut_raw().set_len(len) - } - - #[inline] - pub fn capacity(&self) -> usize { - unsafe { self.as_raw().capacity() } - } - - #[inline] - pub fn reserve_in_place(&mut self, additional: usize) -> Result<(), ReallocFailedError> { - unsafe { self.as_mut_raw().reserve_in_place(additional) } - } - - #[inline] - pub unsafe fn as_ptr(&self) -> *const T { - self.as_raw().as_ptr() - } - - #[inline] - pub fn context(&self) -> &'gc Id::Context { - unsafe { self.as_raw().context() } - } - - // Default methods: - pub fn replace(&mut self, index: usize, val: T) -> T; - pub fn set(&mut self, index: usize, val: T); - pub fn extend_from_slice(&mut self, src: &[T]) - where - T: Copy; - pub fn push(&mut self, val: T); - pub fn pop(&mut self) -> Option; - pub fn swap_remove(&mut self, index: usize) -> T; - pub fn reserve(&mut self, additional: usize); - pub fn is_empty(&self) -> bool; - pub fn new_in(ctx: &'gc Id::Context) -> Self; - pub fn copy_from_slice(src: &[T], ctx: &'gc Id::Context) -> Self - where - T: Copy; - #[cfg(any(feature = "alloc", feature = "std"))] - pub fn from_vec(src: Vec, ctx: &'gc Id::Context) -> Self; - - /* - * Intentionally hidden: - * 1. as_slice_unchecked (just use as_slice) - * 2. get (just use slice::get) - * 3. as_mut_ptr (just use slice::as_mut_ptr) - */ -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Extend for GcVec<'gc, T, Id> { - #[inline] - fn extend>(&mut self, iter: I) { - let iter = iter.into_iter(); - self.reserve(iter.size_hint().1.unwrap_or(0)); - for val in iter { - self.push(val); - } - } -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> IntoIterator for GcVec<'gc, T, Id> { - type Item = T; - - type IntoIter = IntoIter<'gc, T, Id>; - - #[inline] - fn into_iter(mut self) -> Self::IntoIter { - let len = self.len(); - unsafe { - let start = self.as_ptr(); - let end = start.add(len); - self.set_len(0); - IntoIter { - start, - end, - marker: PhantomData, - } - } - } -} - -/// The garbage collected analogue of [`std::vec::Drain`] -pub struct Drain<'a, 'gc, T: GcSafe<'gc, Id>, Id: CollectorId> { - /// Index of tail to preserve - tail_start: usize, - /// The length of tail to preserve - tail_len: usize, - iter: core::slice::Iter<'a, T>, - vec: NonNull>, -} -impl<'a, 'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Drain<'a, 'gc, T, Id> { - unsafe fn cleanup(&mut self) { - if self.tail_len == 0 { - return; - } - /* - * Copy `tail` back to vec. - */ - let v = self.vec.as_mut(); - let old_len = v.len(); - debug_assert!(old_len <= self.tail_start); - if old_len != self.tail_start { - v.as_ptr() - .add(self.tail_start) - .copy_to(v.as_mut_ptr().add(old_len), self.tail_len); - } - v.set_len(old_len + self.tail_len); - } -} -impl<'a, 'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Iterator for Drain<'a, 'gc, T, Id> { - type Item = T; - #[inline] - fn next(&mut self) -> Option { - self.iter.next().map(|e| unsafe { core::ptr::read(e) }) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} -impl<'a, 'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Drop for Drain<'a, 'gc, T, Id> { - fn drop(&mut self) { - while let Some(val) = self.iter.next() { - let _guard = scopeguard::guard(&mut *self, |s| unsafe { s.cleanup() }); - unsafe { - core::ptr::drop_in_place(val as *const T as *mut T); - } - core::mem::forget(_guard); - } - unsafe { self.cleanup() }; - } -} -impl<'a, 'gc, T: GcSafe<'gc, Id>, Id: CollectorId> DoubleEndedIterator for Drain<'a, 'gc, T, Id> { - #[inline] - fn next_back(&mut self) -> Option { - self.iter.next_back().map(|e| unsafe { core::ptr::read(e) }) - } -} -impl<'a, 'gc, T: GcSafe<'gc, Id>, Id: CollectorId> core::iter::ExactSizeIterator - for Drain<'a, 'gc, T, Id> -{ -} -impl<'a, 'gc, T: GcSafe<'gc, Id>, Id: CollectorId> core::iter::FusedIterator - for Drain<'a, 'gc, T, Id> -{ -} - -/// The [GcVec] analogue of [std::vec::IntoIter] -pub struct IntoIter<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> { - start: *const T, - end: *const T, - marker: PhantomData>, -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Iterator for IntoIter<'gc, T, Id> { - type Item = T; - #[inline] - fn next(&mut self) -> Option { - if self.start < self.end { - let val = self.start; - unsafe { - self.start = self.start.add(1); - Some(val.read()) - } - } else { - None - } - } - #[inline] - fn size_hint(&self) -> (usize, Option) { - let len = unsafe { self.end.offset_from(self.start) as usize }; - (len, Some(len)) - } -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> DoubleEndedIterator for IntoIter<'gc, T, Id> { - #[inline] - fn next_back(&mut self) -> Option { - if self.end > self.start { - unsafe { - self.end = self.end.sub(1); - Some(self.end.read()) - } - } else { - None - } - } -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> core::iter::ExactSizeIterator - for IntoIter<'gc, T, Id> -{ -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> core::iter::FusedIterator for IntoIter<'gc, T, Id> {} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Drop for IntoIter<'gc, T, Id> { - fn drop(&mut self) { - for val in self.by_ref() { - drop(val); - } - } -} - -impl<'gc, T: GcSafe<'gc, Id> + Clone, Id: CollectorId> Clone for GcVec<'gc, T, Id> { - #[inline] - fn clone(&self) -> Self { - let mut res = Self::with_capacity_in(self.len(), self.context()); - res.extend(self.iter().cloned()); - res - } -} -/// Because it is implicitly associated with a [GcContext](`crate::GcContext`) (which is thread-local), -/// this must be `!Send` -impl<'gc, T, Id: CollectorId> !Send for GcVec<'gc, T, Id> {} -unsafe_gc_impl!( - target => GcVec<'gc, T, Id>, - params => ['gc, T: GcSafe<'gc, Id>, Id: CollectorId], - bounds => { - TraceImmutable => { where T: Trace }, - Trace => { where T: GcSafe<'gc, Id> + Trace }, - GcRebrand => { where T: GcRebrand<'new_gc, Id>, T::Branded: Sized }, - }, - branded_type => GcVec<'new_gc, T::Branded, Id>, - null_trace => never, - NEEDS_TRACE => true, - NEEDS_DROP => ::NEEDS_DROP /* if our inner type needs a drop */, - trace_mut => |self, visitor| { - unsafe { - visitor.trace_vec(self.as_mut_raw()) - } - }, - trace_immutable => |self, visitor| { - unsafe { - visitor.trace_vec(&mut *self.raw.get()) - } - }, - collector_id => Id -); - -/// Indicates there is insufficient capacity for an operation on a [GcRawVec] -#[derive(Debug)] -pub struct InsufficientCapacityError; diff --git a/src/vec/cell.rs b/src/vec/cell.rs deleted file mode 100644 index 57f2c56..0000000 --- a/src/vec/cell.rs +++ /dev/null @@ -1,151 +0,0 @@ -//! The implementation of [GcVecCell] -#[cfg(all(not(feature = "std"), feature = "alloc"))] -use alloc::vec::Vec; -use core::cell::RefCell; - -use inherent::inherent; - -use crate::prelude::*; -use crate::vec::raw::{IGcVec, ReallocFailedError}; -use crate::SimpleAllocCollectorId; - -/// A garbage collected vector, -/// wrapped in a [RefCell] for interior mutability. -/// -/// Essentially a `Gc>>`. However, -/// this can't be done directly because `RefCell` normally requires -/// `T: NullTrace` (because of the possibility of write barriers). -#[derive(Trace)] -#[zerogc(collector_ids(Id), copy)] -pub struct GcVecCell<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> { - inner: Gc<'gc, RefCell>, Id>, -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> GcVecCell<'gc, T, Id> { - /// Immutably borrow the wrapped [GcVec]. - /// - /// The returned borrow is dynamically tracked, - /// and guarded by the returned - /// [core::cell::Ref] object. - /// - /// All immutable accesses through the [IGcVec] interface - /// implicitly call this method (and thus carry the same risk of panics). - /// - /// ## Panics - /// Panics if this vector has an outstanding mutable borrow. - #[inline] - pub fn borrow(&self) -> core::cell::Ref<'_, GcVec<'gc, T, Id>> { - self.inner.borrow() - } - /// Mutably (and exclusively) borrow the wrapped [GcVec]. - /// - /// The returned borrow is dynamically tracked, - /// and guarded by the returned - /// [core::cell::RefMut] object. - /// - /// All mutable accesses through the [IGcVec] interface - /// implicitly call this method (and thus carry the same risk of panics). - /// - /// ## Panics - /// Panics if this vector has any other outstanding borrows. - #[inline] - pub fn borrow_mut(&self) -> core::cell::RefMut<'_, GcVec<'gc, T, Id>> { - self.inner.borrow_mut() - } - /// Immutably borrow a slice of this vector's contents. - /// - /// Implicitly calls [GcVecCell::borrow], - /// and caries the same risk of panics. - #[inline] - pub fn borrow_slice(&self) -> core::cell::Ref<'_, [T]> { - core::cell::Ref::map(self.borrow(), |v| v.as_slice()) - } - /// Mutably borrow a slice of this vector's contents. - /// - /// Implicitly calls [GcVecCell::borrow_mut], - /// and caries the same risk of panics. - #[inline] - pub fn borrow_mut_slice(&self) -> core::cell::RefMut<'_, [T]> { - core::cell::RefMut::map(self.borrow_mut(), |v| v.as_mut_slice()) - } -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Copy for GcVecCell<'gc, T, Id> {} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Clone for GcVecCell<'gc, T, Id> { - #[inline] - fn clone(&self) -> Self { - *self - } -} -/// Because vectors are associated with a [GcContext](`crate::GcContext`), -/// they contain thread local data (and thus must be `!Send` -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> !Send for GcVecCell<'gc, T, Id> {} -#[inherent] -unsafe impl<'gc, T: GcSafe<'gc, Id>, Id: SimpleAllocCollectorId> IGcVec<'gc, T> - for GcVecCell<'gc, T, Id> -{ - type Id = Id; - - #[inline] - pub fn with_capacity_in(capacity: usize, ctx: &'gc ::Context) -> Self { - GcVecCell { - inner: ctx.alloc(RefCell::new(GcVec::with_capacity_in(capacity, ctx))), - } - } - - #[inline] - pub fn len(&self) -> usize { - self.inner.borrow().len() - } - - #[inline] - pub unsafe fn set_len(&mut self, len: usize) { - self.inner.borrow_mut().set_len(len); - } - - #[inline] - pub fn capacity(&self) -> usize { - self.inner.borrow().capacity() - } - - #[inline] - pub fn reserve_in_place(&mut self, additional: usize) -> Result<(), ReallocFailedError> { - self.inner.borrow_mut().reserve_in_place(additional) - } - - #[inline] - pub unsafe fn as_ptr(&self) -> *const T { - self.inner.borrow().as_ptr() - } - - #[inline] - pub fn context(&self) -> &'gc ::Context { - self.inner.borrow().context() - } - - // Default methods: - pub unsafe fn as_mut_ptr(&mut self) -> *mut T; - pub fn replace(&mut self, index: usize, val: T) -> T; - pub fn set(&mut self, index: usize, val: T); - pub fn extend_from_slice(&mut self, src: &[T]) - where - T: Copy; - pub fn push(&mut self, val: T); - pub fn pop(&mut self) -> Option; - pub fn swap_remove(&mut self, index: usize) -> T; - pub fn reserve(&mut self, additional: usize); - pub fn is_empty(&self) -> bool; - pub fn new_in(ctx: &'gc ::Context) -> Self; - pub fn copy_from_slice(src: &[T], ctx: &'gc ::Context) -> Self - where - T: Copy; - #[cfg(feature = "alloc")] - pub fn from_vec(src: Vec, ctx: &'gc ::Context) -> Self; - pub fn get(&mut self, index: usize) -> Option - where - T: Copy; - pub unsafe fn as_slice_unchecked(&self) -> &[T]; -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Extend for GcVecCell<'gc, T, Id> { - fn extend>(&mut self, iter: A) { - self.inner.borrow_mut().extend(iter); - } -} diff --git a/src/vec/raw.rs b/src/vec/raw.rs deleted file mode 100644 index 8e7ff7c..0000000 --- a/src/vec/raw.rs +++ /dev/null @@ -1,626 +0,0 @@ -//! The underlying representation of a [GcVec](`crate::vec::GcVec`) - -#[cfg(all(not(feature = "std"), feature = "alloc"))] -use alloc::vec::Vec; -use core::marker::PhantomData; - -use crate::{CollectorId, GcRebrand, GcSafe}; - -use zerogc_derive::unsafe_gc_impl; - -/// A marker error to indicate in-place reallocation failed -#[derive(Debug)] -pub enum ReallocFailedError { - /// Indicates that the operation is unsupported - Unsupported, - /// Indicates that the vector is too large to reallocate in-place - SizeUnsupported, - /// Indicates that the garbage collector is out of memory - OutOfMemory, -} - -/// Basic methods shared across all garbage collected vectors. -/// -/// All garbage collectors are implicitly -/// associated with their owning [GcContext](`crate::GcContext`). -/// -/// This can be changed by calling `detatch`, -/// although this is currently unimplemented. -/// -/// ## Safety -/// Undefined behavior if the methods -/// violate their contracts. -/// -/// All of the methods in this trait -/// can be trusted to be implemented correctly. -pub unsafe trait IGcVec<'gc, T: GcSafe<'gc, Self::Id>>: Sized + Extend { - /// The id of the collector - type Id: CollectorId; - /// Copy the specified elements into a new vector, - /// allocating it in the specified context - /// - /// This consumes ownership of the original vector. - #[inline] - #[cfg(any(feature = "alloc", feature = "std"))] - fn from_vec(mut src: Vec, ctx: &'gc ::Context) -> Self { - let len = src.len(); - let mut res = Self::with_capacity_in(len, ctx); - unsafe { - res.as_mut_ptr().copy_from_nonoverlapping(src.as_ptr(), len); - src.set_len(0); - res.set_len(len); - res - } - } - /// Copy the specified elements into a new vector, - /// allocating it in the specified context - #[inline] - fn copy_from_slice(src: &[T], ctx: &'gc ::Context) -> Self - where - T: Copy, - { - let len = src.len(); - let mut res = Self::with_capacity_in(len, ctx); - unsafe { - res.as_mut_ptr().copy_from_nonoverlapping(src.as_ptr(), len); - res.set_len(len); - res - } - } - /// Allocate a new vector inside the specified context - #[inline] - fn new_in(ctx: &'gc ::Context) -> Self { - Self::with_capacity_in(0, ctx) - } - /// Allocate a new vector with the specified capacity, - /// using the specified context. - fn with_capacity_in(capacity: usize, ctx: &'gc ::Context) -> Self; - /// The length of the vector. - /// - /// This is the number of elements that are actually - /// initialized, as opposed to `capacity`, which is the number - /// of elements that are available in total. - fn len(&self) -> usize; - /// Check if this vector is empty - #[inline] - fn is_empty(&self) -> bool { - self.len() == 0 - } - /// Set the length of the vector. - /// - /// ## Safety - /// All the same restrictions apply as [Vec::set_len]. - /// - /// By the time of the next garbage collection, - /// The underlying memory must be initialized up to the specified length, - /// otherwise the vector's memory will be traced incorrectly. - /// - /// Undefined behavior if length is greater than capacity. - unsafe fn set_len(&mut self, len: usize); - /// The total amount of space that is available - /// without needing reallocation. - fn capacity(&self) -> usize; - /// Attempt to reallocate the vector in-place, - /// without moving the underlying pointer. - fn reserve_in_place(&mut self, additional: usize) -> Result<(), ReallocFailedError>; - /// Reserves capacity for at least `additional`. - /// - /// If this type is a [GcVecCell](`crate::vec::cell::GcVecCell`) and there are outstanding borrows references, - /// then this will panic as if calling [`RefCell::borrow_mut`](`core::cell::RefCell::borrow_mut`). - /// - /// ## Safety - /// This method is safe. - /// - /// If this method finishes without panicking, - /// it can be assumed that `new_capacity - old_capcity >= additional` - #[inline] - fn reserve(&mut self, additional: usize) { - // See https://github.com/rust-lang/rust/blob/c7dbe7a8301/library/alloc/src/raw_vec.rs#L402 - if additional > self.capacity().wrapping_sub(self.len()) { - grow_vec(self, additional); - } - } - - /// Push the specified element onto the end - /// of this vector. - /// - /// If this type is a [GcVecCell](`crate::vec::GcVecCell`) and there are outstanding borrows references, - /// then this will panic as if calling [`RefCell::borrow_mut`](`core::cell::RefCell::borrow_mut`). - /// - /// ## Safety - /// This method is safe. - #[inline] - fn push(&mut self, val: T) { - let old_len = self.len(); - // TODO: Write barriers - self.reserve(1); - unsafe { - // NOTE: This implicitly calls `borrow_mut` if we are a `GcVecCell` - self.as_mut_ptr().add(old_len).write(val); - self.set_len(old_len + 1); - } - } - /// Pop an element of the end of the vector, - /// returning `None` if empty. - /// - /// This is analogous to [`Vec::pop`] - /// - /// If this type is a [GcVecCell](`crate::vec::GcVecCell`) and there are outstanding borrowed references, - /// this will panic as if calling [`RefCell::borrow_mut`](`core::cell::RefCell::borrow_mut`). - /// - /// ## Safety - /// This method is safe. - #[inline] - fn pop(&mut self) -> Option { - let old_len = self.len(); - if old_len > 0 { - // TODO: Write barriers? - // NOTE: This implicitly calls `borrow_mut` if we are a `GcVecCell` - unsafe { - self.set_len(old_len - 1); - Some(self.as_ptr().add(old_len - 1).read()) - } - } else { - None - } - } - /// Removes an element from the vector and returns it. - /// - /// The removed element is replaced by the last element in the vector. - /// - /// This doesn't preserve ordering, but it is `O(1)` - #[inline] - fn swap_remove(&mut self, index: usize) -> T { - let len = self.len(); - assert!(index < len); - unsafe { - let last = core::ptr::read(self.as_ptr().add(len - 1)); - let hole = self.as_mut_ptr().add(index); - self.set_len(len - 1); - core::ptr::replace(hole, last) - } - } - /// Extend the vector with elements copied from the specified slice - #[inline] - fn extend_from_slice(&mut self, src: &[T]) - where - T: Copy, - { - let old_len = self.len(); - self.reserve(src.len()); - // TODO: Write barriers? - unsafe { - self.as_mut_ptr() - .add(old_len) - .copy_from_nonoverlapping(src.as_ptr(), src.len()); - self.set_len(old_len + src.len()); - } - } - /// Get the item at the specified index, - /// or `None` if it is out of bounds. - /// - /// This returns a copy of the value. - /// As such it requires `T: Copy` - #[inline] - fn get(&self, idx: usize) -> Option - where - T: Copy, - { - unsafe { self.as_slice_unchecked().get(idx).copied() } - } - /// Set the item at the specified index - /// - /// Panics if the index is out of bounds. - #[inline] - fn set(&mut self, index: usize, val: T) { - assert!(index < self.len()); - unsafe { - *self.as_mut_ptr().add(index) = val; - } - } - /// Replace the item at the specified index, - /// returning the old value. - /// - /// For a traditional `Vec`, - /// the equivalent would be `mem::replace(&mut v[index], new_value)`. - #[inline] - fn replace(&mut self, index: usize, val: T) -> T { - assert!(index < self.len()); - unsafe { core::ptr::replace(self.as_mut_ptr(), val) } - } - /// Get a pointer to this vector's - /// underling data. - /// - /// This is unsafe for the same - /// - /// ## Safety - /// Undefined behavior if the pointer is used - /// are used after the length changes. - unsafe fn as_ptr(&self) -> *const T; - /// Get a mutable pointer to the vector's underlying data. - /// - /// This is unsafe for the same reason [IGcVec::as_ptr] - /// and - /// - /// ## Safety - /// Undefined behavior if the pointer is used - /// after the length changes. - /// - /// Undefined behavior if the elements - /// are mutated while there are multiple outstanding references. - #[inline] - unsafe fn as_mut_ptr(&mut self) -> *mut T { - self.as_ptr() as *mut T - } - /// Get a slice of this vector's elements. - /// - /// ## Safety - /// If this type has shared references (multiple owners), - /// then it is undefined behavior to mutate the length - /// while the slice is still borrowed. - /// - /// In other words, the following is invalid: - /// ```no_run - /// # use zerogc::epsilon::{EpsilonCollectorId, EpsilonContext}; - /// # use zerogc::vec::{GcVecCell}; - /// # fn do_ub(context: &EpsilonContext) { - /// let mut v = GcVecCell::::new_in(context); - /// v.push(23); - /// let mut copy = v; - /// let slice = unsafe { v.as_slice_unchecked() }; - /// copy.push(15); - /// slice[0]; // UB - /// # } - /// ``` - #[inline] - unsafe fn as_slice_unchecked(&self) -> &[T] { - core::slice::from_raw_parts(self.as_ptr(), self.len()) - } - /// Get the [GcContext](`crate::GcContext`) that this vector is associated with. - /// - /// Because each vector is implicitly associated with a [GcContext](`crate::GcContext`) (which is thread-local), - /// vectors are `!Send` unless you call `detatch`. - fn context(&self) -> &'gc ::Context; -} - -/// Slow-case for `reserve`, when reallocation is actually needed -#[cold] -fn grow_vec<'gc, T, V>(vec: &mut V, amount: usize) -where - V: IGcVec<'gc, T>, - T: GcSafe<'gc, V::Id>, -{ - match vec.reserve_in_place(amount) { - Ok(()) => { - return; // success - } - Err(ReallocFailedError::OutOfMemory) => panic!("Out of memory"), - Err(ReallocFailedError::Unsupported) | Err(ReallocFailedError::SizeUnsupported) => { - // fallthrough to realloc - } - } - let requested_capacity = vec.len().checked_add(amount).unwrap(); - let new_capacity = vec - .capacity() - .checked_mul(2) - .unwrap() - .max(requested_capacity); - // Just allocate a new one, copying from the old - let mut new_mem = V::with_capacity_in(new_capacity, vec.context()); - // TODO: Write barriers - unsafe { - new_mem - .as_mut_ptr() - .copy_from_nonoverlapping(vec.as_ptr(), vec.len()); - new_mem.set_len(vec.len()); - let mut old_mem = core::mem::replace(vec, new_mem); - old_mem.set_len(0); // We don't want to drop the old elements - assert!(vec.capacity() >= requested_capacity); - } -} - -/// A garbage collected vector -/// with unchecked interior mutability. -/// -/// See the [module docs](`zerogc::vec`) for -/// more details on the distinctions between vector types. -/// -/// ## Interior Mutability -/// TLDR: This is the [Cell](core::cell::Cell) -/// equivalent of [GcVecCell](zerogc::vec::GcVecCell). -/// -/// It only gives/accepts owned values `T`, and cannot safely -/// give out references like `&T` or `&[T]`. -/// -/// Interior mutability is necessary because garbage collection -/// allows shared references, and the length can be mutated -/// by one reference while another reference is in use. -/// -/// Unlike `UnsafeCell`, not *all* accesses to this are `unsafe`. -/// Calls to `get`, `set`, and `push` are perfectly safe because they take/return `T`, not `&T`. -/// Only calls to `get_slice_unchecked` are `unsafe`. -/// -/// NOTE: This is completely different from the distinction between -/// [Vec] and [RawVec](https://github.com/rust-lang/rust/blob/master/library/alloc/src/raw_vec.rs) -/// in the standard library. -/// In particular, this still contains a `len`. -/// -/// ## Safety -/// This must be implemented consistent with the API of [GcRawVec]. -/// -/// It is undefined behavior to take a slice of the elements -/// while the length is being mutated. -/// -/// This type is `!Send`, -/// because it is implicitly associated -/// with the [GcContext](`crate::GcContext`) it was allocated in. -/// -/// Generally speaking, -/// [GcVec](`crate::vec::GcVec`) and [GcVecCell](`crate::vec::GcVecCell`) should be preferred. -pub unsafe trait GcRawVec<'gc, T: GcSafe<'gc, Self::Id>>: Copy + IGcVec<'gc, T> { - /// Steal ownership of this vector, converting it into a [GcArray]. - /// - /// Assumes that there are no other active references. - /// - /// This avoids copying memory if at all possible - /// (although it might be necessary, depending on the implementation). - /// - /// ## Safety - /// Undefined behavior if there are any other references - /// to the vector or any uses after it has been stolen. - /// - /// This may or may not be debug-checked, - /// depending on the implementation. - /// - /// This logically steals ownership of this vector and invalidates all other references to it, - /// although the safety of it cannot be checked statically. - unsafe fn steal_as_array_unchecked(self) -> crate::array::GcArray<'gc, T, Self::Id>; - /// Iterate over the elements of the vectors - /// - /// Panics if the length changes. - /// - /// Because this copies the elements, - /// it requires `T: Copy` - #[inline] - fn iter(&self) -> RawVecIter<'gc, T, Self> - where - T: Copy, - { - RawVecIter { - target: *self, - marker: PhantomData, - index: 0, - end: self.len(), - original_len: self.len(), - } - } -} - -/// An iterator over a [GcRawVec] -/// -/// Because the length may change, -/// this iterates over the indices -/// and requires that `T: Copy`. -/// -/// It is equivalent to the following code: -/// ```ignore -/// for idx in 0..vec.len() { -/// yield vec.get(idx); -/// } -/// ``` -/// -/// It will panic if the length changes, -/// either increasing or decreasing. -pub struct RawVecIter<'gc, T: GcSafe<'gc, V::Id> + Copy, V: GcRawVec<'gc, T>> { - target: V, - marker: PhantomData>, - index: usize, - /// The end of the iterator - /// - /// Typically this will be `original_len`, - /// although this can change if using `DoubleEndedIterator::next_back` - end: usize, - original_len: usize, -} -impl<'gc, T: GcSafe<'gc, V::Id> + Copy, V: GcRawVec<'gc, T>> RawVecIter<'gc, T, V> { - /// Return the original length of the vector - /// when iteration started. - /// - /// The iterator should panic if this changes. - #[inline] - pub fn original_len(&self) -> usize { - self.original_len - } - /// Check that the vector's length is unmodified. - #[inline] - #[track_caller] - fn check_unmodified_length(&self) { - assert_eq!( - self.target.len(), - self.original_len, - "Vector length mutated during iteration" - ); - } -} -impl<'gc, T: GcSafe<'gc, V::Id> + Copy, V: GcRawVec<'gc, T>> Iterator for RawVecIter<'gc, T, V> { - type Item = T; - #[inline] - #[track_caller] - fn next(&mut self) -> Option { - self.check_unmodified_length(); - if self.index < self.end { - self.index += 1; - Some(unsafe { self.target.get(self.index - 1).unwrap_unchecked() }) - } else { - None - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let len = self.end - self.index; - (len, Some(len)) - } - - #[inline] - fn count(self) -> usize - where - Self: Sized, - { - self.len() - } -} -impl<'gc, T: GcSafe<'gc, V::Id> + Copy, V: GcRawVec<'gc, T>> DoubleEndedIterator - for RawVecIter<'gc, T, V> -{ - #[inline] - fn next_back(&mut self) -> Option { - self.check_unmodified_length(); - if self.end > self.index { - self.end -= 1; - Some(unsafe { self.target.get(self.end).unwrap_unchecked() }) - } else { - None - } - } -} -impl<'gc, T: GcSafe<'gc, V::Id> + Copy, V: GcRawVec<'gc, T>> ExactSizeIterator - for RawVecIter<'gc, T, V> -{ -} -impl<'gc, T: GcSafe<'gc, V::Id> + Copy, V: GcRawVec<'gc, T>> core::iter::FusedIterator - for RawVecIter<'gc, T, V> -{ -} -/// Dummy implementation of [GcRawVec] for collectors -/// which do not support [GcVec](`crate::vec::GcVec`) -pub struct Unsupported<'gc, T, Id: CollectorId> { - /// The marker `PhantomData` needed to construct this type - pub marker: PhantomData>, - /// indicates this type should never exist at runtime - // TODO: Replace with `!` once stabilized - pub never: core::convert::Infallible, -} -unsafe_gc_impl! { - target => Unsupported<'gc, T, Id>, - params => ['gc, T: GcSafe<'gc, Id>, Id: CollectorId], - bounds => { - GcSafe => always, - GcRebrand => { where T: GcRebrand<'new_gc, Id>, T::Branded: Sized }, - }, - /* - * NOTE: We are *not* NullTrace, - * because we don't want users to rely - * on that and then have trouble switching away later - */ - null_trace => never, - branded_type => Unsupported<'new_gc, T::Branded, Id>, - NEEDS_TRACE => false, - NEEDS_DROP => false, - collector_id => Id, - trace_template => |self, visitor| { /* nop */ Ok(()) } -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Extend for Unsupported<'gc, T, Id> { - fn extend>(&mut self, _iter: I) { - unimplemented!() - } -} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Copy for Unsupported<'gc, T, Id> {} -impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> Clone for Unsupported<'gc, T, Id> { - fn clone(&self) -> Self { - *self - } -} - -unsafe impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> GcRawVec<'gc, T> for Unsupported<'gc, T, Id> { - unsafe fn steal_as_array_unchecked(self) -> crate::GcArray<'gc, T, Id> { - unimplemented!() - } -} -#[allow(unused_variables)] -unsafe impl<'gc, T: GcSafe<'gc, Id>, Id: CollectorId> IGcVec<'gc, T> for Unsupported<'gc, T, Id> { - type Id = Id; - - fn len(&self) -> usize { - unimplemented!() - } - - unsafe fn set_len(&mut self, _len: usize) { - unimplemented!() - } - - fn capacity(&self) -> usize { - unimplemented!() - } - - fn with_capacity_in(capacity: usize, ctx: &'gc ::Context) -> Self { - unimplemented!() - } - - fn reserve_in_place(&mut self, additional: usize) -> Result<(), ReallocFailedError> { - unimplemented!() - } - - fn reserve(&mut self, additional: usize) { - unimplemented!() - } - - unsafe fn as_ptr(&self) -> *const T { - unimplemented!() - } - - fn context(&self) -> &'gc Id::Context { - unimplemented!() - } -} - -/// Drains a [`GcRawVec`] by repeatedly calling `IGcVec::swap_remove` -/// -/// Panics if the length is modified while iteration is in progress. -pub struct DrainRaw<'a, 'gc, T: GcSafe<'gc, V::Id>, V: GcRawVec<'gc, T>> { - start: usize, - expected_len: usize, - target: &'a mut V, - marker: PhantomData>, -} -impl<'a, 'gc, T: GcSafe<'gc, V::Id>, V: GcRawVec<'gc, T>> Iterator for DrainRaw<'a, 'gc, T, V> { - type Item = T; - #[inline] - fn next(&mut self) -> Option { - assert_eq!( - self.expected_len, - self.target.len(), - "Length changed while iteration was in progress" - ); - if self.start < self.expected_len { - self.expected_len -= 1; - Some(self.target.swap_remove(self.start)) - } else { - None - } - } - #[inline] - fn size_hint(&self) -> (usize, Option) { - let len = self.expected_len - self.start; - (len, Some(len)) - } -} -impl<'a, 'gc, T: GcSafe<'gc, V::Id>, V: GcRawVec<'gc, T>> DoubleEndedIterator - for DrainRaw<'a, 'gc, T, V> -{ - fn next_back(&mut self) -> Option { - if self.start < self.expected_len { - self.expected_len -= 1; - Some(self.target.pop().unwrap()) - } else { - None - } - } -} -impl<'a, 'gc, T: GcSafe<'gc, V::Id>, V: GcRawVec<'gc, T>> Drop for DrainRaw<'a, 'gc, T, V> { - fn drop(&mut self) { - for val in self.by_ref() { - drop(val); - } - } -} From 145fab5e9fe5e2c01201942622e317d6e9ae0a75 Mon Sep 17 00:00:00 2001 From: Techcable Date: Wed, 8 May 2024 14:32:40 -0700 Subject: [PATCH 23/35] Remove safepoint macros Will not be used with initial version of v0.3 API --- src/lib.rs | 160 ------------------------------------------------- src/prelude.rs | 2 - 2 files changed, 162 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 8060c8d..ef5d792 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,166 +68,6 @@ mod macros; pub mod cell; pub mod prelude; -/// Invoke the closure with a temporary [GcContext], -/// then perform a safepoint afterwards. -/// -/// Normally returns a tuple `($updated_root, $closure_result)`. -/// -/// If a value is provided it is considered as a root of garbage collection -/// both for the safepoint and the duration of the entire context. -/// -/// # Safety -/// This macro is completely safe, although it expands to unsafe code internally. -#[macro_export(local_inner_macros)] -macro_rules! safepoint_recurse { - ($context:ident, |$sub_context:ident| $closure:expr) => {{ - let ((), result) = safepoint_recurse!($context, (), |$sub_context, new_root| { - let () = new_root; - $closure - }); - result - }}; - ($context:ident, $root:expr, |$sub_context:ident, $new_root:ident| $closure:expr) => {{ - let mut root = $root; - let result = unsafe { - __recurse_context!($context, &mut root, |$sub_context, $new_root| { $closure }) - }; - /* - * NOTE: We're assuming result is unmanaged here - * The borrow checker will verify this is true (by marking this as a mutation). - * If you need a manged result, use the @managed_result variant - */ - let updated_root = safepoint!($context, $root); - (updated_root, result) - }}; - ($context:ident, $root:expr, @managed_result, |$sub_context:ident, $new_root:ident| $closure:expr) => {{ - use $crate::GcContext; - let mut root = $root; - let erased_result = unsafe { - __recurse_context!($context, &mut root, |$sub_context, $new_root| { - let result = $closure; - $sub_context.rebrand_static(result) - }) - }; - /* - * Rebrand back to the current collector lifetime - * It could have possibly been allocated from the sub-context inside the closure, - * but as long as it was valid at the end of the closure it's valid now. - * We trust that GcContext::recurse_context - * did not perform a collection after calling the closure. - */ - let result = unsafe { $context.rebrand_self(**erased_result) }; - safepoint!($context, (root, result)) - }}; -} - -/// Create a new sub-context for the duration of the closure -/// -/// The specified `root` object will be appended to the shadow-stack -/// and is guarenteed to live for the entire lifetime of the closure -/// (and the created sub-context). -/// -/// Unlike [safepoint_recurse!] this doesn't imply a safepoint anywhere. -/// -/// # Safety -/// This doesn't actually mutate the original collector. -/// It is possible user code could trigger a collection in the closure -/// without the borrow checker noticing invalid pointers elsewhere. -/// (See docs for [GcContext::recurse_context]) -/// -/// It is not publicly exposed for this reason -#[macro_export] -#[doc(hidden)] -macro_rules! __recurse_context { - ($context:ident, $root:expr, |$sub_context:ident, $new_root:ident| $closure:expr) => {{ - use $crate::GcContext; - // TODO: Panic safety - $context.recurse_context(&mut $root, |mut $sub_context, erased_root| { - /* - * NOTE: Guarenteed to live for the lifetime of the entire closure. - * However, it could be relocated if 'sub_collector' is collected - */ - let $new_root = $sub_context.rebrand_self(*erased_root); - $closure - }) - }}; -} - -/// Indicate it's safe to begin a garbage collection, -/// while keeping the specified root alive. -/// -/// All other garbage collected pointers that aren't reachable -/// from the root are invalidated. -/// They have a lifetime that references the [GcContext] -/// and the borrow checker considers the safepoint a 'mutation'. -/// -/// The root is exempted from the "mutation" and rebound to the new lifetime. -/// -/// ## Example -/// ``` -/// # use ::zerogc::safepoint; -/// # let mut context = zerogc::epsilon::EpsilonSystem::leak().new_context(); -/// # // TODO: Can we please get support for non-Sized types like `String`?!?!?! -/// let root = zerogc::epsilon::leaked(String::from("potato")); -/// let root = safepoint!(context, root); -/// assert_eq!(*root, "potato"); -/// ``` -/// -/// ## Safety -/// This macro is completely safe, although it expands to unsafe code internally. -#[macro_export] -macro_rules! safepoint { - ($context:ident, $value:expr) => { - unsafe { - use $crate::GcContext; - // TODO: What happens if we panic during a collection - /* - * Some collectors support multiple running instances - * with different ids, handing out different GC pointers. - * TODO: Should we be checking somehow that the ids match? - */ - let mut erased = $context.rebrand_static($value); - $context.basic_safepoint(&mut &mut erased); - $context.rebrand_self(erased) - } - }; -} - -/// Indicate its safe to begin a garbage collection (like [safepoint!]) -/// and then "freeze" the specified context. -/// -/// Until it's unfrozen, the context can't be used for allocation. -/// Its roots are marked invalid, since the collector could be relocating them. -/// However, the roots of any parent contexts are still considered valid. -/// -/// This allows other threads to perform collections *without blocking this thread*. -#[macro_export] -macro_rules! freeze_context { - ($context:ident) => { - unsafe { - use $crate::{FrozenContext, GcContext}; - let mut context = $context; - context.freeze(); - FrozenContext::new(context) - } - }; -} - -/// Unfreeze the context, allowing it to be used again -/// -/// Returns a [GcContext] struct. -#[macro_export] -macro_rules! unfreeze_context { - ($frozen:ident) => { - unsafe { - use $crate::{FrozenContext, GcContext}; - let mut context = FrozenContext::into_context($frozen); - context.unfreeze(); - context - } - }; -} - /// A garbage collector implementation. /// /// These implementations should be completely safe and zero-overhead. diff --git a/src/prelude.rs b/src/prelude.rs index 0453c71..1e27c61 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -5,8 +5,6 @@ //! This should really contain everything a garbage //! collected program needs to use the API. -// macros -pub use crate::{freeze_context, safepoint, safepoint_recurse, unfreeze_context}; // Basic collector types pub use crate::{Gc, GcContext, GcHandle, GcSimpleAlloc, GcSystem, GcVisitor, HandleCollectorId}; // Traits for user code to implement From 08146291ff68d01a99b70896d211a9b4aa0d57f7 Mon Sep 17 00:00:00 2001 From: Techcable Date: Wed, 8 May 2024 14:33:50 -0700 Subject: [PATCH 24/35] Remove GcContext trait Will be handled differently in v0.3 API --- src/lib.rs | 282 +------------------------------------------------ src/prelude.rs | 2 +- 2 files changed, 5 insertions(+), 279 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ef5d792..c9ec8e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -54,7 +54,6 @@ use core::cmp::Ordering; use core::fmt::{self, Debug, Display, Formatter}; use core::hash::{Hash, Hasher}; use core::marker::{PhantomData, Unsize}; -use core::mem::{self}; use core::ops::{CoerceUnsized, Deref, DerefMut}; use core::ptr::NonNull; @@ -68,268 +67,12 @@ mod macros; pub mod cell; pub mod prelude; -/// A garbage collector implementation. -/// -/// These implementations should be completely safe and zero-overhead. +/// A garbage collector implementation, +/// conforming to the zerogc API. pub unsafe trait GcSystem { /// The type of collector IDs given by this system type Id: CollectorId; - /// The type of contexts used in this sytem - type Context: GcContext; -} - -/// The context of garbage collection, -/// which can be frozen at a safepoint. -/// -/// This is essentially used to maintain a shadow-stack to a set of roots, -/// which are guarenteed not to be collected until a safepoint. -/// -/// This context doesn't necessarily support allocation (see [GcSimpleAlloc] for that). -pub unsafe trait GcContext: Sized { - /// The system used with this context - type System: GcSystem; - /// The type of ids used in the system - type Id: CollectorId; - - /// Potentially perform a garbage collection, freeing - /// all objects that aren't reachable from the specified root. - /// - /// This is a less safe version of [GcContext::basic_safepoint] - /// that doesn't explicitly mark this context as "mutated". - /// However, just like any other safepoint, - /// it still logically invalidates all unreachable objects - /// because the collector might free them. - /// - /// ## Safety - /// All objects that are potentially in-use must be reachable from the specified root. - /// Otherwise, they may be accidentally freed during garbage collection. - /// - /// It is the user's responsibility to check this, - /// since this doesn't statically invalidate (or mutate) - /// the outstanding references. - unsafe fn unchecked_safepoint(&self, root: &mut &mut T); - - /// Potentially perform a garbage collection, freeing - /// all objects that aren't reachable from the specified root. - /// - /// This is what the [safepoint!] macro expands to internally. - /// For example - /// ```no_run - /// # use zerogc::prelude::*; - /// # use zerogc_derive::Trace; - /// # use zerogc::epsilon::{Gc, EpsilonCollectorId, EpsilonContext as ExampleContext}; - /// # #[derive(Trace)] - /// # struct Obj { val: u32 } - /// fn example<'gc>(ctx: &'gc mut ExampleContext, root: Gc<'gc, Obj>) { - /// let temp = ctx.alloc(Obj { val: 12 }); // Lives until the call to 'basic_safepoint' - /// assert_eq!(temp.val, 12); // VALID - /// // This is what `let root = safepoint!(ctx, root)` would expand to: - /// let root = unsafe { - /// /* - /// * Change the lifetime of root from 'gc -> 'static - /// * This is necessary so the mutation of 'basic_safepoint' - /// * wont invalidate it. - /// */ - /// let mut preserved_root: Gc<'static, Obj> = ctx.rebrand_static(root); - /// /* - /// * This invalidates 'temp' (because it mutates the owning context) - /// * However, preserved_root is fine because it has been changed the 'static - /// * lifetime. - /// */ - /// ctx.basic_safepoint(&mut &mut preserved_root); - /// /* - /// * It would be unsafe for the user to access - /// * preserved_root, because its lifetime of 'static is too large. - /// * The call to 'rebrand_self' changes the lifetime of preserved_root - /// * back to the (new) lifetime of the context. - /// * - /// * The safe result is returned as the result of the block. - /// * The user has chosen to re-assign the result to the `root` variable - /// * via `let root = safepoint!(ctx, root)`. - /// */ - /// ctx.rebrand_self(preserved_root) - /// }; - /// // assert_eq!(temp.val, 12); // INVALID. `temp` was mutated and bound to the old lifetime - /// assert_eq!(root.val, 4); // VALID - The lifetime has been updated - /// } - /// ``` - /// - /// ## Safety - /// Any garbage collected objects not reachable from the roots - /// are invalidated after this method. - /// - /// The user must ensure that invalidated objects are not used - /// after the safepoint, - /// although the (logical) mutation of the context - /// should significantly assist with that. - #[inline] - unsafe fn basic_safepoint(&mut self, root: &mut &mut T) { - self.unchecked_safepoint(root) - } - - /// Inform the garbage collection system we are at a safepoint - /// and are ready for a potential garbage collection. - /// - /// Unlike a `basic_safepoint`, the collector continues to - /// stay at the safepoint instead of returning immediately. - /// The context can't be used for anything (including allocations), - /// until it is unfrozen. - /// - /// This allows other threads to perform collections while this - /// thread does other work (without using the GC). - /// - /// The current contexts roots are considered invalid - /// for the duration of the collection, - /// since the collector could potentially relocate them. - /// - /// Any parent contexts are fine and their roots will be - /// preserved by collections. - /// - /// ## Safety - /// Assumes this context is valid and not already frozen. - /// - /// Don't invoke this directly - unsafe fn freeze(&mut self); - - /// Unfreeze this context, allowing it to be used again. - /// - /// ## Safety - /// Must be a valid context! - /// Must be currently frozen! - /// - /// Don't invoke this directly - unsafe fn unfreeze(&mut self); - - /// Rebrand to the specified root so that it - /// lives for the `'static` lifetime. - /// - /// ## Safety - /// This function is unsafe to use directly. - /// - /// It should only be used as part of the [safepoint!] macro. - #[inline(always)] - unsafe fn rebrand_static(&self, value: T) -> T::Branded - where - T: GcRebrand<'static, Self::Id>, - T::Branded: Sized, - { - let branded = mem::transmute_copy(&value); - mem::forget(value); - branded - } - /// Rebrand the specified root so that it - /// lives for the lifetime of this context. - /// - /// This effectively undoes the effect of [GcContext::rebrand_static]. - /// - /// ## Safety - /// This function is unsafe to use directly. - /// - /// It should only be used as part of the [safepoint!] macro. - #[inline(always)] - unsafe fn rebrand_self<'gc, T>(&'gc self, value: T) -> T::Branded - where - T: GcRebrand<'gc, Self::Id>, - T::Branded: Sized, - { - let branded = mem::transmute_copy(&value); - mem::forget(value); - branded - } - - /// Invoke the closure with a temporary [GcContext], - /// preserving the specified object as a root of garbage collection. - /// - /// This will add the specified root to the shadow stack - /// before invoking the closure. Therefore, any safepoints - /// invoked inside the closure/sub-function will implicitly preserve all - /// objects reachable from the root in the parent function. - /// - /// This doesn't nessicarrily imply a safepoint, - /// although its wrapper macro ([safepoint_recurse!]) typically does. - /// - /// ## Safety - /// All objects that are in use must be reachable from the specified root. - /// - /// The specified closure could trigger a collection in the sub-context, - /// so all live objects in the parent function need to be reachable from - /// the specified root. - /// - /// See the [safepoint_recurse!] macro for a safe wrapper - unsafe fn recurse_context(&self, root: &mut &mut T, func: F) -> R - where - T: Trace, - F: for<'gc> FnOnce(&'gc mut Self, &'gc mut T) -> R; - - /// Get the [GcSystem] associated with this context - fn system(&self) -> &'_ Self::System; - - /// Get the id of this context - fn id(&self) -> Self::Id; } -/// A simple interface to allocating from a [GcContext]. -/// -/// Some garbage collectors implement more complex interfaces, -/// so implementing this is optional -pub unsafe trait GcSimpleAlloc: GcContext { - /// Allocate room for a object in, but don't finish initializing it. - /// - /// ## Safety - /// The object **must** be initialized by the time the next collection - /// rolls around, so that the collector can properly trace it - unsafe fn alloc_uninit<'gc, T>(&'gc self) -> *mut T - where - T: GcSafe<'gc, Self::Id>; - - /// Allocate the specified object in this garbage collector, - /// binding it to the lifetime of this collector. - /// - /// The object will never be collected until the next safepoint, - /// which is considered a mutation by the borrow checker and will be statically checked. - /// Therefore, we can statically guarantee the pointers will survive until the next safepoint. - /// - /// See `safepoint!` docs on how to properly invoke a safepoint - /// and transfer values across it. - /// - /// This gives a immutable reference to the resulting object. - /// Once allocated, the object can only be correctly modified with a `GcCell` - #[inline] - fn alloc<'gc, T>(&'gc self, value: T) -> Gc<'gc, T, Self::Id> - where - T: GcSafe<'gc, Self::Id>, - { - unsafe { - let ptr = self.alloc_uninit::(); - ptr.write(value); - Gc::from_raw(NonNull::new_unchecked(ptr)) - } - } -} -/// The internal representation of a frozen context -/// -/// ## Safety -/// Don't use this directly!!! -#[doc(hidden)] -#[must_use] -pub struct FrozenContext { - /// The frozen context - context: C, -} -impl FrozenContext { - #[doc(hidden)] - #[inline] - pub unsafe fn new(context: C) -> Self { - FrozenContext { context } - } - #[doc(hidden)] - #[inline] - pub unsafe fn into_context(self) -> C { - self.context - } -} - -/// A trait alias for [CollectorId]s that support [GcSimpleAlloc] -pub trait SimpleAllocCollectorId = CollectorId where ::Context: GcSimpleAlloc; /// A [CollectorId] that supports allocating [GcHandle]s /// @@ -372,9 +115,7 @@ pub unsafe trait CollectorId: Copy + Eq + Hash + Debug + NullTrace + TrustedDrop + 'static + for<'gc> GcSafe<'gc, Self> { /// The type of the garbage collector system - type System: GcSystem; - /// The type of [GcContext] associated with this id. - type Context: GcContext; + type System: GcSystem; /// Get the runtime id of the collector that allocated the [Gc] /// @@ -468,7 +209,7 @@ pub struct Gc<'gc, T: ?Sized, Id: CollectorId> { /// /// The runtime instance of this value can be /// computed from the pointer itself: `NonNull` -> `&CollectorId` - collector_id: PhantomData<&'gc Id::Context>, + collector_id: PhantomData<&'gc Id::System>, } impl<'gc, T: GcSafe<'gc, Id> + ?Sized, Id: CollectorId> Gc<'gc, T, Id> { /// Create a GC pointer from a raw pointer @@ -742,21 +483,6 @@ pub unsafe trait GcHandle + ?Sized>: * How much does it limit flexibility? */ fn use_critical(&self, func: impl FnOnce(&T) -> R) -> R; - - /// Associate this handle with the specified context, - /// allowing its underlying object to be accessed - /// as long as the context is valid. - /// - /// The underlying object can be accessed just like any - /// other object that would be allocated from the context. - /// It'll be properly collected and can even be used as a root - /// at the next safepoint. - fn bind_to<'new_gc>( - &self, - context: &'new_gc ::Context, - ) -> Gc<'new_gc, >::Branded, Self::Id> - where - T: GcRebrand<'new_gc, Self::Id>; } /// Safely trigger a write barrier before diff --git a/src/prelude.rs b/src/prelude.rs index 1e27c61..c44a563 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -6,7 +6,7 @@ //! collected program needs to use the API. // Basic collector types -pub use crate::{Gc, GcContext, GcHandle, GcSimpleAlloc, GcSystem, GcVisitor, HandleCollectorId}; +pub use crate::{Gc, GcHandle, GcSystem, GcVisitor, HandleCollectorId}; // Traits for user code to implement pub use crate::cell::GcCell; pub use crate::AssumeNotTraced; From e6aae10664dfe29ab1d5ceb608555bdd3bbf59b3 Mon Sep 17 00:00:00 2001 From: Techcable Date: Fri, 10 May 2024 14:23:59 -0700 Subject: [PATCH 25/35] Split lib.rs into submodules Doesn't delete any code right now --- src/cell.rs | 2 +- src/lib.rs | 851 +------------------------------- src/manually_traced/core.rs | 1 - src/manually_traced/stdalloc.rs | 3 - src/prelude.rs | 10 +- src/system.rs | 137 +++++ src/trace.rs | 373 ++++++++++++++ src/trace/barrier.rs | 65 +++ src/trace/gcptr.rs | 293 +++++++++++ 9 files changed, 881 insertions(+), 854 deletions(-) create mode 100644 src/system.rs create mode 100644 src/trace.rs create mode 100644 src/trace/barrier.rs create mode 100644 src/trace/gcptr.rs diff --git a/src/cell.rs b/src/cell.rs index d99b8da..9b8fc69 100644 --- a/src/cell.rs +++ b/src/cell.rs @@ -16,7 +16,7 @@ use core::cell::Cell; use zerogc_derive::unsafe_gc_impl; -use crate::{CollectorId, GcDirectBarrier, GcRebrand, GcSafe, NullTrace, Trace}; +use crate::prelude::*; /// A `Cell` pointing to a garbage collected object. /// diff --git a/src/lib.rs b/src/lib.rs index c9ec8e3..d5cfbfe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,22 +43,10 @@ extern crate alloc; */ extern crate self as zerogc; - /* * I want this library to use 'mostly' stable features, * unless there's good justification to use an unstable feature. */ -#[cfg(all(not(feature = "std"), feature = "alloc"))] -use alloc::vec::Vec; -use core::cmp::Ordering; -use core::fmt::{self, Debug, Display, Formatter}; -use core::hash::{Hash, Hasher}; -use core::marker::{PhantomData, Unsize}; -use core::ops::{CoerceUnsized, Deref, DerefMut}; -use core::ptr::NonNull; - -use zerogc_derive::unsafe_gc_impl; -pub use zerogc_derive::{NullTrace, Trace}; #[macro_use] mod manually_traced; @@ -66,843 +54,16 @@ mod manually_traced; mod macros; pub mod cell; pub mod prelude; +pub mod system; +pub mod trace; -/// A garbage collector implementation, -/// conforming to the zerogc API. -pub unsafe trait GcSystem { - /// The type of collector IDs given by this system - type Id: CollectorId; -} - -/// A [CollectorId] that supports allocating [GcHandle]s -/// -/// Not all collectors necessarily support handles. -pub unsafe trait HandleCollectorId: CollectorId { - /// The type of [GcHandle] for this collector. - /// - /// This is parameterized by the *erased* type, - /// not by the original type. - type Handle: GcHandle - where - T: GcSafe<'static, Self> + ?Sized; - - /// Create a handle to the specified GC pointer, - /// which can be used without a context - /// - /// NOTE: Users should only use from [Gc::create_handle]. - /// - /// The system is implicit in the [Gc] - #[doc(hidden)] - fn create_handle<'gc, T>(gc: Gc<'gc, T, Self>) -> Self::Handle - where - T: GcSafe<'gc, Self> + GcRebrand<'static, Self> + ?Sized; -} - -/// Uniquely identifies the collector in case there are -/// multiple collectors. -/// -/// ## Safety -/// To simply the typing, this contains no references to the -/// lifetime of the associated [GcSystem]. -/// -/// It's implicitly held and is unsafe to access. -/// As long as the collector is valid, -/// this id should be too. -/// -/// It should be safe to assume that a collector exists -/// if any of its pointers still do! -pub unsafe trait CollectorId: - Copy + Eq + Hash + Debug + NullTrace + TrustedDrop + 'static + for<'gc> GcSafe<'gc, Self> -{ - /// The type of the garbage collector system - type System: GcSystem; - - /// Get the runtime id of the collector that allocated the [Gc] - /// - /// Assumes that `T: GcSafe<'gc, Self>`, although that can't be - /// proven at compile time. - fn from_gc_ptr<'a, 'gc, T>(gc: &'a Gc<'gc, T, Self>) -> &'a Self - where - T: ?Sized, - 'gc: 'a; - - /// Perform a write barrier before writing to a garbage collected field - /// - /// ## Safety - /// Similar to the [GcDirectBarrier] trait, it can be assumed that - /// the field offset is correct and the types match. - unsafe fn gc_write_barrier<'gc, O: GcSafe<'gc, Self> + ?Sized, V: GcSafe<'gc, Self> + ?Sized>( - owner: &Gc<'gc, O, Self>, - value: &Gc<'gc, V, Self>, - field_offset: usize, - ); - - /// Assume the ID is valid and use it to access the [GcSystem] - /// - /// NOTE: The system is bound to the lifetime of *THIS* id. - /// A CollectorId may have an internal pointer to the system - /// and the pointer may not have a stable address. In other words, - /// it may be difficult to reliably take a pointer to a pointer. - /// - /// ## Safety - /// Undefined behavior if the associated collector no longer exists. - unsafe fn assume_valid_system(&self) -> &Self::System; -} - -/// A garbage collected pointer to a value. -/// -/// This is the equivalent of a garbage collected smart-pointer. -/// It's so smart, you can even coerce it to a reference bound to the lifetime of the `GarbageCollectorRef`. -/// However, all those references are invalidated by the borrow checker as soon as -/// your reference to the collector reaches a safepoint. -/// The objects can only survive garbage collection if they live in this smart-pointer. -/// -/// The smart pointer is simply a guarantee to the garbage collector -/// that this points to a garbage collected object with the correct header, -/// and not some arbitrary bits that you've decided to heap allocate. -/// -/// ## Safety -/// A `Gc` can be safely transmuted back and forth from its corresponding pointer. -/// -/// Unsafe code can rely on a pointer always dereferencing to the same value in between -/// safepoints. This is true even for copying/moving collectors. -/// -/// ## Lifetime -/// The borrow does *not* refer to the value `&'gc T`. -/// Instead, it refers to the *context* `&'gc Id::Context` -/// -/// This is necessary because `T` may have borrowed interior data -/// with a shorter lifetime `'a < 'gc`, making `&'gc T` invalid -/// (because that would imply 'gc: 'a, which is false). -/// -/// This ownership can be thought of in terms of the following (simpler) system. -/// ```no_run -/// # trait GcSafe{} -/// # use core::marker::PhantomData; -/// struct GcContext { -/// values: Vec> -/// } -/// struct Gc<'gc, T: GcSafe> { -/// index: usize, -/// marker: PhantomData, -/// ctx: &'gc GcContext -/// } -/// ``` -/// -/// In this system, safepoints can be thought of mutations -/// that remove dead values from the `Vec`. -/// -/// This ownership equivalency is also the justification for why -/// the `'gc` lifetime can be [covariant](https://doc.rust-lang.org/nomicon/subtyping.html#variance) -/// -/// The only difference is that the real `Gc` structure -/// uses pointers instead of indices. -#[repr(transparent)] -pub struct Gc<'gc, T: ?Sized, Id: CollectorId> { - /// The pointer to the garbage collected value. - /// - /// NOTE: The logical lifetime here is **not** `&'gc T` - /// See the comments on 'Lifetime' for details. - value: NonNull, - /// Marker struct used to statically identify the collector's type, - /// and indicate that 'gc is a logical reference the system. - /// - /// The runtime instance of this value can be - /// computed from the pointer itself: `NonNull` -> `&CollectorId` - collector_id: PhantomData<&'gc Id::System>, -} -impl<'gc, T: GcSafe<'gc, Id> + ?Sized, Id: CollectorId> Gc<'gc, T, Id> { - /// Create a GC pointer from a raw pointer - /// - /// ## Safety - /// Undefined behavior if the underlying pointer is not valid - /// and doesn't correspond to the appropriate id. - #[inline] - pub unsafe fn from_raw(value: NonNull) -> Self { - Gc { - collector_id: PhantomData, - value, - } - } - /// Create a [GcHandle] referencing this object, - /// allowing it to be used without a context - /// and referenced across safepoints. - /// - /// Requires that the collector [supports handles](`HandleCollectorId`) - #[inline] - pub fn create_handle(&self) -> Id::Handle - where - Id: HandleCollectorId, - T: GcRebrand<'static, Id>, - { - Id::create_handle(*self) - } - - /// Get a reference to the system - /// - /// ## Safety - /// This is based on the assumption that a [GcSystem] must outlive - /// all of the pointers it owns. - /// Although it could be restricted to the lifetime of the [CollectorId] - /// (in theory that may have an internal pointer) it will still live for '&self'. - #[inline] - pub fn system(&self) -> &'_ Id::System { - // This assumption is safe - see the docs - unsafe { self.collector_id().assume_valid_system() } - } -} -impl<'gc, T: ?Sized, Id: CollectorId> Gc<'gc, T, Id> { - /// The value of the underlying pointer - #[inline(always)] - pub const fn value(&self) -> &'gc T { - unsafe { *(&self.value as *const NonNull as *const &'gc T) } - } - /// Cast this reference to a raw pointer - /// - /// ## Safety - /// It's undefined behavior to mutate the - /// value. - /// The pointer is only valid as long as - /// the reference is. - #[inline] - pub unsafe fn as_raw_ptr(&self) -> *mut T { - self.value.as_ptr() as *const T as *mut T - } - - /// Get a reference to the collector's id - /// - /// The underlying collector it points to is not necessarily always valid - #[inline] - pub fn collector_id(&self) -> &'_ Id { - Id::from_gc_ptr(self) - } -} - -/// Double-indirection is completely safe -unsafe impl<'gc, T: ?Sized + GcSafe<'gc, Id>, Id: CollectorId> TrustedDrop for Gc<'gc, T, Id> {} -unsafe impl<'gc, T: ?Sized + GcSafe<'gc, Id>, Id: CollectorId> GcSafe<'gc, Id> for Gc<'gc, T, Id> { - #[inline] - unsafe fn trace_inside_gc(gc: &mut Gc<'gc, Self, Id>, visitor: &mut V) -> Result<(), V::Err> - where - V: GcVisitor, - { - // Double indirection is fine. It's just a `Sized` type - visitor.trace_gc(gc) - } -} -/// Rebrand -unsafe impl<'gc, 'new_gc, T, Id> GcRebrand<'new_gc, Id> for Gc<'gc, T, Id> -where - T: GcSafe<'gc, Id> + ?Sized + GcRebrand<'new_gc, Id>, - Id: CollectorId, - Self: Trace, -{ - type Branded = Gc<'new_gc, >::Branded, Id>; -} -unsafe impl<'gc, T: ?Sized + GcSafe<'gc, Id>, Id: CollectorId> Trace for Gc<'gc, T, Id> { - // We always need tracing.... - const NEEDS_TRACE: bool = true; - // we never need to be dropped because we are `Copy` - const NEEDS_DROP: bool = false; - - #[inline] - fn trace(&mut self, visitor: &mut V) -> Result<(), V::Err> { - unsafe { - // We're delegating with a valid pointer. - >::trace_inside_gc(self, visitor) - } - } -} -impl<'gc, T: GcSafe<'gc, Id> + ?Sized, Id: CollectorId> Deref for Gc<'gc, T, Id> { - type Target = T; - - #[inline(always)] - fn deref(&self) -> &Self::Target { - self.value() - } -} -unsafe impl<'gc, O, V, Id> GcDirectBarrier<'gc, Gc<'gc, O, Id>> for Gc<'gc, V, Id> -where - O: GcSafe<'gc, Id> + 'gc, - V: GcSafe<'gc, Id> + 'gc, - Id: CollectorId, -{ - #[inline(always)] - unsafe fn write_barrier(&self, owner: &Gc<'gc, O, Id>, field_offset: usize) { - Id::gc_write_barrier(owner, self, field_offset) - } -} -// We can be copied freely :) -impl<'gc, T: ?Sized, Id: CollectorId> Copy for Gc<'gc, T, Id> {} -impl<'gc, T: ?Sized, Id: CollectorId> Clone for Gc<'gc, T, Id> { - #[inline(always)] - fn clone(&self) -> Self { - *self - } -} -// Delegating impls -impl<'gc, T: GcSafe<'gc, Id> + Hash, Id: CollectorId> Hash for Gc<'gc, T, Id> { - #[inline] - fn hash(&self, state: &mut H) { - self.value().hash(state) - } -} -impl<'gc, T: GcSafe<'gc, Id> + PartialEq, Id: CollectorId> PartialEq for Gc<'gc, T, Id> { - #[inline] - fn eq(&self, other: &Self) -> bool { - // NOTE: We compare by value, not identity - self.value() == other.value() - } -} -impl<'gc, T: GcSafe<'gc, Id> + Eq, Id: CollectorId> Eq for Gc<'gc, T, Id> {} -impl<'gc, T: GcSafe<'gc, Id> + PartialEq, Id: CollectorId> PartialEq for Gc<'gc, T, Id> { - #[inline] - fn eq(&self, other: &T) -> bool { - self.value() == other - } -} -impl<'gc, T: GcSafe<'gc, Id> + PartialOrd, Id: CollectorId> PartialOrd for Gc<'gc, T, Id> { - #[inline] - fn partial_cmp(&self, other: &Self) -> Option { - self.value().partial_cmp(other.value()) - } -} -impl<'gc, T: GcSafe<'gc, Id> + PartialOrd, Id: CollectorId> PartialOrd for Gc<'gc, T, Id> { - #[inline] - fn partial_cmp(&self, other: &T) -> Option { - self.value().partial_cmp(other) - } -} -impl<'gc, T: GcSafe<'gc, Id> + Ord, Id: CollectorId> Ord for Gc<'gc, T, Id> { - #[inline] - fn cmp(&self, other: &Self) -> Ordering { - self.value().cmp(other) - } -} -impl<'gc, T: ?Sized + GcSafe<'gc, Id> + Debug, Id: CollectorId> Debug for Gc<'gc, T, Id> { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - if !f.alternate() { - // Pretend we're a newtype by default - f.debug_tuple("Gc").field(&self.value()).finish() - } else { - // Alternate spec reveals `collector_id` - f.debug_struct("Gc") - .field("collector_id", &self.collector_id) - .field("value", &self.value()) - .finish() - } - } -} -impl<'gc, T: ?Sized + GcSafe<'gc, Id> + Display, Id: CollectorId> Display for Gc<'gc, T, Id> { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - Display::fmt(&self.value(), f) - } -} - -/// In order to send *references* between threads, -/// the underlying type must be sync. -/// -/// This is the same reason that `Arc: Send` requires `T: Sync` -unsafe impl<'gc, T, Id> Send for Gc<'gc, T, Id> -where - T: GcSafe<'gc, Id> + ?Sized + Sync, - Id: CollectorId + Sync, -{ -} - -/// If the underlying type is `Sync`, it's safe -/// to share garbage collected references between threads. -/// -/// The safety of the collector itself depends on whether [CollectorId] is Sync. -/// If it is, the whole garbage collection implementation should be as well. -unsafe impl<'gc, T, Id> Sync for Gc<'gc, T, Id> -where - T: GcSafe<'gc, Id> + ?Sized + Sync, - Id: CollectorId + Sync, -{ -} - -/// Indicates that a mutable reference to a type -/// is safe to use without triggering a write barrier. -/// -/// This means one of either two things: -/// 1. This type doesn't need any write barriers -/// 2. Mutating this type implicitly triggers a write barrier. -/// -/// This is the bound for `RefCell`. Since a RefCell doesn't explicitly trigger write barriers, -/// a `RefCell` can only be used with `T` if either: -/// 1. `T` doesn't need any write barriers or -/// 2. `T` implicitly triggers write barriers on any mutation -pub unsafe trait ImplicitWriteBarrier {} -unsafe impl ImplicitWriteBarrier for T {} - -/// A owned handle which points to a garbage collected object. -/// -/// This is considered a root by the garbage collector that is independent -/// of any specific [GcContext]. Safepoints -/// don't need to be informed of this object for collection to start. -/// The root is manually managed by user-code, much like a [Box] or -/// a reference counted pointer. -/// -/// This can be cloned and stored independently from a context, -/// bridging the gap between native memory and managed memory. -/// These are useful to pass to C APIs or any other code -/// that doesn't cooperate with zerogc. -/// -/// ## Tracing -/// The object behind this handle is already considered a root of the collection. -/// It should always be considered reachable by the garbage collector. -/// -/// Validity is tracked by this smart-pointer and not by tracing. -/// Therefore it is safe to implement [NullTrace] for handles. -/* - * TODO: Should we drop the Clone requirement? - */ -pub unsafe trait GcHandle + ?Sized>: - Sized + Clone + NullTrace + for<'gc> GcSafe<'gc, Self::Id> -{ - /// The type of the system used with this handle - type System: GcSystem; - /// The type of [CollectorId] used with this sytem - type Id: CollectorId; - - /// Access this handle inside the closure, - /// possibly associating it with the specified - /// - /// This is accesses the object within "critical section" - /// that will **block collections** - /// for as long as the closure is in use. - /// - /// These calls cannot be invoked recursively or they - /// may cause a deadlock. - /// - /// This is similar in purpose to JNI's [GetPrimitiveArrayCritical](https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#GetPrimitiveArrayCritical_ReleasePrimitiveArrayCritical). - /// However it never performs a copy, it is just guarenteed to block any collections. - /* - * TODO: Should we require this of all collectors? - * How much does it limit flexibility? - */ - fn use_critical(&self, func: impl FnOnce(&T) -> R) -> R; -} - -/// Safely trigger a write barrier before -/// writing to a garbage collected value. -/// -/// The value must be in managed memory, -/// a *direct* part of a garbage collected object. -/// Write barriers (and writes) must include a reference -/// to its owning object. -/// -/// ## Safety -/// It is undefined behavior to forget to trigger a write barrier. -/// -/// Field offsets are unchecked. They must refer to the correct -/// offset (in bytes). -/// -/// ### Indirection -/// This trait only support "direct" writes, -/// where the destination field is inline with the source object. -/// -/// For example it's correct to implement `GcDirectWrite for (A, B)`, -/// since since `A` is inline with the owning tuple. -/// -/// It is **incorrect** to implement `GcDirectWrite for Vec`, -/// since it `T` is indirectly referred to by the vector. -/// There's no "field offset" we can use to get from `*mut Vec` -> `*mut T`. -/// -/// The only exception to this rule is [Gc] itself. -/// GcRef can freely implement [GcDirectBarrier] for any (and all values), -/// even though it's just a pointer. -/// It's the final destination of all write barriers and is expected -/// to internally handle the indirection. -pub unsafe trait GcDirectBarrier<'gc, OwningRef>: Trace { - /// Trigger a write barrier, - /// before writing to one of the owning object's managed fields - /// - /// It is undefined behavior to mutate a garbage collected field - /// without inserting a write barrier before it. - /// - /// Generational, concurrent and incremental GCs need this to maintain - /// the tricolor invariant. - /// - /// ## Safety - /// The specified field offset must point to a valid field - /// in the source object. - /// - /// The type of this value must match the appropriate field - unsafe fn write_barrier(&self, owner: &OwningRef, field_offset: usize); -} - -/// Indicates that a type's [Drop](core::ops::Drop) implementation is trusted -/// not to resurrect any garbage collected object. -/// -/// This is a requirement for a type to be allocated in [GcSimpleAlloc], -/// and also for a type to be `GcSafe`. -/// -/// Unlike java finalizers, these trusted destructors -/// avoids a second pass to check for resurrected objects. -/// This is important giving the frequency of destructors -/// when interoperating with native code. -/// -/// The collector is of course free to implement support java-style finalizers in addition -/// to supporting these "trusted" destructors. -/// -/// ## Safety -/// To implement this trait, the type's destructor -/// must never reference garbage collected pointers that may already be dead -/// and must never resurrect dead objects. -/// The garbage collector may have already freed the other objects -/// before calling this type's drop function. -/// -pub unsafe trait TrustedDrop: Trace {} - -/// A marker trait correlating all garbage collected pointers -/// corresponding to the specified `Id` are valid for the `'gc` lifetime. -/// -/// If this type is implemented for a specific [CollectorId] `Id`, -/// it indicates the possibility of containing pointers belonging to that collector. -/// -/// If a type is `NullTrace, it should implement `GcSafe` for all possible collectors. -/// However, if a type `NEEDS_TRACE`, it will usually only implement GcSafe for the specific -/// [CollectorId]s it happens to contain (although this is not guarenteed). -/// -/// ## Mixing with other lifetimes -/// Note that `T: GcSafe<'gc, T>` does *not* necessarily imply `T: 'gc`. -/// This allows a garbage collected lifetime to contain shorter lifetimes -/// -/// For example, -/// ``` -/// # use zerogc::epsilon::{Gc, EpsilonContext as GcContext, EpsilonCollectorId as CollectorId}; -/// # use zerogc::prelude::*; -/// #[derive(Trace)] -/// #[zerogc(ignore_lifetimes("'a"), collector_ids(CollectorId))] -/// struct TempLifetime<'gc, 'a> { -/// temp: &'a i32, -/// gc: Gc<'gc, i32> -/// } -/// fn alloc_ref_temp<'gc>(ctx: &'gc GcContext, long_lived: Gc<'gc, i32>) { -/// let temp = 5; // Lives for 'a (shorter than 'gc) -/// let temp_ref = ctx.alloc(TempLifetime { -/// temp: &temp, gc: long_lived -/// }); -/// assert_eq!(&temp as *const _, temp_ref.temp as *const _) -/// } -/// ``` -/// -/// ## Mixing collectors -/// The `Id` parameter allows mixing and matching pointers from different collectors, -/// each with their own 'gc lifetime. -/// For example, -/// ```ignore // TODO: Support this. See issue #33 -/// # use zerogc::{Gc, CollectorId, GcSafe}; -/// # use zerogc_derive::Trace; -/// # type JsGcId = zerogc::epsilon::EpsilonCollectorId; -/// # type OtherGcId = zerogc::epsilon::EpsilonCollectorId; -/// #[derive(Trace)] -/// struct MixedGc<'gc, 'js> { -/// internal_ptr: Gc<'gc, i32, OtherGcId>, -/// js_ptr: Gc<'js, i32, JsGcId> -/// } -/// impl<'gc, 'js> MixedGc<'gc, 'js> { -/// fn verify(&self) { -/// assert!(>::assert_gc_safe()); -/// assert!(>::assert_gc_safe()); -/// // NOT implemented: > -/// } -/// } -/// ``` -/// -/// ## Safety -/// In addition to the guarantees of [Trace] and [TrustedDrop], -/// implementing this type requires that all [Gc] pointers of -/// the specified `Id` have the `'gc` lifetime (if there are any at all). -pub unsafe trait GcSafe<'gc, Id: CollectorId>: Trace + TrustedDrop { - /// Assert this type is GC safe - /// - /// Only used by procedural derive - #[doc(hidden)] - fn assert_gc_safe() -> bool - where - Self: Sized, - { - true - } - /// Trace this object behind a [Gc] pointer. - /// - /// This is **required** to delegate to one of the following methods on [GcVisitor]: - /// 1. [GcVisitor::trace_gc] - For regular, `Sized` types - /// 2. [GcVisitor::trace_array] - For slices and arrays - /// 3. [GcVisitor::trace_trait_object] - For trait objects - /// - /// ## Safety - /// This must delegate to the appropriate method on [GcVisitor], - /// or undefined behavior will result. - /// - /// The user is required to supply an appropriate [Gc] pointer. - unsafe fn trace_inside_gc(gc: &mut Gc<'gc, Self, Id>, visitor: &mut V) -> Result<(), V::Err> - where - V: GcVisitor; -} - -/// A [GcSafe] type with all garbage-collected pointers -/// erased to the `'static` type. -/// -/// This should generally only be used by internal code. -/// -/// ## Safety -/// This type is incredibly unsafe. It eliminates all the safety invariants -/// of lifetimes. -pub trait GcSafeErased = GcSafe<'static, Id>; +// core traits, used by macros +pub use self::system::{CollectorId, GcSystem}; +pub use self::trace::barrier::*; +pub use self::trace::*; /// Assert that a type implements Copy /// /// Used by the derive code #[doc(hidden)] pub fn assert_copy() {} - -/// A wrapper type that assumes its contents don't need to be traced -#[repr(transparent)] -#[derive(Copy, Clone, Debug)] -pub struct AssumeNotTraced(T); -impl AssumeNotTraced { - /// Assume the specified value doesn't need to be traced - /// - /// ## Safety - /// Undefined behavior if the value contains anything that need to be traced - /// by a garbage collector. - #[inline] - pub unsafe fn new(value: T) -> Self { - AssumeNotTraced(value) - } - /// Unwrap the inner value of this wrapper - #[inline] - pub fn into_inner(self) -> T { - self.0 - } -} -impl Deref for AssumeNotTraced { - type Target = T; - - #[inline] - fn deref(&self) -> &Self::Target { - &self.0 - } -} -impl DerefMut for AssumeNotTraced { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} -unsafe_gc_impl! { - target => AssumeNotTraced, - params => [T], - bounds => { - // Unconditionally implement all traits - Trace => always, - TraceImmutable => always, - TrustedDrop => always, - GcSafe => always, - GcRebrand => always, - }, - null_trace => always, - branded_type => Self, - NEEDS_TRACE => false, - NEEDS_DROP => core::mem::needs_drop::(), - trace_template => |self, visitor| { /* nop */ Ok(()) } -} - -/// Allows changing the lifetime of all [`Gc`] references. -/// -/// Any other lifetimes should be unaffected by the 'branding'. -/// Since we control the only lifetime, -/// we don't have to worry about interior references. -/// -/// ## Safety -/// Assuming the `'new_gc` lifetime is correct, -/// It must be safe to transmute back and forth to `Self::Branded`, -/// switching all garbage collected references from `Gc<'old_gc, T>` to `Gc<'new_gc, T>` -pub unsafe trait GcRebrand<'new_gc, Id: CollectorId>: Trace { - /// This type with all garbage collected lifetimes - /// changed to `'new_gc` - /// - /// This must have the same in-memory repr as `Self`, - /// so that it's safe to transmute. - type Branded: GcSafe<'new_gc, Id> + ?Sized; - - /// Assert this type can be rebranded - /// - /// Only used by procedural derive - #[doc(hidden)] - fn assert_rebrand() {} -} - -/// Indicates that a type can be traced by a garbage collector. -/// -/// This doesn't necessarily mean that the type is safe to allocate in a garbage collector ([GcSafe]). -/// -/// ## Safety -/// See the documentation of the `trace` method for more info. -/// Essentially, this object must faithfully trace anything that -/// could contain garbage collected pointers or other `Trace` items. -pub unsafe trait Trace { - /// Whether this type needs to be traced by the garbage collector. - /// - /// Some primitive types don't need to be traced at all, - /// and can be simply ignored by the garbage collector. - /// - /// Collections should usually delegate this decision to their element type, - /// claiming the need for tracing only if their elements do. - /// For example, to decide `Vec::NEEDS_TRACE` you'd check whether `u32::NEEDS_TRACE` (false), - /// and so then `Vec` doesn't need to be traced. - /// By the same logic, `Vec>` does need to be traced, - /// since it contains a garbage collected pointer. - /// - /// If there are multiple types involved, you should check if any of them need tracing. - /// One perfect example of this is structure/tuple types which check - /// `field1::NEEDS_TRACE || field2::NEEDS_TRACE || field3::needs_trace`. - /// The fields which don't need tracing will always ignored by `GarbageCollector::trace`, - /// while the fields that do will be properly traced. - /// - /// False negatives will always result in completely undefined behavior. - /// False positives could result in un-necessary tracing, but are perfectly safe otherwise. - /// Therefore, when in doubt you always assume this is true. - /// - /// If this is true `NullTrace` should (but doesn't have to) be implemented. - /* - * TODO: Should we move this to `GcSafe`? - * Needing tracing for `Id1` doesn't nessicitate - * needing tracing for `Id2` - */ - const NEEDS_TRACE: bool; - /// If this type needs a destructor run. - /// - /// This is usually equivalent to [core::mem::needs_drop]. - /// However, procedurally derived code can sometimes provide - /// a no-op drop implementation (for safety) - /// which would lead to a false positive with `core::mem::needs_drop()` - const NEEDS_DROP: bool; - /// Trace each field in this type. - /// - /// Structures should trace each of their fields, - /// and collections should trace each of their elements. - /// - /// ### Safety - /// Some types (like `Gc`) need special actions taken when they're traced, - /// but those are somewhat rare and are usually already provided by the garbage collector. - /// - /// Behavior is restricted during tracing: - /// ## Permitted Behavior - /// - Reading your own memory (includes iteration) - /// - Interior mutation is undefined behavior, even if you use `GcCell` - /// - Calling `GcVisitor::trace` with the specified collector - /// - `GcVisitor::trace` already verifies that the ids match, so you don't need to do that - /// - Panicking on unrecoverable errors - /// - This should be reserved for cases where you are seriously screwed up, - /// and can't fulfill your contract to trace your interior properly. - /// - One example is `Gc` which panics if the garbage collectors are mismatched - /// - Garbage collectors may chose to [abort](std::process::abort) if they encounter a panic, - /// so you should avoid doing it if possible. - /// ## Never Permitted Behavior - /// - Forgetting a element of a collection, or field of a structure - /// - If you forget an element undefined behavior will result - /// - This is why you should always prefer automatically derived implementations where possible. - /// - With an automatically derived implementation you will never miss a field - /// - It is undefined behavior to mutate any of your own data. - /// - The mutable `&mut self` is just so copying collectors can relocate GC pointers - /// - Calling other operations on the garbage collector (including allocations) - fn trace(&mut self, visitor: &mut V) -> Result<(), V::Err>; -} - -/// A type that can be safely traced/relocated -/// without having to use a mutable reference -/// -/// Types with interior mutability (like `RefCell` or `Cell>`) -/// can safely implement this, since they allow safely relocating the pointer -/// without a mutable reference. -/// Likewise primitives (with new garbage collected data) can also -/// implement this (since they have nothing to trace). -pub unsafe trait TraceImmutable: Trace { - /// Trace an immutable reference to this type - /// - /// The visitor may want to relocate garbage collected pointers, - /// so any `Gc` pointers must be behind interior mutability. - fn trace_immutable(&self, visitor: &mut V) -> Result<(), V::Err>; -} - -/// A type that can be traced via dynamic dispatch, -/// specialized for a particular [CollectorId]. -/// -/// This indicates that the underlying type implements both [Trace] -/// and [GcSafe], -/// even though the specifics may not be known at compile time. -/// If the type is allocated inside a [Gc] pointer, -/// collectors can usually use their own runtime type information -/// to dispatch to the correct tracing code. -/// -/// This is useful for use in trait objects, -/// because this marker type is object safe (unlike the regular [Trace] trait). -/// -/// ## Safety -/// This type should never be implemented directly. -/// It is automatically implemented for all types that are `Trace + GcSafe`. -/// -/// If an object implements this trait, then it the underlying value -/// **must** implement [Trace] and [GcSafe] at runtime, -/// even though that can't be proved at compile time. -/// -/// The garbage collector will be able to use its runtime type information -/// to find the appropriate implementation at runtime, -/// even though its not known at compile tme. -pub unsafe trait DynTrace<'gc, Id: CollectorId> {} -unsafe impl<'gc, Id: CollectorId, T: ?Sized + Trace + GcSafe<'gc, Id>> DynTrace<'gc, Id> for T {} - -impl<'gc, T, U, Id> CoerceUnsized> for Gc<'gc, T, Id> -where - T: ?Sized + GcSafe<'gc, Id> + Unsize, - U: ?Sized + GcSafe<'gc, Id>, - Id: CollectorId, -{ -} - -/// Marker types for types that don't need to be traced -/// -/// If this trait is implemented `Trace::NEEDS_TRACE` must be false -pub unsafe trait NullTrace: Trace + TraceImmutable { - /// Dummy method for macros to verify that a type actually implements `NullTrace` - #[doc(hidden)] - #[inline] - fn verify_null_trace() - where - Self: Sized, - { - } -} - -/// Visits garbage collected objects -/// -/// This should only be used by a [GcSystem] -pub unsafe trait GcVisitor: Sized { - /// The type of errors returned by this visitor - type Err: Debug; - - /// Trace a reference to the specified value - #[inline] - fn trace(&mut self, value: &mut T) -> Result<(), Self::Err> { - value.trace(self) - } - /// Trace an immutable reference to the specified value - #[inline] - fn trace_immutable(&mut self, value: &T) -> Result<(), Self::Err> { - value.trace_immutable(self) - } - - /// Visit a garbage collected pointer - /// - /// ## Safety - /// Undefined behavior if the GC pointer isn't properly visited. - unsafe fn trace_gc<'gc, T, Id>(&mut self, gc: &mut Gc<'gc, T, Id>) -> Result<(), Self::Err> - where - T: GcSafe<'gc, Id>, - Id: CollectorId; -} diff --git a/src/manually_traced/core.rs b/src/manually_traced/core.rs index c3acab6..82599d1 100644 --- a/src/manually_traced/core.rs +++ b/src/manually_traced/core.rs @@ -9,7 +9,6 @@ use core::marker::PhantomData; use core::num::Wrapping; use crate::prelude::*; -use crate::GcDirectBarrier; use zerogc_derive::unsafe_gc_impl; diff --git a/src/manually_traced/stdalloc.rs b/src/manually_traced/stdalloc.rs index 25fe7fc..5cf3f15 100644 --- a/src/manually_traced/stdalloc.rs +++ b/src/manually_traced/stdalloc.rs @@ -2,13 +2,10 @@ //! //! These can be used in `#![no_std]` crates without requiring //! the entire standard library. -#[cfg(not(feature = "std"))] use alloc::boxed::Box; use alloc::rc::Rc; -#[cfg(not(feature = "std"))] use alloc::string::String; use alloc::sync::Arc; -#[cfg(not(feature = "std"))] use alloc::vec::Vec; use crate::prelude::*; diff --git a/src/prelude.rs b/src/prelude.rs index c44a563..66e4298 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -6,9 +6,11 @@ //! collected program needs to use the API. // Basic collector types -pub use crate::{Gc, GcHandle, GcSystem, GcVisitor, HandleCollectorId}; +pub use crate::system::{GcHandle, GcSystem, HandleCollectorId}; +pub use crate::trace::{Gc, GcVisitor}; // Traits for user code to implement pub use crate::cell::GcCell; -pub use crate::AssumeNotTraced; -pub use crate::CollectorId; -pub use crate::{GcRebrand, GcSafe, NullTrace, Trace, TraceImmutable, TrustedDrop}; +pub use crate::system::CollectorId; +pub use crate::trace::barrier::GcDirectBarrier; +pub use crate::trace::AssumeNotTraced; +pub use crate::trace::{GcRebrand, GcSafe, NullTrace, Trace, TraceImmutable, TrustedDrop}; diff --git a/src/system.rs b/src/system.rs new file mode 100644 index 0000000..f2d519b --- /dev/null +++ b/src/system.rs @@ -0,0 +1,137 @@ +//! Defines the [`GcSystem`] API for collector backends.d + +use core::fmt::Debug; +use core::hash::Hash; + +use crate::trace::{Gc, GcRebrand, GcSafe, NullTrace, TrustedDrop}; + +/// A garbage collector implementation, +/// conforming to the zerogc API. +pub unsafe trait GcSystem { + /// The type of collector IDs given by this system + type Id: CollectorId; +} + +/// A [CollectorId] that supports allocating [GcHandle]s +/// +/// Not all collectors necessarily support handles. +pub unsafe trait HandleCollectorId: CollectorId { + /// The type of [GcHandle] for this collector. + /// + /// This is parameterized by the *erased* type, + /// not by the original type. + type Handle: GcHandle + where + T: GcSafe<'static, Self> + ?Sized; + + /// Create a handle to the specified GC pointer, + /// which can be used without a context + /// + /// NOTE: Users should only use from [Gc::create_handle]. + /// + /// The system is implicit in the [Gc] + #[doc(hidden)] + fn create_handle<'gc, T>(gc: Gc<'gc, T, Self>) -> Self::Handle + where + T: GcSafe<'gc, Self> + GcRebrand<'static, Self> + ?Sized; +} + +/// Uniquely identifies the collector in case there are +/// multiple collectors. +/// +/// ## Safety +/// To simply the typing, this contains no references to the +/// lifetime of the associated [GcSystem]. +/// +/// It's implicitly held and is unsafe to access. +/// As long as the collector is valid, +/// this id should be too. +/// +/// It should be safe to assume that a collector exists +/// if any of its pointers still do! +pub unsafe trait CollectorId: + Copy + Eq + Hash + Debug + NullTrace + TrustedDrop + 'static + for<'gc> GcSafe<'gc, Self> +{ + /// The type of the garbage collector system + type System: GcSystem; + + /// Get the runtime id of the collector that allocated the [Gc] + /// + /// Assumes that `T: GcSafe<'gc, Self>`, although that can't be + /// proven at compile time. + fn from_gc_ptr<'a, 'gc, T>(gc: &'a Gc<'gc, T, Self>) -> &'a Self + where + T: ?Sized, + 'gc: 'a; + + /// Perform a write barrier before writing to a garbage collected field + /// + /// ## Safety + /// Similar to the [GcDirectBarrier] trait, it can be assumed that + /// the field offset is correct and the types match. + unsafe fn gc_write_barrier<'gc, O: GcSafe<'gc, Self> + ?Sized, V: GcSafe<'gc, Self> + ?Sized>( + owner: &Gc<'gc, O, Self>, + value: &Gc<'gc, V, Self>, + field_offset: usize, + ); + + /// Assume the ID is valid and use it to access the [GcSystem] + /// + /// NOTE: The system is bound to the lifetime of *THIS* id. + /// A CollectorId may have an internal pointer to the system + /// and the pointer may not have a stable address. In other words, + /// it may be difficult to reliably take a pointer to a pointer. + /// + /// ## Safety + /// Undefined behavior if the associated collector no longer exists. + unsafe fn assume_valid_system(&self) -> &Self::System; +} + +/// A owned handle which points to a garbage collected object. +/// +/// This is considered a root by the garbage collector that is independent +/// of any specific [GcContext]. Safepoints +/// don't need to be informed of this object for collection to start. +/// The root is manually managed by user-code, much like a [Box] or +/// a reference counted pointer. +/// +/// This can be cloned and stored independently from a context, +/// bridging the gap between native memory and managed memory. +/// These are useful to pass to C APIs or any other code +/// that doesn't cooperate with zerogc. +/// +/// ## Tracing +/// The object behind this handle is already considered a root of the collection. +/// It should always be considered reachable by the garbage collector. +/// +/// Validity is tracked by this smart-pointer and not by tracing. +/// Therefore it is safe to implement [NullTrace] for handles. +/* + * TODO: Should we drop the Clone requirement? + */ +pub unsafe trait GcHandle + ?Sized>: + Sized + Clone + NullTrace + for<'gc> GcSafe<'gc, Self::Id> +{ + /// The type of the system used with this handle + type System: GcSystem; + /// The type of [CollectorId] used with this sytem + type Id: CollectorId; + + /// Access this handle inside the closure, + /// possibly associating it with the specified + /// + /// This is accesses the object within "critical section" + /// that will **block collections** + /// for as long as the closure is in use. + /// + /// These calls cannot be invoked recursively or they + /// may cause a deadlock. + /// + /// This is similar in purpose to JNI's [GetPrimitiveArrayCritical](https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#GetPrimitiveArrayCritical_ReleasePrimitiveArrayCritical). + /// However it never performs a copy, it is just guarenteed to block any collections. + /* + * TODO: Should we require this of all collectors? + * How much does it limit flexibility? + */ + fn use_critical(&self, func: impl FnOnce(&T) -> R) -> R; +} diff --git a/src/trace.rs b/src/trace.rs new file mode 100644 index 0000000..e1d6151 --- /dev/null +++ b/src/trace.rs @@ -0,0 +1,373 @@ +//! Defines the core [Trace] trait. + +use core::fmt::Debug; +use core::ops::{Deref, DerefMut}; + +use zerogc_derive::unsafe_gc_impl; + +use crate::system::CollectorId; + +pub mod barrier; +mod gcptr; + +pub use self::gcptr::Gc; + +/// Allows changing the lifetime of all [`Gc`] references. +/// +/// Any other lifetimes should be unaffected by the 'branding'. +/// Since we control the only lifetime, +/// we don't have to worry about interior references. +/// +/// ## Safety +/// Assuming the `'new_gc` lifetime is correct, +/// It must be safe to transmute back and forth to `Self::Branded`, +/// switching all garbage collected references from `Gc<'old_gc, T>` to `Gc<'new_gc, T>` +pub unsafe trait GcRebrand<'new_gc, Id: CollectorId>: Trace { + /// This type with all garbage collected lifetimes + /// changed to `'new_gc` + /// + /// This must have the same in-memory repr as `Self`, + /// so that it's safe to transmute. + type Branded: GcSafe<'new_gc, Id> + ?Sized; + + /// Assert this type can be rebranded + /// + /// Only used by procedural derive + #[doc(hidden)] + fn assert_rebrand() {} +} + +/// Indicates that a type can be traced by a garbage collector. +/// +/// This doesn't necessarily mean that the type is safe to allocate in a garbage collector ([GcSafe]). +/// +/// ## Safety +/// See the documentation of the `trace` method for more info. +/// Essentially, this object must faithfully trace anything that +/// could contain garbage collected pointers or other `Trace` items. +pub unsafe trait Trace { + /// Whether this type needs to be traced by the garbage collector. + /// + /// Some primitive types don't need to be traced at all, + /// and can be simply ignored by the garbage collector. + /// + /// Collections should usually delegate this decision to their element type, + /// claiming the need for tracing only if their elements do. + /// For example, to decide `Vec::NEEDS_TRACE` you'd check whether `u32::NEEDS_TRACE` (false), + /// and so then `Vec` doesn't need to be traced. + /// By the same logic, `Vec>` does need to be traced, + /// since it contains a garbage collected pointer. + /// + /// If there are multiple types involved, you should check if any of them need tracing. + /// One perfect example of this is structure/tuple types which check + /// `field1::NEEDS_TRACE || field2::NEEDS_TRACE || field3::needs_trace`. + /// The fields which don't need tracing will always ignored by `GarbageCollector::trace`, + /// while the fields that do will be properly traced. + /// + /// False negatives will always result in completely undefined behavior. + /// False positives could result in un-necessary tracing, but are perfectly safe otherwise. + /// Therefore, when in doubt you always assume this is true. + /// + /// If this is true `NullTrace` should (but doesn't have to) be implemented. + /* + * TODO: Should we move this to `GcSafe`? + * Needing tracing for `Id1` doesn't nessicitate + * needing tracing for `Id2` + */ + const NEEDS_TRACE: bool; + /// If this type needs a destructor run. + /// + /// This is usually equivalent to [core::mem::needs_drop]. + /// However, procedurally derived code can sometimes provide + /// a no-op drop implementation (for safety) + /// which would lead to a false positive with `core::mem::needs_drop()` + const NEEDS_DROP: bool; + /// Trace each field in this type. + /// + /// Structures should trace each of their fields, + /// and collections should trace each of their elements. + /// + /// ### Safety + /// Some types (like `Gc`) need special actions taken when they're traced, + /// but those are somewhat rare and are usually already provided by the garbage collector. + /// + /// Behavior is restricted during tracing: + /// ## Permitted Behavior + /// - Reading your own memory (includes iteration) + /// - Interior mutation is undefined behavior, even if you use `GcCell` + /// - Calling `GcVisitor::trace` with the specified collector + /// - `GcVisitor::trace` already verifies that the ids match, so you don't need to do that + /// - Panicking on unrecoverable errors + /// - This should be reserved for cases where you are seriously screwed up, + /// and can't fulfill your contract to trace your interior properly. + /// - One example is `Gc` which panics if the garbage collectors are mismatched + /// - Garbage collectors may chose to [abort](std::process::abort) if they encounter a panic, + /// so you should avoid doing it if possible. + /// ## Never Permitted Behavior + /// - Forgetting a element of a collection, or field of a structure + /// - If you forget an element undefined behavior will result + /// - This is why you should always prefer automatically derived implementations where possible. + /// - With an automatically derived implementation you will never miss a field + /// - It is undefined behavior to mutate any of your own data. + /// - The mutable `&mut self` is just so copying collectors can relocate GC pointers + /// - Calling other operations on the garbage collector (including allocations) + fn trace(&mut self, visitor: &mut V) -> Result<(), V::Err>; +} + +/// A type that can be safely traced/relocated +/// without having to use a mutable reference +/// +/// Types with interior mutability (like `RefCell` or `Cell>`) +/// can safely implement this, since they allow safely relocating the pointer +/// without a mutable reference. +/// Likewise primitives (with new garbage collected data) can also +/// implement this (since they have nothing to trace). +pub unsafe trait TraceImmutable: Trace { + /// Trace an immutable reference to this type + /// + /// The visitor may want to relocate garbage collected pointers, + /// so any `Gc` pointers must be behind interior mutability. + fn trace_immutable(&self, visitor: &mut V) -> Result<(), V::Err>; +} + +/// A type that can be traced via dynamic dispatch, +/// specialized for a particular [CollectorId]. +/// +/// This indicates that the underlying type implements both [Trace] +/// and [GcSafe], +/// even though the specifics may not be known at compile time. +/// If the type is allocated inside a [Gc] pointer, +/// collectors can usually use their own runtime type information +/// to dispatch to the correct tracing code. +/// +/// This is useful for use in trait objects, +/// because this marker type is object safe (unlike the regular [Trace] trait). +/// +/// ## Safety +/// This type should never be implemented directly. +/// It is automatically implemented for all types that are `Trace + GcSafe`. +/// +/// If an object implements this trait, then it the underlying value +/// **must** implement [Trace] and [GcSafe] at runtime, +/// even though that can't be proved at compile time. +/// +/// The garbage collector will be able to use its runtime type information +/// to find the appropriate implementation at runtime, +/// even though its not known at compile tme. +pub unsafe trait DynTrace<'gc, Id: CollectorId> {} +unsafe impl<'gc, Id: CollectorId, T: ?Sized + Trace + GcSafe<'gc, Id>> DynTrace<'gc, Id> for T {} + +/// Marker types for types that don't need to be traced +/// +/// If this trait is implemented `Trace::NEEDS_TRACE` must be false +pub unsafe trait NullTrace: Trace + TraceImmutable { + /// Dummy method for macros to verify that a type actually implements `NullTrace` + #[doc(hidden)] + #[inline] + fn verify_null_trace() + where + Self: Sized, + { + } +} + +/// Visits garbage collected objects +/// +/// This should only be used by a [GcSystem] +pub unsafe trait GcVisitor: Sized { + /// The type of errors returned by this visitor + type Err: Debug; + + /// Trace a reference to the specified value + #[inline] + fn trace(&mut self, value: &mut T) -> Result<(), Self::Err> { + value.trace(self) + } + /// Trace an immutable reference to the specified value + #[inline] + fn trace_immutable(&mut self, value: &T) -> Result<(), Self::Err> { + value.trace_immutable(self) + } + + /// Visit a garbage collected pointer + /// + /// ## Safety + /// Undefined behavior if the GC pointer isn't properly visited. + unsafe fn trace_gc<'gc, T, Id>(&mut self, gc: &mut Gc<'gc, T, Id>) -> Result<(), Self::Err> + where + T: GcSafe<'gc, Id>, + Id: CollectorId; +} + +/// Indicates that a type's [Drop](core::ops::Drop) implementation is trusted +/// not to resurrect any garbage collected object. +/// +/// This is a requirement for a type to be allocated in [GcSimpleAlloc], +/// and also for a type to be `GcSafe`. +/// +/// Unlike java finalizers, these trusted destructors +/// avoids a second pass to check for resurrected objects. +/// This is important giving the frequency of destructors +/// when interoperating with native code. +/// +/// The collector is of course free to implement support java-style finalizers in addition +/// to supporting these "trusted" destructors. +/// +/// ## Safety +/// To implement this trait, the type's destructor +/// must never reference garbage collected pointers that may already be dead +/// and must never resurrect dead objects. +/// The garbage collector may have already freed the other objects +/// before calling this type's drop function. +/// +pub unsafe trait TrustedDrop: Trace {} + +/// A marker trait correlating all garbage collected pointers +/// corresponding to the specified `Id` are valid for the `'gc` lifetime. +/// +/// If this type is implemented for a specific [CollectorId] `Id`, +/// it indicates the possibility of containing pointers belonging to that collector. +/// +/// If a type is `NullTrace, it should implement `GcSafe` for all possible collectors. +/// However, if a type `NEEDS_TRACE`, it will usually only implement GcSafe for the specific +/// [CollectorId]s it happens to contain (although this is not guarenteed). +/// +/// ## Mixing with other lifetimes +/// Note that `T: GcSafe<'gc, T>` does *not* necessarily imply `T: 'gc`. +/// This allows a garbage collected lifetime to contain shorter lifetimes +/// +/// For example, +/// ``` +/// # use zerogc::epsilon::{Gc, EpsilonContext as GcContext, EpsilonCollectorId as CollectorId}; +/// # use zerogc::prelude::*; +/// #[derive(Trace)] +/// #[zerogc(ignore_lifetimes("'a"), collector_ids(CollectorId))] +/// struct TempLifetime<'gc, 'a> { +/// temp: &'a i32, +/// gc: Gc<'gc, i32> +/// } +/// fn alloc_ref_temp<'gc>(ctx: &'gc GcContext, long_lived: Gc<'gc, i32>) { +/// let temp = 5; // Lives for 'a (shorter than 'gc) +/// let temp_ref = ctx.alloc(TempLifetime { +/// temp: &temp, gc: long_lived +/// }); +/// assert_eq!(&temp as *const _, temp_ref.temp as *const _) +/// } +/// ``` +/// +/// ## Mixing collectors +/// The `Id` parameter allows mixing and matching pointers from different collectors, +/// each with their own 'gc lifetime. +/// For example, +/// ```ignore // TODO: Support this. See issue #33 +/// # use zerogc::{Gc, CollectorId, GcSafe}; +/// # use zerogc_derive::Trace; +/// # type JsGcId = zerogc::epsilon::EpsilonCollectorId; +/// # type OtherGcId = zerogc::epsilon::EpsilonCollectorId; +/// #[derive(Trace)] +/// struct MixedGc<'gc, 'js> { +/// internal_ptr: Gc<'gc, i32, OtherGcId>, +/// js_ptr: Gc<'js, i32, JsGcId> +/// } +/// impl<'gc, 'js> MixedGc<'gc, 'js> { +/// fn verify(&self) { +/// assert!(>::assert_gc_safe()); +/// assert!(>::assert_gc_safe()); +/// // NOT implemented: > +/// } +/// } +/// ``` +/// +/// ## Safety +/// In addition to the guarantees of [Trace] and [TrustedDrop], +/// implementing this type requires that all [Gc] pointers of +/// the specified `Id` have the `'gc` lifetime (if there are any at all). +pub unsafe trait GcSafe<'gc, Id: CollectorId>: Trace + TrustedDrop { + /// Assert this type is GC safe + /// + /// Only used by procedural derive + #[doc(hidden)] + fn assert_gc_safe() -> bool + where + Self: Sized, + { + true + } + /// Trace this object behind a [Gc] pointer. + /// + /// This is **required** to delegate to one of the following methods on [GcVisitor]: + /// 1. [GcVisitor::trace_gc] - For regular, `Sized` types + /// 2. [GcVisitor::trace_array] - For slices and arrays + /// 3. [GcVisitor::trace_trait_object] - For trait objects + /// + /// ## Safety + /// This must delegate to the appropriate method on [GcVisitor], + /// or undefined behavior will result. + /// + /// The user is required to supply an appropriate [Gc] pointer. + unsafe fn trace_inside_gc(gc: &mut Gc<'gc, Self, Id>, visitor: &mut V) -> Result<(), V::Err> + where + V: GcVisitor; +} + +/// A [GcSafe] type with all garbage-collected pointers +/// erased to the `'static` type. +/// +/// This should generally only be used by internal code. +/// +/// ## Safety +/// This type is incredibly unsafe. It eliminates all the safety invariants +/// of lifetimes. +pub trait GcSafeErased = GcSafe<'static, Id>; + +/// A wrapper type that assumes its contents don't need to be traced +#[repr(transparent)] +#[derive(Copy, Clone, Debug)] +pub struct AssumeNotTraced(T); +impl AssumeNotTraced { + /// Assume the specified value doesn't need to be traced + /// + /// ## Safety + /// Undefined behavior if the value contains anything that need to be traced + /// by a garbage collector. + #[inline] + pub unsafe fn new(value: T) -> Self { + AssumeNotTraced(value) + } + /// Unwrap the inner value of this wrapper + #[inline] + pub fn into_inner(self) -> T { + self.0 + } +} +impl Deref for AssumeNotTraced { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} +impl DerefMut for AssumeNotTraced { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} +unsafe_gc_impl! { + target => AssumeNotTraced, + params => [T], + bounds => { + // Unconditionally implement all traits + Trace => always, + TraceImmutable => always, + TrustedDrop => always, + GcSafe => always, + GcRebrand => always, + }, + null_trace => always, + branded_type => Self, + NEEDS_TRACE => false, + NEEDS_DROP => core::mem::needs_drop::(), + trace_template => |self, visitor| { /* nop */ Ok(()) } +} diff --git a/src/trace/barrier.rs b/src/trace/barrier.rs new file mode 100644 index 0000000..7419408 --- /dev/null +++ b/src/trace/barrier.rs @@ -0,0 +1,65 @@ +//! Traits controlling write barriers + +pub use crate::trace::{NullTrace, Trace}; + +/// Indicates that a mutable reference to a type +/// is safe to use without triggering a write barrier. +/// +/// This means one of either two things: +/// 1. This type doesn't need any write barriers +/// 2. Mutating this type implicitly triggers a write barrier. +/// +/// This is the bound for `RefCell`. Since a RefCell doesn't explicitly trigger write barriers, +/// a `RefCell` can only be used with `T` if either: +/// 1. `T` doesn't need any write barriers or +/// 2. `T` implicitly triggers write barriers on any mutation +pub unsafe trait ImplicitWriteBarrier {} +unsafe impl ImplicitWriteBarrier for T {} + +/// Safely trigger a write barrier before +/// writing to a garbage collected value. +/// +/// The value must be in managed memory, +/// a *direct* part of a garbage collected object. +/// Write barriers (and writes) must include a reference +/// to its owning object. +/// +/// ## Safety +/// It is undefined behavior to forget to trigger a write barrier. +/// +/// Field offsets are unchecked. They must refer to the correct +/// offset (in bytes). +/// +/// ### Indirection +/// This trait only support "direct" writes, +/// where the destination field is inline with the source object. +/// +/// For example it's correct to implement `GcDirectWrite for (A, B)`, +/// since since `A` is inline with the owning tuple. +/// +/// It is **incorrect** to implement `GcDirectWrite for Vec`, +/// since it `T` is indirectly referred to by the vector. +/// There's no "field offset" we can use to get from `*mut Vec` -> `*mut T`. +/// +/// The only exception to this rule is [Gc] itself. +/// GcRef can freely implement [GcDirectBarrier] for any (and all values), +/// even though it's just a pointer. +/// It's the final destination of all write barriers and is expected +/// to internally handle the indirection. +pub unsafe trait GcDirectBarrier<'gc, OwningRef>: Trace { + /// Trigger a write barrier, + /// before writing to one of the owning object's managed fields + /// + /// It is undefined behavior to mutate a garbage collected field + /// without inserting a write barrier before it. + /// + /// Generational, concurrent and incremental GCs need this to maintain + /// the tricolor invariant. + /// + /// ## Safety + /// The specified field offset must point to a valid field + /// in the source object. + /// + /// The type of this value must match the appropriate field + unsafe fn write_barrier(&self, owner: &OwningRef, field_offset: usize); +} diff --git a/src/trace/gcptr.rs b/src/trace/gcptr.rs new file mode 100644 index 0000000..1ac697e --- /dev/null +++ b/src/trace/gcptr.rs @@ -0,0 +1,293 @@ +//! Implements the [`Gc`]` smart pointer. + +use core::cmp::Ordering; +use core::fmt::{self, Debug, Display, Formatter}; +use core::hash::{Hash, Hasher}; +use core::marker::{PhantomData, Unsize}; +use core::ops::{CoerceUnsized, Deref}; +use core::ptr::NonNull; + +use crate::system::{CollectorId, HandleCollectorId}; +use crate::trace::barrier::GcDirectBarrier; +use crate::trace::{GcRebrand, GcSafe, GcVisitor, Trace, TrustedDrop}; + +/// A garbage collected pointer to a value. +/// +/// This is the equivalent of a garbage collected smart-pointer. +/// It's so smart, you can even coerce it to a reference bound to the lifetime of the `GarbageCollectorRef`. +/// However, all those references are invalidated by the borrow checker as soon as +/// your reference to the collector reaches a safepoint. +/// The objects can only survive garbage collection if they live in this smart-pointer. +/// +/// The smart pointer is simply a guarantee to the garbage collector +/// that this points to a garbage collected object with the correct header, +/// and not some arbitrary bits that you've decided to heap allocate. +/// +/// ## Safety +/// A `Gc` can be safely transmuted back and forth from its corresponding pointer. +/// +/// Unsafe code can rely on a pointer always dereferencing to the same value in between +/// safepoints. This is true even for copying/moving collectors. +/// +/// ## Lifetime +/// The borrow does *not* refer to the value `&'gc T`. +/// Instead, it refers to the *context* `&'gc Id::Context` +/// +/// This is necessary because `T` may have borrowed interior data +/// with a shorter lifetime `'a < 'gc`, making `&'gc T` invalid +/// (because that would imply 'gc: 'a, which is false). +/// +/// This ownership can be thought of in terms of the following (simpler) system. +/// ```no_run +/// # trait GcSafe{} +/// # use core::marker::PhantomData; +/// struct GcContext { +/// values: Vec> +/// } +/// struct Gc<'gc, T: GcSafe> { +/// index: usize, +/// marker: PhantomData, +/// ctx: &'gc GcContext +/// } +/// ``` +/// +/// In this system, safepoints can be thought of mutations +/// that remove dead values from the `Vec`. +/// +/// This ownership equivalency is also the justification for why +/// the `'gc` lifetime can be [covariant](https://doc.rust-lang.org/nomicon/subtyping.html#variance) +/// +/// The only difference is that the real `Gc` structure +/// uses pointers instead of indices. +#[repr(transparent)] +pub struct Gc<'gc, T: ?Sized, Id: CollectorId> { + /// The pointer to the garbage collected value. + /// + /// NOTE: The logical lifetime here is **not** `&'gc T` + /// See the comments on 'Lifetime' for details. + value: NonNull, + /// Marker struct used to statically identify the collector's type, + /// and indicate that 'gc is a logical reference the system. + /// + /// The runtime instance of this value can be + /// computed from the pointer itself: `NonNull` -> `&CollectorId` + collector_id: PhantomData<&'gc Id::System>, +} +impl<'gc, T: GcSafe<'gc, Id> + ?Sized, Id: CollectorId> Gc<'gc, T, Id> { + /// Create a GC pointer from a raw pointer + /// + /// ## Safety + /// Undefined behavior if the underlying pointer is not valid + /// and doesn't correspond to the appropriate id. + #[inline] + pub unsafe fn from_raw(value: NonNull) -> Self { + Gc { + collector_id: PhantomData, + value, + } + } + /// Create a [GcHandle] referencing this object, + /// allowing it to be used without a context + /// and referenced across safepoints. + /// + /// Requires that the collector [supports handles](`HandleCollectorId`) + #[inline] + pub fn create_handle(&self) -> Id::Handle + where + Id: HandleCollectorId, + T: GcRebrand<'static, Id>, + { + Id::create_handle(*self) + } + + /// Get a reference to the system + /// + /// ## Safety + /// This is based on the assumption that a [GcSystem] must outlive + /// all of the pointers it owns. + /// Although it could be restricted to the lifetime of the [CollectorId] + /// (in theory that may have an internal pointer) it will still live for '&self'. + #[inline] + pub fn system(&self) -> &'_ Id::System { + // This assumption is safe - see the docs + unsafe { self.collector_id().assume_valid_system() } + } +} +impl<'gc, T: ?Sized, Id: CollectorId> Gc<'gc, T, Id> { + /// The value of the underlying pointer + #[inline(always)] + pub const fn value(&self) -> &'gc T { + unsafe { *(&self.value as *const NonNull as *const &'gc T) } + } + /// Cast this reference to a raw pointer + /// + /// ## Safety + /// It's undefined behavior to mutate the + /// value. + /// The pointer is only valid as long as + /// the reference is. + #[inline] + pub unsafe fn as_raw_ptr(&self) -> *mut T { + self.value.as_ptr() as *const T as *mut T + } + + /// Get a reference to the collector's id + /// + /// The underlying collector it points to is not necessarily always valid + #[inline] + pub fn collector_id(&self) -> &'_ Id { + Id::from_gc_ptr(self) + } +} + +/// Double-indirection is completely safe +unsafe impl<'gc, T: ?Sized + GcSafe<'gc, Id>, Id: CollectorId> TrustedDrop for Gc<'gc, T, Id> {} +unsafe impl<'gc, T: ?Sized + GcSafe<'gc, Id>, Id: CollectorId> GcSafe<'gc, Id> for Gc<'gc, T, Id> { + #[inline] + unsafe fn trace_inside_gc(gc: &mut Gc<'gc, Self, Id>, visitor: &mut V) -> Result<(), V::Err> + where + V: GcVisitor, + { + // Double indirection is fine. It's just a `Sized` type + visitor.trace_gc(gc) + } +} +/// Rebrand +unsafe impl<'gc, 'new_gc, T, Id> GcRebrand<'new_gc, Id> for Gc<'gc, T, Id> +where + T: GcSafe<'gc, Id> + ?Sized + GcRebrand<'new_gc, Id>, + Id: CollectorId, + Self: Trace, +{ + type Branded = Gc<'new_gc, T::Branded, Id>; +} +unsafe impl<'gc, T: ?Sized + GcSafe<'gc, Id>, Id: CollectorId> Trace for Gc<'gc, T, Id> { + // We always need tracing.... + const NEEDS_TRACE: bool = true; + // we never need to be dropped because we are `Copy` + const NEEDS_DROP: bool = false; + + #[inline] + fn trace(&mut self, visitor: &mut V) -> Result<(), V::Err> { + unsafe { + // We're delegating with a valid pointer. + T::trace_inside_gc(self, visitor) + } + } +} +impl<'gc, T: GcSafe<'gc, Id> + ?Sized, Id: CollectorId> Deref for Gc<'gc, T, Id> { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.value() + } +} +unsafe impl<'gc, O, V, Id> GcDirectBarrier<'gc, Gc<'gc, O, Id>> for Gc<'gc, V, Id> +where + O: GcSafe<'gc, Id> + 'gc, + V: GcSafe<'gc, Id> + 'gc, + Id: CollectorId, +{ + #[inline(always)] + unsafe fn write_barrier(&self, owner: &Gc<'gc, O, Id>, field_offset: usize) { + Id::gc_write_barrier(owner, self, field_offset) + } +} +// We can be copied freely :) +impl<'gc, T: ?Sized, Id: CollectorId> Copy for Gc<'gc, T, Id> {} +impl<'gc, T: ?Sized, Id: CollectorId> Clone for Gc<'gc, T, Id> { + #[inline(always)] + fn clone(&self) -> Self { + *self + } +} +// Delegating impls +impl<'gc, T: GcSafe<'gc, Id> + Hash, Id: CollectorId> Hash for Gc<'gc, T, Id> { + #[inline] + fn hash(&self, state: &mut H) { + self.value().hash(state) + } +} +impl<'gc, T: GcSafe<'gc, Id> + PartialEq, Id: CollectorId> PartialEq for Gc<'gc, T, Id> { + #[inline] + fn eq(&self, other: &Self) -> bool { + // NOTE: We compare by value, not identity + self.value() == other.value() + } +} +impl<'gc, T: GcSafe<'gc, Id> + Eq, Id: CollectorId> Eq for Gc<'gc, T, Id> {} +impl<'gc, T: GcSafe<'gc, Id> + PartialEq, Id: CollectorId> PartialEq for Gc<'gc, T, Id> { + #[inline] + fn eq(&self, other: &T) -> bool { + self.value() == other + } +} +impl<'gc, T: GcSafe<'gc, Id> + PartialOrd, Id: CollectorId> PartialOrd for Gc<'gc, T, Id> { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + self.value().partial_cmp(other.value()) + } +} +impl<'gc, T: GcSafe<'gc, Id> + PartialOrd, Id: CollectorId> PartialOrd for Gc<'gc, T, Id> { + #[inline] + fn partial_cmp(&self, other: &T) -> Option { + self.value().partial_cmp(other) + } +} +impl<'gc, T: GcSafe<'gc, Id> + Ord, Id: CollectorId> Ord for Gc<'gc, T, Id> { + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + self.value().cmp(other) + } +} +impl<'gc, T: ?Sized + GcSafe<'gc, Id> + Debug, Id: CollectorId> Debug for Gc<'gc, T, Id> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + if !f.alternate() { + // Pretend we're a newtype by default + f.debug_tuple("Gc").field(&self.value()).finish() + } else { + // Alternate spec reveals `collector_id` + f.debug_struct("Gc") + .field("collector_id", &self.collector_id) + .field("value", &self.value()) + .finish() + } + } +} +impl<'gc, T: ?Sized + GcSafe<'gc, Id> + Display, Id: CollectorId> Display for Gc<'gc, T, Id> { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + Display::fmt(&self.value(), f) + } +} + +/// In order to send *references* between threads, +/// the underlying type must be sync. +/// +/// This is the same reason that `Arc: Send` requires `T: Sync` +unsafe impl<'gc, T, Id> Send for Gc<'gc, T, Id> +where + T: GcSafe<'gc, Id> + ?Sized + Sync, + Id: CollectorId + Sync, +{ +} + +/// If the underlying type is `Sync`, it's safe +/// to share garbage collected references between threads. +/// +/// The safety of the collector itself depends on whether [CollectorId] is Sync. +/// If it is, the whole garbage collection implementation should be as well. +unsafe impl<'gc, T, Id> Sync for Gc<'gc, T, Id> +where + T: GcSafe<'gc, Id> + ?Sized + Sync, + Id: CollectorId + Sync, +{ +} + +impl<'gc, T, U, Id> CoerceUnsized> for Gc<'gc, T, Id> +where + T: ?Sized + GcSafe<'gc, Id> + Unsize, + U: ?Sized + GcSafe<'gc, Id>, + Id: CollectorId, +{ +} From 896871b91eb28c4cd335e7347271727f2558e576 Mon Sep 17 00:00:00 2001 From: Techcable Date: Wed, 15 May 2024 01:42:03 -0700 Subject: [PATCH 26/35] Hide unstable features behind nightly flag Reduces some features for derive macros (specifically warnings), but is not really needed for the main crate. --- Cargo.toml | 2 + libs/derive/Cargo.toml | 3 ++ libs/derive/src/derive.rs | 19 +++++++--- libs/derive/src/lib.rs | 78 +++++++++++++++++++++++++++++++++------ libs/derive/src/macros.rs | 13 +++++-- src/lib.rs | 19 ++-------- src/trace.rs | 10 ----- src/trace/gcptr.rs | 11 +++++- 8 files changed, 107 insertions(+), 48 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9441f90..6a598fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,3 +62,5 @@ std = ["alloc"] # # This implements `Trace` for `Box` and collections like `Vec` alloc = [] +# Use nightly features +nightly = ["zerogc-derive/nightly"] \ No newline at end of file diff --git a/libs/derive/Cargo.toml b/libs/derive/Cargo.toml index 7f6be20..20aafb2 100644 --- a/libs/derive/Cargo.toml +++ b/libs/derive/Cargo.toml @@ -26,3 +26,6 @@ proc-macro-kwargs = "0.1.1" indexmap = "1" itertools = "0.10.1" + +[features] +nightly = [] \ No newline at end of file diff --git a/libs/derive/src/derive.rs b/libs/derive/src/derive.rs index 6610810..a7e2af3 100644 --- a/libs/derive/src/derive.rs +++ b/libs/derive/src/derive.rs @@ -12,7 +12,7 @@ use syn::{ Path, PathArguments, Type, TypeParam, TypePath, }; -use crate::{FromLitStr, MetaList}; +use crate::{FromLitStr, MetaList, WarningList}; type ExpandFunc<'a> = &'a mut dyn FnMut( TraceDeriveKind, @@ -79,9 +79,9 @@ impl TraceGenerics { .iter() .filter(|param| !param.ignore && !param.collector_id) } - fn normalize(&mut self) -> Result<(), Error> { + fn normalize(&mut self, warnings: &WarningList) -> Result<(), Error> { for tp in &mut self.type_params { - tp.normalize()?; + tp.normalize(warnings)?; } if let Some(ref gc) = self.gc_lifetime { if self.ignored_lifetimes.contains(gc) { @@ -158,7 +158,7 @@ pub struct TraceTypeParam { collector_id: bool, } impl TraceTypeParam { - fn normalize(&mut self) -> Result<(), Error> { + fn normalize(&mut self, warnings: &WarningList) -> Result<(), Error> { if self.ignore && self.collector_id { return Err(Error::custom( "Type parameter can't be ignored but also collector_id", @@ -166,6 +166,7 @@ impl TraceTypeParam { } if self.ident == "Id" && !self.collector_id { crate::emit_warning( + warnings, "Type parameter is named `Id` but isn't marked #[zerogc(collector_id)]. Specify collector_id = false if this is intentional.", self.ident.span() ) @@ -431,9 +432,14 @@ impl TraceDeriveInput { .cloned() .collect() } - pub fn normalize(&mut self, kind: TraceDeriveKind) -> Result<(), Error> { + pub fn normalize( + &mut self, + kind: TraceDeriveKind, + warnings: &WarningList, + ) -> Result<(), Error> { if *self.nop_trace { crate::emit_warning( + warnings, "#[zerogc(nop_trace)] is deprecated (use #[derive(NullTrace)] instead)", self.nop_trace.span(), ) @@ -475,7 +481,7 @@ impl TraceDeriveInput { tp.collector_id = true; } } - self.generics.normalize()?; + self.generics.normalize(warnings)?; if self.is_copy && self.unsafe_skip_drop { return Err(Error::custom( @@ -484,6 +490,7 @@ impl TraceDeriveInput { } if self.is_copy && matches!(kind, TraceDeriveKind::NullTrace) { crate::emit_warning( + warnings, "#[zerogc(copy)] is meaningless on NullTrace", self.ident.span(), ) diff --git a/libs/derive/src/lib.rs b/libs/derive/src/lib.rs index f1d6b82..b672397 100644 --- a/libs/derive/src/lib.rs +++ b/libs/derive/src/lib.rs @@ -1,14 +1,15 @@ -#![feature( +#![cfg_attr(feature = "nightly", feature( proc_macro_tracked_env, // Used for `DEBUG_DERIVE` proc_macro_span, // Used for source file ids proc_macro_diagnostic, // Used for warnings -)] +))] extern crate proc_macro; use crate::derive::TraceDeriveKind; use darling::{FromDeriveInput, FromMeta}; use proc_macro2::{Span, TokenStream}; -use quote::{quote, ToTokens}; +use quote::{quote, quote_spanned, ToTokens}; +use std::cell::RefCell; use std::fmt::Display; use std::io::Write; use syn::parse::Parse; @@ -77,10 +78,45 @@ pub(crate) fn sort_params(generics: &mut Generics) { generics.params = pairs.into_iter().collect(); } -pub(crate) fn emit_warning(msg: impl ToString, span: Span) { - let mut d = proc_macro::Diagnostic::new(proc_macro::Level::Warning, msg.to_string()); - d.set_spans(span.unwrap()); - d.emit(); +struct WarningList { + warnings: RefCell>, +} +impl WarningList { + pub fn new() -> Self { + WarningList { + warnings: RefCell::new(Vec::new()), + } + } +} + +impl ToTokens for WarningList { + fn to_tokens(&self, tokens: &mut TokenStream) { + let warnings = self.warnings.borrow(); + tokens.extend(warnings.iter().cloned()); + } + + fn into_token_stream(self) -> TokenStream + where + Self: Sized, + { + self.warnings.into_inner().into_iter().collect() + } +} + +pub(crate) fn emit_warning(list: &WarningList, msg: impl ToString, span: Span) { + #[cfg(feature = "nightly")] + { + let mut d = proc_macro::Diagnostic::new(proc_macro::Level::Warning, msg.to_string()); + d.set_spans(span.unwrap()); + d.emit(); + } + // Fallback for warnings on stable + if cfg!(not(feature = "nightly")) { + let text = msg.to_string(); + list.warnings.borrow_mut().push(quote_spanned! { span => + #[deprecated(note = #text)] + }); + } } pub(crate) fn move_bounds_to_where_clause(mut generics: Generics) -> Generics { @@ -203,9 +239,14 @@ fn impl_derive_trace( input: &DeriveInput, kind: TraceDeriveKind, ) -> Result { + let warnings = WarningList::new(); let mut input = derive::TraceDeriveInput::from_derive_input(input)?; - input.normalize(kind)?; - input.expand(kind) + input.normalize(kind, &warnings)?; + let first = input.expand(kind)?; + Ok(quote! { + #warnings + #first + }) } /// A list like `#[zerogc(a, b, c)] parsed as a `Punctuated`, @@ -243,6 +284,12 @@ pub(crate) fn is_explicitly_unsized(param: &syn::TypeParam) -> bool { }) } +#[cfg(not(feature = "nightly"))] +fn span_file_loc(_span: Span) -> String { + "Span(?)".into() +} + +#[cfg(feature = "nightly")] fn span_file_loc(span: Span) -> String { /* * Source file identifiers in the form `:` @@ -260,8 +307,17 @@ fn span_file_loc(span: Span) -> String { fn debug_derive(key: &str, target: &dyn ToString, message: &dyn Display, value: &dyn Display) { let target = target.to_string(); - // TODO: Use proc_macro::tracked_env::var - match ::proc_macro::tracked_env::var("DEBUG_DERIVE") { + let fetch_env_func: fn(&'static str) -> Result; + // TODO: Use proc_macro::tracked_env::var unconditionally + #[cfg(feature = "nightly")] + { + fetch_env_func = ::proc_macro::tracked_env::var; + } + #[cfg(not(feature = "nightly"))] + { + fetch_env_func = |s: &'static str| std::env::var(s); + } + match fetch_env_func("DEBUG_DERIVE") { Ok(ref var) if var == "*" || var == "1" || var.is_empty() => {} Ok(ref var) if var == "0" => { return; /* disabled */ diff --git a/libs/derive/src/macros.rs b/libs/derive/src/macros.rs index a40ac41..d8ece5a 100644 --- a/libs/derive/src/macros.rs +++ b/libs/derive/src/macros.rs @@ -16,7 +16,7 @@ use syn::{ PathArguments, PredicateType, Token, Type, TypeParamBound, WhereClause, WherePredicate, }; -use super::zerogc_crate; +use super::{zerogc_crate, WarningList}; use indexmap::{indexmap, IndexMap}; use quote::{quote, quote_spanned}; use syn::ext::IdentExt; @@ -202,6 +202,7 @@ impl MacroInput { generics } pub fn expand_output(&self) -> Result { + let warnings = WarningList::new(); let zerogc_crate = zerogc_crate(); let target_type = &self.target_type; let trace_impl = self.expand_trace_impl(true)?.expect("Trace impl required"); @@ -226,10 +227,11 @@ impl MacroInput { } else { quote!() }; - let deserialize_impl = self.expand_deserialize_impl()?; + let deserialize_impl = self.expand_deserialize_impl(&warnings)?; let rebrand_impl = self.expand_rebrand_impl()?; let trusted_drop = self.expand_trusted_drop_impl(); Ok(quote! { + #warnings #trace_impl #trace_immutable_impl #trusted_drop @@ -335,7 +337,10 @@ impl MacroInput { }) } - fn expand_deserialize_impl(&self) -> Result>, Error> { + fn expand_deserialize_impl( + &self, + warnings: &WarningList, + ) -> Result>, Error> { let zerogc_crate = zerogc_crate(); let target_type = &self.target_type; let strategy = match self.deserialize_strategy { @@ -369,7 +374,7 @@ impl MacroInput { }; match replaced_params.segments.last_mut().unwrap().arguments { PathArguments::None => { - crate::emit_warning(r##"The "horrible hack" isn't necessary if the type doesn't have generic args...."##, span); + crate::emit_warning(warnings, r##"The "horrible hack" isn't necessary if the type doesn't have generic args...."##, span); }, PathArguments::Parenthesized(_) => { return Err(Error::new( diff --git a/src/lib.rs b/src/lib.rs index d5cfbfe..652d76c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,19 +1,8 @@ -#![feature( - ptr_metadata, // RFC 2580 - Pointer meta - coerce_unsized, // RFC 0982 - DST coercion +// Unstable features +#![cfg_attr(feature = "nightly", feature( + coerce_unsized, // RFC 0982 - DST coercion unsize, - trait_alias, // RFC 1733 - Trait aliases - // Needed for epsilon collector: - negative_impls, // More elegant than marker types - alloc_layout_extra, - const_mut_refs, - const_option, - slice_range, // Convenient for bounds checking :) -)] -#![cfg_attr(feature = "error", backtrace)] -#![cfg_attr(feature = "allocator-api", feature(allocator_api))] -#![feature(maybe_uninit_slice)] -#![feature(new_uninit)] +))] #![deny(missing_docs)] #![allow( clippy::missing_safety_doc, // TODO: Add missing safety docs and make this #[deny(...)] diff --git a/src/trace.rs b/src/trace.rs index e1d6151..4dc24ff 100644 --- a/src/trace.rs +++ b/src/trace.rs @@ -310,16 +310,6 @@ pub unsafe trait GcSafe<'gc, Id: CollectorId>: Trace + TrustedDrop { V: GcVisitor; } -/// A [GcSafe] type with all garbage-collected pointers -/// erased to the `'static` type. -/// -/// This should generally only be used by internal code. -/// -/// ## Safety -/// This type is incredibly unsafe. It eliminates all the safety invariants -/// of lifetimes. -pub trait GcSafeErased = GcSafe<'static, Id>; - /// A wrapper type that assumes its contents don't need to be traced #[repr(transparent)] #[derive(Copy, Clone, Debug)] diff --git a/src/trace/gcptr.rs b/src/trace/gcptr.rs index 1ac697e..4427c1d 100644 --- a/src/trace/gcptr.rs +++ b/src/trace/gcptr.rs @@ -3,10 +3,16 @@ use core::cmp::Ordering; use core::fmt::{self, Debug, Display, Formatter}; use core::hash::{Hash, Hasher}; -use core::marker::{PhantomData, Unsize}; -use core::ops::{CoerceUnsized, Deref}; +use core::marker::PhantomData; +use core::ops::Deref; use core::ptr::NonNull; +// nightly feature: Unsized coercion +#[cfg(feature = "nightly")] +use std::marker::Unsize; +#[cfg(feature = "nightly")] +use std::ops::CoerceUnsized; + use crate::system::{CollectorId, HandleCollectorId}; use crate::trace::barrier::GcDirectBarrier; use crate::trace::{GcRebrand, GcSafe, GcVisitor, Trace, TrustedDrop}; @@ -284,6 +290,7 @@ where { } +#[cfg(feature = "nightly")] // nightly feature: CoerceUnsized impl<'gc, T, U, Id> CoerceUnsized> for Gc<'gc, T, Id> where T: ?Sized + GcSafe<'gc, Id> + Unsize, From 4bbbc41dea326d944ae9e68e4b0304d2c92daec3 Mon Sep 17 00:00:00 2001 From: Techcable Date: Wed, 15 May 2024 16:57:53 -0700 Subject: [PATCH 27/35] Remove GcDeserialize derive code Very unsafe --- libs/derive/src/derive.rs | 272 +------------------------------- libs/derive/src/lib.rs | 18 --- libs/derive/src/macros.rs | 157 +----------------- src/manually_traced/core.rs | 62 +------- src/manually_traced/mod.rs | 8 +- src/manually_traced/stdalloc.rs | 2 - 6 files changed, 8 insertions(+), 511 deletions(-) diff --git a/libs/derive/src/derive.rs b/libs/derive/src/derive.rs index a7e2af3..a9ae7bb 100644 --- a/libs/derive/src/derive.rs +++ b/libs/derive/src/derive.rs @@ -8,8 +8,8 @@ use quote::{format_ident, quote, quote_spanned, ToTokens}; use std::collections::HashSet; use syn::spanned::Spanned; use syn::{ - parse_quote, GenericArgument, GenericParam, Generics, Lifetime, LifetimeDef, LitStr, Meta, - Path, PathArguments, Type, TypeParam, TypePath, + parse_quote, GenericArgument, GenericParam, Generics, Lifetime, LifetimeDef, Meta, Path, + PathArguments, Type, TypeParam, TypePath, }; use crate::{FromLitStr, MetaList, WarningList}; @@ -25,7 +25,6 @@ type ExpandFunc<'a> = &'a mut dyn FnMut( pub enum TraceDeriveKind { NullTrace, Regular, - Deserialize, } trait PossiblyIgnoredParam { @@ -247,10 +246,6 @@ struct TraceField { /// Both options are completely safe. #[darling(default)] avoid_const_cycle: Option, - #[darling(default, rename = "serde")] - serde_opts: Option, - #[darling(forward_attrs(serde))] - attrs: Vec, } impl TraceField { fn expand_trace(&self, idx: usize, access: &FieldAccess, immutable: bool) -> TokenStream { @@ -278,8 +273,6 @@ impl TraceField { struct TraceVariant { ident: Ident, fields: darling::ast::Fields, - #[darling(forward_attrs(serde))] - attrs: Vec, } impl TraceVariant { fn fields(&self) -> impl Iterator + '_ { @@ -320,72 +313,6 @@ impl TraceVariant { } } -/// Custom `#[serde(bound(deserialize = ""))] -#[derive(Debug, Clone, FromMeta)] -struct CustomSerdeBounds { - /// The custom deserialize bound - deserialize: LitStr, -} - -/// Options for `#[zerogc(serde)]` on a type -#[derive(Debug, Clone, Default, FromMeta)] -struct SerdeTypeOpts { - /// Delegate directly to the `Deserialize` implementation, - /// without generating a wrapper. - /// - /// Effectively calls `zerogc::derive_delegating_deserialize!` - /// - /// Requires `Self: serde::Deserialize` - /// - /// If this is present, - /// then all other options are ignored. - #[darling(default)] - delegate: bool, - /// Override the inferred bounds - /// - /// Equivalent to `#[serde(bound(....))]` - #[darling(default, rename = "bound")] - custom_bounds: Option, - /// Require that Id::Context: GcSimpleAlloc - /// - /// This is necessary for the standard implementation - /// of `GcDeserialize for Gc` to apply. - /// - /// It is automatically inferred if you have any `Gc`, `GcArray` - /// or `GcString` fields (ignoring fully qualified paths). - #[darling(default)] - require_simple_alloc: Option, -} - -#[derive(Debug, Clone, Default, FromMeta)] -struct SerdeFieldOpts { - /// Delegate to the `serde::Deserialize` - /// implementation instead of using `GcDeserialize` - /// - /// If this option is present, - /// then all other options are ignored. - #[darling(default)] - delegate: bool, - /// Override the inferred bounds for the field. - #[darling(default, rename = "bound")] - custom_bounds: Option, - /// Deserialize this field using a custom - /// deserialization function. - /// - /// Equivalent to `#[serde(deserialize_with = "...")]` - #[darling(default)] - deserialize_with: Option, - /// Skip deserializing this field. - /// - /// Equivalent to `#[serde(skip_deserializing)]`. - /// - /// May choose to override the default with a - /// regular `#[serde(default = "...")]` - /// (but not with the #[zerogc(serde(...))])` syntax) - #[darling(default)] - skip_deserializing: bool, -} - #[derive(Debug, FromDeriveInput)] #[darling(attributes(zerogc))] pub struct TraceDeriveInput { @@ -412,10 +339,6 @@ pub struct TraceDeriveInput { /// If the type should implement `TraceImmutable` in addition to `Trace #[darling(default, rename = "immutable")] wants_immutable_trace: bool, - #[darling(default, rename = "serde")] - serde_opts: Option, - #[darling(forward_attrs(serde))] - attrs: Vec, } impl TraceDeriveInput { fn all_fields(&self) -> Vec<&TraceField> { @@ -541,7 +464,6 @@ impl TraceDeriveInput { TraceDeriveKind::NullTrace => { quote!(zerogc::NullTrace) } - TraceDeriveKind::Deserialize => unreachable!(), }; for tp in self.generics.regular_type_params() { let tp = &tp.ident; @@ -556,13 +478,11 @@ impl TraceDeriveInput { .make_where_clause() .predicates .push(parse_quote!(#tp: #requirement)), - TraceDeriveKind::Deserialize => unreachable!(), } } let assertion: Ident = match kind { TraceDeriveKind::NullTrace => parse_quote!(verify_null_trace), TraceDeriveKind::Regular => parse_quote!(assert_gc_safe), - TraceDeriveKind::Deserialize => unreachable!(), }; let ty_generics = self.generics.original.split_for_impl().1; let (impl_generics, _, where_clause) = generics.split_for_impl(); @@ -613,194 +533,9 @@ impl TraceDeriveInput { self.expand_gcsafe_sepcific(kind, initial, id, gc_lt) }, ), - TraceDeriveKind::Deserialize => unreachable!(), } } - fn expand_deserialize(&self) -> Result { - if !crate::DESERIALIZE_ENABLED { - return Err(Error::custom( - "The `zerogc/serde1` feature is disabled (please enable it)", - )); - } - let (gc_lifetime, generics) = self.generics_with_gc_lifetime(parse_quote!('gc)); - let default_should_require_simple_alloc = self.all_fields().iter().any(|field| { - let is_gc_allocated = match field.ty { - Type::Path(ref p) if p.path.segments.len() == 1 => { - /* - * If we exactly match 'Gc', 'GcArray' or 'GcString', - * then it can be assumed we are garbage collected - * and that we should require Id::Context: GcSimpleAlloc - */ - let name = &p.path.segments.last().unwrap().ident; - name == "Gc" || name == "GcArrray" || name == "GcString" - } - _ => false, - }; - if let Some(ref custom_opts) = field.serde_opts { - if custom_opts.delegate - || custom_opts.skip_deserializing - || custom_opts.custom_bounds.is_some() - || custom_opts.deserialize_with.is_some() - { - return false; - } - } - is_gc_allocated - }); - let should_require_simple_alloc = self - .serde_opts - .as_ref() - .and_then(|opts| opts.require_simple_alloc) - .unwrap_or(default_should_require_simple_alloc); - let do_require_simple_alloc = |id: &dyn ToTokens| quote!(<#id as zerogc::CollectorId>::Context: zerogc::GcSimpleAlloc); - self.expand_for_each_regular_id( - generics, TraceDeriveKind::Deserialize, gc_lifetime, - &mut |kind, initial, id, gc_lt| { - assert!(matches!(kind, TraceDeriveKind::Deserialize)); - let type_opts = self.serde_opts.clone().unwrap_or_default(); - let mut generics = initial.unwrap(); - let id_is_generic = generics.type_params() - .any(|param| id.is_ident(¶m.ident)); - generics.params.push(parse_quote!('deserialize)); - let requirement = quote!(for<'deser2> zerogc::serde::GcDeserialize::<#gc_lt, 'deser2, #id>); - if !type_opts.delegate { - for target in self.generics.regular_type_params() { - let target = &target.ident; - generics.make_where_clause().predicates.push(parse_quote!(#target: #requirement)); - } - if should_require_simple_alloc { - generics.make_where_clause().predicates.push( - syn::parse2(do_require_simple_alloc(&id)).unwrap() - ); - } - } - let ty_generics = self.generics.original.split_for_impl().1; - let (impl_generics, _, where_clause) = generics.split_for_impl(); - let target_type = &self.ident; - let forward_attrs = &self.attrs; - let deserialize_field = |f: &TraceField| { - let named = f.ident.as_ref().map(|name| quote!(#name: )); - let ty = &f.ty; - let forwarded_attrs = &f.attrs; - let serde_opts = f.serde_opts.clone().unwrap_or_default(); - let serde_attr = if serde_opts.delegate { - quote!() - } else { - let deserialize_with = serde_opts.deserialize_with.as_ref().map_or_else( - || String::from("deserialize_hack"), - |with| with.value() - ); - let custom_bound = if serde_opts.skip_deserializing || serde_opts.deserialize_with.is_some() { - quote!() - } else { - let bound = serde_opts.custom_bounds - .as_ref().map_or_else( - || format!( - "{}: for<'deserialize> zerogc::serde::GcDeserialize<{}, 'deserialize, {}>", ty.to_token_stream(), - gc_lt.to_token_stream(), id.to_token_stream() - ), - |bounds| bounds.deserialize.value() - ); - quote!(, bound(deserialize = #bound)) - }; - quote!(# [serde(deserialize_with = #deserialize_with #custom_bound)]) - }; - quote! { - #(#forwarded_attrs)* - #serde_attr - #named #ty - } - }; - let handle_fields = |fields: &darling::ast::Fields| { - let handled_fields = fields.fields.iter().map(deserialize_field); - match fields.style { - Style::Tuple => { - quote!{ ( #(#handled_fields),* ) } - } - Style::Struct => { - quote!({ #(#handled_fields),* }) - } - Style::Unit => quote!() - } - }; - let original_generics = &self.generics.original; - let inner = match self.data { - Data::Enum(ref variants) => { - let variants = variants.iter().map(|v| { - let forward_attrs = &v.attrs; - let name = &v.ident; - let inner = handle_fields(&v.fields); - quote! { - #(#forward_attrs)* - #name #inner - } - }); - quote!(enum HackRemoteDeserialize #original_generics { #(#variants),* }) - } - Data::Struct(ref f) => { - let fields = handle_fields(f); - quote!(struct HackRemoteDeserialize #original_generics # fields) - } - }; - let remote_name = target_type.to_token_stream().to_string(); - let id_decl = if id_is_generic { - Some(quote!(#id: zerogc::CollectorId,)) - } else { None }; - if !type_opts.delegate && !original_generics.lifetimes().any(|lt| lt.lifetime == *gc_lt) { - return Err(Error::custom("No 'gc lifetime found during #[derive(GcDeserialize)]. Consider #[zerogc(serde(delegate))] or a PhantomData.")) - } - if type_opts.delegate { - Ok(quote! { - impl #impl_generics zerogc::serde::GcDeserialize<#gc_lt, 'deserialize, #id> for #target_type #ty_generics #where_clause { - fn deserialize_gc>(_ctx: &#gc_lt <#id as zerogc::CollectorId>::Context, deserializer: D) -> Result { - >::deserialize(deserializer) - } - } - }) - } else { - let custom_bound = if let Some(ref bounds) = type_opts.custom_bounds { - let de_bounds = bounds.deserialize.value(); - quote!(, bound(deserialize = #de_bounds)) - } else if should_require_simple_alloc { - let de_bounds = format!("{}", do_require_simple_alloc(&id)); - quote!(, bound(deserialize = #de_bounds)) - } else { - quote!() - }; - let hack_where_bound = if should_require_simple_alloc && id_is_generic { - do_require_simple_alloc("e!(Id)) - } else { - quote!() - }; - Ok(quote! { - impl #impl_generics zerogc::serde::GcDeserialize<#gc_lt, 'deserialize, #id> for #target_type #ty_generics #where_clause { - fn deserialize_gc>(ctx: &#gc_lt <#id as zerogc::CollectorId>::Context, deserializer: D) -> Result { - use serde::Deserializer; - let _guard = unsafe { zerogc::serde::hack::set_context(ctx) }; - unsafe { - debug_assert_eq!(_guard.get_unchecked() as *const _, ctx as *const _); - } - /// Hack function to deserialize via `serde::hack`, with the appropriate `Id` type - /// - /// Needed because the actual function is unsafe - #[track_caller] - fn deserialize_hack<'gc, 'de, #id_decl D: serde::de::Deserializer<'de>, T: zerogc::serde::GcDeserialize<#gc_lt, 'de, #id>>(deser: D) -> Result - where #hack_where_bound { - unsafe { zerogc::serde::hack::unchecked_deserialize_hack::<'gc, 'de, D, #id, T>(deser) } - } - # [derive(serde::Deserialize)] - # [serde(remote = #remote_name #custom_bound)] - #(#forward_attrs)* - #inner ; - HackRemoteDeserialize::deserialize(deserializer) - } - } - }) - } - } - ) - } fn expand_for_each_regular_id( &self, generics: Generics, @@ -1336,9 +1071,6 @@ impl TraceDeriveInput { }) } pub fn expand(&self, kind: TraceDeriveKind) -> Result { - if matches!(kind, TraceDeriveKind::Deserialize) { - return self.expand_deserialize(); - } let gcsafe = self.expand_gcsafe(kind)?; let trace_immutable = if self.wants_immutable_trace { Some(self.expand_trace(kind, true)?) diff --git a/libs/derive/src/lib.rs b/libs/derive/src/lib.rs index b672397..2f49315 100644 --- a/libs/derive/src/lib.rs +++ b/libs/derive/src/lib.rs @@ -217,24 +217,6 @@ pub fn derive_trace(input: proc_macro::TokenStream) -> proc_macro::TokenStream { res } -pub(crate) const DESERIALIZE_ENABLED: bool = cfg!(feature = "__serde-internal"); - -#[proc_macro_derive(GcDeserialize, attributes(zerogc))] -pub fn gc_deserialize(input: proc_macro::TokenStream) -> proc_macro::TokenStream { - let input = parse_macro_input!(input as DeriveInput); - let res = From::from( - impl_derive_trace(&input, TraceDeriveKind::Deserialize) - .unwrap_or_else(|e| e.write_errors()), - ); - debug_derive( - "derive(GcDeserialize)", - &input.ident.to_string(), - &format_args!("#[derive(GcDeserialize) for {}", input.ident), - &res, - ); - res -} - fn impl_derive_trace( input: &DeriveInput, kind: TraceDeriveKind, diff --git a/libs/derive/src/macros.rs b/libs/derive/src/macros.rs index d8ece5a..a0ee57a 100644 --- a/libs/derive/src/macros.rs +++ b/libs/derive/src/macros.rs @@ -13,7 +13,7 @@ use syn::parse::ParseStream; use syn::spanned::Spanned; use syn::{ braced, parse_quote, Error, Expr, GenericArgument, GenericParam, Generics, Lifetime, Path, - PathArguments, PredicateType, Token, Type, TypeParamBound, WhereClause, WherePredicate, + PredicateType, Token, Type, TypeParamBound, WhereClause, WherePredicate, }; use super::{zerogc_crate, WarningList}; @@ -65,34 +65,6 @@ impl MacroArg for CollectorIdInfo { } } -#[derive(Debug)] -pub enum DeserializeStrategy { - UnstableHorribleHack(Span), - Delegate(Span), - ExplicitClosure(KnownArgClosure), -} -impl MacroArg for DeserializeStrategy { - fn parse_macro_arg(stream: ParseStream) -> syn::Result { - mod kw { - syn::custom_keyword!(unstable_horrible_hack); - syn::custom_keyword!(delegate); - } - if stream.peek(Token![|]) { - Ok(DeserializeStrategy::ExplicitClosure( - KnownArgClosure::parse_with_fixed_args(stream, &["ctx", "deserializer"])?, - )) - } else if stream.peek(kw::unstable_horrible_hack) { - let span = stream.parse::()?.span; - Ok(DeserializeStrategy::UnstableHorribleHack(span)) - } else if stream.peek(kw::delegate) { - let span = stream.parse::()?.span; - Ok(DeserializeStrategy::Delegate(span)) - } else { - Err(stream.error("Unknown deserialize strategy. Try specifying an explicit closure |ctx, deserializer| { }")) - } - } -} - #[derive(Debug, MacroKeywordArgs)] pub struct MacroInput { /// The target type we are implementing @@ -143,8 +115,6 @@ pub struct MacroInput { trace_mut_closure: Option, #[kwarg(optional, rename = "trace_immutable")] trace_immutable_closure: Option, - #[kwarg(optional, rename = "deserialize")] - deserialize_strategy: Option, } impl MacroInput { fn parse_visitor(&self) -> syn::Result { @@ -227,7 +197,6 @@ impl MacroInput { } else { quote!() }; - let deserialize_impl = self.expand_deserialize_impl(&warnings)?; let rebrand_impl = self.expand_rebrand_impl()?; let trusted_drop = self.expand_trusted_drop_impl(); Ok(quote! { @@ -238,7 +207,6 @@ impl MacroInput { #null_trace_impl #(#gcsafe_impl)* #(#rebrand_impl)* - #(#deserialize_impl)* }) } fn expand_trace_impl(&self, mutable: bool) -> Result, Error> { @@ -337,93 +305,6 @@ impl MacroInput { }) } - fn expand_deserialize_impl( - &self, - warnings: &WarningList, - ) -> Result>, Error> { - let zerogc_crate = zerogc_crate(); - let target_type = &self.target_type; - let strategy = match self.deserialize_strategy { - Some(ref strategy) => strategy, - _ => return Ok(Vec::new()), - }; - if !crate::DESERIALIZE_ENABLED { - return Ok(Vec::new()); - } - let de_lt = parse_quote!('deserialize); - self.for_each_id_type(self.basic_generics(), true, |mut generics, id_type, gc_lt| { - generics.params.push(parse_quote!('deserialize)); - generics.make_where_clause().predicates.extend( - match self.bounds.deserialize_clause(id_type, gc_lt, &de_lt, &self.params.elements) { - Some(clause) => clause.predicates, - None => return Ok(None) - } - ); - crate::sort_params(&mut generics); - let deserialize = match *strategy { - DeserializeStrategy::Delegate(_span) => { - // NOTE: quote_spanned messes up hygiene - quote! { { - >::deserialize(deserializer) - } } - }, - DeserializeStrategy::UnstableHorribleHack(span) => { - let mut replaced_params = match self.target_type { - Type::Path(syn::TypePath { qself: None, ref path }) => path.clone(), - _ => return Err(syn::Error::new(self.target_type.span(), r##"To use the "horrible hack" strategy for deserilaztion, the type must be a 'path' type (without a qualified self)"##)) - }; - match replaced_params.segments.last_mut().unwrap().arguments { - PathArguments::None => { - crate::emit_warning(warnings, r##"The "horrible hack" isn't necessary if the type doesn't have generic args...."##, span); - }, - PathArguments::Parenthesized(_) => { - return Err(Error::new( - self.target_type.span(), - r##"NYI: The "horrible hack" deserialization strategy doesn't support paranthesized generics"## - )) - }, - PathArguments::AngleBracketed(ref mut args) => { - for arg in args.args.iter_mut() { - match *arg { - GenericArgument::Type(ref mut tp) => { - *tp = parse_quote!(zerogc::serde::hack::DeserializeHackWrapper::<#tp, #id_type>) - }, - GenericArgument::Constraint(ref c) => { - return Err(syn::Error::new(c.span(), "NYI: Horrible hack support for 'constraints'")) - }, - _ => {} - } - } - } - } - quote_spanned! { span => - let hack = <#replaced_params as serde::de::Deserialize<'deserialize>>::deserialize(deserializer)?; - /* - * SAFETY: Should be safe to go from Vec -> Vec and HashMap -> HashMap - * as long as the reprs are transparent - * - * TODO: If this is safe, why does transmute not like Option> -> Option - */ - Ok(unsafe { zerogc::serde::hack::transmute_mismatched::<#replaced_params, Self>(hack) }) - } - }, - DeserializeStrategy::ExplicitClosure(ref closure) => { - let mut tokens = TokenStream::new(); - use quote::ToTokens; - closure.brace.surround(&mut tokens, |tokens| closure.body.to_tokens(tokens)); - tokens - } - }; - let (impl_generics, _, where_clause) = generics.split_for_impl(); - Ok(Some(quote! { - impl #impl_generics #zerogc_crate::serde::GcDeserialize<#gc_lt, #de_lt, #id_type> for #target_type #where_clause { - fn deserialize_gc>(ctx: &#gc_lt <#id_type as zerogc::CollectorId>::Context, deserializer: D) -> Result { - #deserialize - } - } - })) - }) - } fn for_each_id_type( &self, mut generics: Generics, @@ -592,7 +473,7 @@ impl MacroInput { #[derive(Debug, Clone)] pub struct KnownArgClosure { body: TokenStream, - brace: ::syn::token::Brace, + _brace: ::syn::token::Brace, } impl KnownArgClosure { pub fn parse_with_fixed_args(input: ParseStream, fixed_args: &[&str]) -> syn::Result { @@ -634,7 +515,7 @@ impl KnownArgClosure { let body = body.parse::()?; Ok(KnownArgClosure { body: quote!({ #body }), - brace, + _brace: brace, }) } } @@ -682,8 +563,6 @@ pub struct CustomBounds { /// The requirements to implement `TrustedDrop` #[kwarg(optional, rename = "TrustedDrop")] trusted_drop: Option, - #[kwarg(optional, rename = "GcDeserialize")] - deserialize: Option, #[kwarg(optional)] visit_inside_gc: Option>, } @@ -747,36 +626,6 @@ impl CustomBounds { } Ok(res) } - fn deserialize_clause( - &self, - id_type: &Path, - gc_lt: &Lifetime, - de_lt: &Lifetime, - generic_params: &[GenericParam], - ) -> Option { - let zerogc_crate = zerogc_crate(); - match self.deserialize { - Some(TraitRequirements::Never) => None, // skip this impl - Some(TraitRequirements::Always) => Some(empty_clause()), // No requirements - Some(TraitRequirements::Where(ref explicit)) => Some(explicit.clone()), - None => { - create_clause_with_default_and_ignored( - &self.trace_immutable, - generic_params, - vec![ - parse_quote!(#zerogc_crate::serde::GcDeserialize<#gc_lt, #de_lt, #id_type>), - ], - Some(&mut |param| { - /* - * HACK: Ignore `Id: GcSafe<'gc, Id>` bound because type inference is unable - * to resolve it. - */ - matches!(param, GenericParam::Type(ref tp) if Some(&tp.ident) == id_type.get_ident()) - }), - ) - } - } - } } fn create_clause_with_default( target: &Option, diff --git a/src/manually_traced/core.rs b/src/manually_traced/core.rs index 82599d1..cf7f281 100644 --- a/src/manually_traced/core.rs +++ b/src/manually_traced/core.rs @@ -16,14 +16,10 @@ macro_rules! trace_tuple { { $single_param:ident } => { trace_tuple_impl!(); trace_tuple_impl!($single_param); - #[cfg(feature = "serde1")] - deser_tuple_impl!($single_param); }; { $first_param:ident, $($param:ident),* } => { trace_tuple! { $($param),* } trace_tuple_impl!( $first_param, $($param),*); - #[cfg(feature = "serde1")] - deser_tuple_impl!($first_param, $($param),*); }; } @@ -107,60 +103,6 @@ macro_rules! trace_tuple_impl { }; } -#[cfg(feature = "serde1")] -macro_rules! deser_tuple_impl { - ( $($param:ident),+ ) => { - - impl<'gc, 'de, Id: $crate::CollectorId, $($param),*> $crate::serde::GcDeserialize<'gc, 'de, Id> for ($($param,)*) - where $($param: $crate::serde::GcDeserialize<'gc, 'de, Id>),* { - #[allow(non_snake_case, unused)] - fn deserialize_gc>( - ctx: &'gc ::Context, - deser: Deser - ) -> Result>::Error> { - use serde::de::{Visitor, Error, SeqAccess}; - use std::marker::PhantomData; - use $crate::{CollectorId, GcSystem}; - struct TupleVisitor<'gc, 'de, Id: $crate::CollectorId, $($param: $crate::serde::GcDeserialize<'gc, 'de, Id>),*> { - ctx: &'gc Id::Context, - marker: PhantomData<(&'de (), ( $($param,)*) )> - } - impl<'gc, 'de, Id: CollectorId, $($param: $crate::serde::GcDeserialize<'gc, 'de, Id>),*> - Visitor<'de> for TupleVisitor<'gc, 'de, Id, $($param),*> { - type Value = ($($param,)*); - fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - let mut count = 0; - $( - let $param = (); - count += 1; - )* - write!(f, "a tuple of len {}", count) - } - fn visit_seq>(self, mut seq: SeqAcc) -> Result { - let mut idx = 0; - $( - let $param = match seq.next_element_seed($crate::serde::GcDeserializeSeed::new(self.ctx))? { - Some(value) => value, - None => return Err(Error::invalid_length(idx, &self)) - }; - idx += 1; - )* - Ok(($($param,)*)) - } - } - let mut len = 0; - $( - let _hack = PhantomData::<$param>; - len += 1; - )* - deser.deserialize_tuple(len, TupleVisitor { - marker: PhantomData, ctx - }) - } - } - }; -} - unsafe_trace_primitive!(i8); unsafe_trace_primitive!(i16); unsafe_trace_primitive!(i32); @@ -176,7 +118,7 @@ unsafe_trace_primitive!(f64); unsafe_trace_primitive!(bool); unsafe_trace_primitive!(char); // TODO: Get proper support for unsized types (issue #15) -unsafe_trace_primitive!(&'static str; @); +unsafe_trace_primitive!(&'static str); unsafe_gc_impl! { target => PhantomData, @@ -348,7 +290,6 @@ unsafe_gc_impl! { Some(ref #mutability value) => visitor.#trace_func::(value), } }, - deserialize => unstable_horrible_hack, } unsafe impl<'gc, OwningRef, V> GcDirectBarrier<'gc, OwningRef> for Option where @@ -386,7 +327,6 @@ unsafe_gc_impl! { visitor.#trace_func(#b self.0) }, collector_id => *, - deserialize => unstable_horrible_hack, } #[cfg(test)] diff --git a/src/manually_traced/mod.rs b/src/manually_traced/mod.rs index 8dfe5b8..9578fc3 100644 --- a/src/manually_traced/mod.rs +++ b/src/manually_traced/mod.rs @@ -98,8 +98,7 @@ macro_rules! unsafe_trace_lock { /// (which are already undefined behavior for tracing). #[macro_export] macro_rules! unsafe_trace_primitive { - ($target:ty) => (unsafe_trace_primitive!($target; @ deserialize => delegate);); - ($target:ty; @ $(deserialize => $strategy:ident)?) => { + ($target:ty) => { unsafe_gc_impl! { target => $target, params => [], @@ -108,13 +107,10 @@ macro_rules! unsafe_trace_primitive { NEEDS_DROP => core::mem::needs_drop::<$target>(), collector_id => *, trace_template => |self, visitor| { /* nop */ Ok(()) }, - $(deserialize => $strategy)* } unsafe impl<'gc, OwningRef> $crate::GcDirectBarrier<'gc, OwningRef> for $target { #[inline(always)] - unsafe fn write_barrier( - &self, _owner: &OwningRef, _field_offset: usize, - ) { + unsafe fn write_barrier(&self, _owner: &OwningRef, _field_offset: usize) { /* * TODO: We don't have any GC fields, * so what does it mean to have a write barrier? diff --git a/src/manually_traced/stdalloc.rs b/src/manually_traced/stdalloc.rs index 5cf3f15..1e27f4c 100644 --- a/src/manually_traced/stdalloc.rs +++ b/src/manually_traced/stdalloc.rs @@ -23,7 +23,6 @@ unsafe_gc_impl! { // Delegate to slice visitor.#trace_func::<[T]>(#b**self as #b [T]) }, - deserialize => unstable_horrible_hack, } unsafe_gc_impl! { target => Box, @@ -35,7 +34,6 @@ unsafe_gc_impl! { trace_template => |self, visitor| { visitor.#trace_func::(#b **self) }, - deserialize => unstable_horrible_hack, } // We can only trace `Rc` and `Arc` if the inner type implements `TraceImmutable` unsafe_gc_impl! { From 80ee1d948c10feabf990044030ecc0479f2b430c Mon Sep 17 00:00:00 2001 From: Techcable Date: Wed, 15 May 2024 17:03:10 -0700 Subject: [PATCH 28/35] Remove GcHandle code --- src/prelude.rs | 2 +- src/system.rs | 75 +--------------------------------------------- src/trace/gcptr.rs | 15 +--------- 3 files changed, 3 insertions(+), 89 deletions(-) diff --git a/src/prelude.rs b/src/prelude.rs index 66e4298..b0afb2f 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -6,7 +6,7 @@ //! collected program needs to use the API. // Basic collector types -pub use crate::system::{GcHandle, GcSystem, HandleCollectorId}; +pub use crate::system::GcSystem; pub use crate::trace::{Gc, GcVisitor}; // Traits for user code to implement pub use crate::cell::GcCell; diff --git a/src/system.rs b/src/system.rs index f2d519b..18e113e 100644 --- a/src/system.rs +++ b/src/system.rs @@ -3,7 +3,7 @@ use core::fmt::Debug; use core::hash::Hash; -use crate::trace::{Gc, GcRebrand, GcSafe, NullTrace, TrustedDrop}; +use crate::trace::{Gc, GcSafe, NullTrace, TrustedDrop}; /// A garbage collector implementation, /// conforming to the zerogc API. @@ -12,30 +12,6 @@ pub unsafe trait GcSystem { type Id: CollectorId; } -/// A [CollectorId] that supports allocating [GcHandle]s -/// -/// Not all collectors necessarily support handles. -pub unsafe trait HandleCollectorId: CollectorId { - /// The type of [GcHandle] for this collector. - /// - /// This is parameterized by the *erased* type, - /// not by the original type. - type Handle: GcHandle - where - T: GcSafe<'static, Self> + ?Sized; - - /// Create a handle to the specified GC pointer, - /// which can be used without a context - /// - /// NOTE: Users should only use from [Gc::create_handle]. - /// - /// The system is implicit in the [Gc] - #[doc(hidden)] - fn create_handle<'gc, T>(gc: Gc<'gc, T, Self>) -> Self::Handle - where - T: GcSafe<'gc, Self> + GcRebrand<'static, Self> + ?Sized; -} - /// Uniquely identifies the collector in case there are /// multiple collectors. /// @@ -86,52 +62,3 @@ pub unsafe trait CollectorId: /// Undefined behavior if the associated collector no longer exists. unsafe fn assume_valid_system(&self) -> &Self::System; } - -/// A owned handle which points to a garbage collected object. -/// -/// This is considered a root by the garbage collector that is independent -/// of any specific [GcContext]. Safepoints -/// don't need to be informed of this object for collection to start. -/// The root is manually managed by user-code, much like a [Box] or -/// a reference counted pointer. -/// -/// This can be cloned and stored independently from a context, -/// bridging the gap between native memory and managed memory. -/// These are useful to pass to C APIs or any other code -/// that doesn't cooperate with zerogc. -/// -/// ## Tracing -/// The object behind this handle is already considered a root of the collection. -/// It should always be considered reachable by the garbage collector. -/// -/// Validity is tracked by this smart-pointer and not by tracing. -/// Therefore it is safe to implement [NullTrace] for handles. -/* - * TODO: Should we drop the Clone requirement? - */ -pub unsafe trait GcHandle + ?Sized>: - Sized + Clone + NullTrace + for<'gc> GcSafe<'gc, Self::Id> -{ - /// The type of the system used with this handle - type System: GcSystem; - /// The type of [CollectorId] used with this sytem - type Id: CollectorId; - - /// Access this handle inside the closure, - /// possibly associating it with the specified - /// - /// This is accesses the object within "critical section" - /// that will **block collections** - /// for as long as the closure is in use. - /// - /// These calls cannot be invoked recursively or they - /// may cause a deadlock. - /// - /// This is similar in purpose to JNI's [GetPrimitiveArrayCritical](https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#GetPrimitiveArrayCritical_ReleasePrimitiveArrayCritical). - /// However it never performs a copy, it is just guarenteed to block any collections. - /* - * TODO: Should we require this of all collectors? - * How much does it limit flexibility? - */ - fn use_critical(&self, func: impl FnOnce(&T) -> R) -> R; -} diff --git a/src/trace/gcptr.rs b/src/trace/gcptr.rs index 4427c1d..38018bc 100644 --- a/src/trace/gcptr.rs +++ b/src/trace/gcptr.rs @@ -13,7 +13,7 @@ use std::marker::Unsize; #[cfg(feature = "nightly")] use std::ops::CoerceUnsized; -use crate::system::{CollectorId, HandleCollectorId}; +use crate::system::CollectorId; use crate::trace::barrier::GcDirectBarrier; use crate::trace::{GcRebrand, GcSafe, GcVisitor, Trace, TrustedDrop}; @@ -92,19 +92,6 @@ impl<'gc, T: GcSafe<'gc, Id> + ?Sized, Id: CollectorId> Gc<'gc, T, Id> { value, } } - /// Create a [GcHandle] referencing this object, - /// allowing it to be used without a context - /// and referenced across safepoints. - /// - /// Requires that the collector [supports handles](`HandleCollectorId`) - #[inline] - pub fn create_handle(&self) -> Id::Handle - where - Id: HandleCollectorId, - T: GcRebrand<'static, Id>, - { - Id::create_handle(*self) - } /// Get a reference to the system /// From 78ad4dc04e459a181c9270a63b743de2c322e00c Mon Sep 17 00:00:00 2001 From: Techcable Date: Thu, 16 May 2024 17:39:27 -0700 Subject: [PATCH 29/35] Bump edition to 2021 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6a598fc..8480ad8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,7 @@ version = "0.2.0-alpha.7" authors = ["Techcable "] repository = "https://github.com/DuckLogic/zerogc" license = "MIT" -edition = "2018" +edition = "2021" [features] default = ["std"] From 89220060a2d93d451439bbd0deadee36fa5dc3fa Mon Sep 17 00:00:00 2001 From: Techcable Date: Fri, 17 May 2024 10:58:38 -0700 Subject: [PATCH 30/35] Update some dependency versions Remove unused dependencies. --- Cargo.toml | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8480ad8..2a053d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,30 +9,18 @@ edition.workspace = true readme = "README.md" [dependencies] -scopeguard = "1.1" +scopeguard = "1" inherent = "1" # Manually included tracing support for third party libraries # Providing support for these important libraries, # gives zerogc batteries included support. -indexmap = { version = "1.6", optional = true } -parking_lot = { version = "0.11", optional = true } +indexmap = { version = "2", optional = true } +parking_lot = { version = "0.12", optional = true } arrayvec = { version = "0.7", optional = true } anyhow = { version = "1", optional = true } -# Serde support (optional) -serde = { version = "1", optional = true, features = ["derive"] } +hashbrown = { version = "0.14.5", optional = true } # Used for macros zerogc-derive = { path = "libs/derive", version = "0.2.0-alpha.6" } -# Used for the "epsilon" no-op collector -bumpalo = { version = "3", optional = true } -# Used for our custom hashmap -ahash = { version = "0.7.0", default-features = false, optional = true } - -[dependencies.hashbrown] -# Hashbrown is used for our custom hashmap implementation -# We also implement Trace regardless -version = "0.11" -optional = true -features = ["raw", "nightly"] [dev-dependencies] serde_json = "1" From 0bda65c17de586727b740c2ba9e4db131839f8b0 Mon Sep 17 00:00:00 2001 From: Techcable Date: Sat, 25 May 2024 12:43:16 -0700 Subject: [PATCH 31/35] Fix usage of indexmap mutable keys in Trace Only offered with special extension trait `MutableKeys`. Drop unused `deserialize` option for macro (GcDeserialize removed). Drop unused dependencies. --- Cargo.toml | 2 -- src/manually_traced/arrayvec.rs | 43 --------------------------------- src/manually_traced/indexmap.rs | 2 +- 3 files changed, 1 insertion(+), 46 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2a053d7..b0a75e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,8 +9,6 @@ edition.workspace = true readme = "README.md" [dependencies] -scopeguard = "1" -inherent = "1" # Manually included tracing support for third party libraries # Providing support for these important libraries, # gives zerogc batteries included support. diff --git a/src/manually_traced/arrayvec.rs b/src/manually_traced/arrayvec.rs index 235c0a7..2f6b9a1 100644 --- a/src/manually_traced/arrayvec.rs +++ b/src/manually_traced/arrayvec.rs @@ -12,7 +12,6 @@ unsafe_gc_impl!( NEEDS_DROP => false, branded_type => ArrayString, trace_template => |self, visitor| { Ok(()) }, - deserialize => delegate ); unsafe_gc_impl!( @@ -31,46 +30,4 @@ unsafe_gc_impl!( } Ok(()) }, - deserialize => |ctx, deserializer| { - use core::marker::PhantomData; - use crate::CollectorId; - use crate::serde::{GcDeserialize, GcDeserializeSeed}; - use serde::de::{Visitor, Error, SeqAccess}; - struct ArrayVecVisitor< - 'gc, 'de, Id: CollectorId, - T: GcDeserialize<'gc, 'de, Id>, - const SIZE: usize - > { - ctx: &'gc Id::Context, - marker: PhantomData ArrayVec> - } - impl< - 'gc, 'de, Id: CollectorId, - T: GcDeserialize<'gc, 'de, Id>, - const SIZE: usize - > Visitor<'de> for ArrayVecVisitor<'gc, 'de, Id, T, SIZE> { - type Value = ArrayVec; - fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "a array with size <= {}", SIZE) - } - #[inline] - fn visit_seq(self, mut access: A) -> Result - where A: SeqAccess<'de>, { - let mut values = Self::Value::new(); - while let Some(value) = access.next_element_seed( - GcDeserializeSeed::new(self.ctx) - )? { - match values.try_push(value) { - Ok(()) => {}, - Err(_) => { - return Err(A::Error::invalid_length(SIZE + 1, &self)) - } - } - } - Ok(values) - } - } - let visitor: ArrayVecVisitor = ArrayVecVisitor { ctx, marker: PhantomData }; - deserializer.deserialize_seq(visitor) - } ); diff --git a/src/manually_traced/indexmap.rs b/src/manually_traced/indexmap.rs index c8d1727..66cfc9d 100644 --- a/src/manually_traced/indexmap.rs +++ b/src/manually_traced/indexmap.rs @@ -21,7 +21,7 @@ unsafe_gc_impl! { collector_id => *, trace_mut => |self, visitor| { for idx in 0..self.len() { - let (key, value) = self.get_index_mut(idx).unwrap(); + let (key, value) = indexmap::map::MutableKeys::get_index_mut2(self, idx).unwrap(); visitor.trace::(key)?; visitor.trace::(value)?; } From 4c6b2c8db77901ac2aafc3ed1a265127120ac0a9 Mon Sep 17 00:00:00 2001 From: Techcable Date: Sat, 25 May 2024 12:46:35 -0700 Subject: [PATCH 32/35] Add GcHeader trait & GcContext type The GcHeader api allows access to an object's internal header, which contains the CollectorId. For an array, the header also contains the length. Implmentations can (and will) add extra metadata to their header like mark bits and type info. Make methods on Gc associated functions instead of methods in order to avoid conflicts with the Deref trait. This is what Arc and Box both do. --- src/context.rs | 79 ++++++++++++++++ src/lib.rs | 6 ++ src/system.rs | 228 +++++++++++++++++++++++++++++++++++++++++++-- src/trace/gcptr.rs | 74 +++++++++------ 4 files changed, 352 insertions(+), 35 deletions(-) create mode 100644 src/context.rs diff --git a/src/context.rs b/src/context.rs new file mode 100644 index 0000000..33c893e --- /dev/null +++ b/src/context.rs @@ -0,0 +1,79 @@ +//! Defines the [`GcContext`] API, +//! +//! This is the primary safe interface with the collector. + +use crate::system::{GcContextState, TrustedAllocInit, UntrustedAllocInit}; +use crate::{CollectorId, Gc, GcSafe}; + +/// A single context for interfacing with a garbage collector. +/// +/// For any given collector, there should be only one of these per thread. +/// +/// ## Safepoint +/// A collection can only occur at a [`safepoint`](GcContext::safepoint). +/// This semantically "mutates" the context, +/// invalidating all outstanding [`Gc`] references. +pub struct GcContext { + state: Id::ContextState, +} +impl GcContext { + /// Return the id of the collector. + #[inline] + pub fn id(&self) -> Id { + self.state.id() + } + + /// Indicate that the thread is ready for a potential collection, + /// invaliding all outstanding [`Gc`] references. + /// + /// If other threads have active contexts, + /// this will need to wait for them before a collection can occur. + #[inline] + pub fn safepoint(&mut self) { + unsafe { + self.state.safepoint(); + } + } + + /// Trigger a [`safepoint`](Self::safepoint) and force a garbage collection. + /// + /// This will block until all threads reach a safepoint. + pub fn force_collect(&mut self) { + unsafe { + self.state.force_collect(); + } + } + + /// Allocate garbage collected memory with the specified value. + /// + /// In some cases [`Self::alloc_with`] could be more efficient, + /// since it can avoid moving a value. + #[inline(always)] + pub fn alloc<'gc, T>(&'gc self, value: T) -> Gc<'gc, T, Id> + where + T: GcSafe<'gc, Id>, + { + // SAFETY: Lifetime is correct + // SAFETY: Initialization will never fail. + unsafe { + self.state.alloc( + #[inline(always)] + || value, + TrustedAllocInit::new_unchecked(), + ) + } + } + + /// Allocate a garbage collected value, + /// initializing it using the specified callback. + /// + /// The initialization function is permitted to panic. + #[inline(always)] + pub fn alloc_with<'gc, T>(&'gc self, func: impl FnOnce() -> T) -> Gc<'gc, T, Id> + where + T: GcSafe<'gc, Id>, + { + // SAFETY: Lifetime is correct + unsafe { self.state.alloc(func, UntrustedAllocInit::new()) } + } +} diff --git a/src/lib.rs b/src/lib.rs index 652d76c..7c9a3ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,6 +42,7 @@ mod manually_traced; #[macro_use] mod macros; pub mod cell; +pub mod context; pub mod prelude; pub mod system; pub mod trace; @@ -56,3 +57,8 @@ pub use self::trace::*; /// Used by the derive code #[doc(hidden)] pub fn assert_copy() {} + +/// Internal module for sealing trait implementations. +pub(crate) mod sealed { + pub trait Sealed {} +} diff --git a/src/system.rs b/src/system.rs index 18e113e..162b123 100644 --- a/src/system.rs +++ b/src/system.rs @@ -1,5 +1,6 @@ //! Defines the [`GcSystem`] API for collector backends.d +use crate::sealed::Sealed; use core::fmt::Debug; use core::hash::Hash; @@ -7,35 +8,243 @@ use crate::trace::{Gc, GcSafe, NullTrace, TrustedDrop}; /// A garbage collector implementation, /// conforming to the zerogc API. +/// +/// ## Safety +/// For [`Gc`] references to be memory safe, +/// the implementation must be correct. pub unsafe trait GcSystem { - /// The type of collector IDs given by this system + /// The type of the collector id. + type Id: CollectorId; + + /// Return the collector's id. + fn id(&self) -> Self::Id; +} + +/// The collector-specific state of a [`GcContext`](crate::context::GcContext). +/// +/// Depending on whether the collector is multithreaded, +/// there may be one context per thread or a single global context. +/// +/// Owning a reference to the state prevents the [`GcSystem`] from being dropped. +/// +/// ## Safety +/// A context must always be bound to a single thread. +/// +/// This trait must be implemented correctly for the safety of other code. +pub unsafe trait GcContextState { + /// The type of the collector id. + type Id: CollectorId; + + /// Return the collector's id. + fn id(&self) -> Self::Id; + + /// Return a reference to the owning system. + fn system(&self) -> &'_ ::System; + + /// Indicate that . + unsafe fn safepoint(&mut self); + + /// Unconditionally force a garbage collection + unsafe fn force_collect(&mut self); + + /// Allocate a regular garbage collected object. + /// + /// Initializes the object using the specified + /// initialization function. + /// + /// ## Safety + /// The lifetime of the returned [`Gc`] pointer is not statically checked. + /// It is the caller's responsibility to ensure it is correct. + unsafe fn alloc<'gc, T: GcSafe<'gc, Self::Id>, S: AllocInitSafety>( + &self, + init: impl FnOnce() -> T, + safety: S, + ) -> Gc<'gc, T, Self::Id>; +} + +/// Whether an initialization function can be trusted to not panic. +/// +/// ## Safety +/// The members of this trait must be implemented correctly +/// or the collector might trace uninitialized values. +pub unsafe trait AllocInitSafety: crate::sealed::Sealed { + /// Whether the initialization function could possibly panic. + /// + /// If the function is trusted to never panic, + /// this can avoid the need to handle initialization failures. + /// + /// ## Safety + /// An incorrect implementation here can cause uninitialized + /// values to be observed by the trace function, + /// which is undefined behavior. + const MIGHT_PANIC: bool; +} + +/// An allocation initialization function that is trusted to never fail. +/// +/// ## Safety +/// If the initialization function panics, then undefined behavior will occur. +pub(crate) struct TrustedAllocInit { + _priv: (), +} +impl TrustedAllocInit { + /// Creates a marker indicating that the allocation is guaranteed to never fail. + /// + /// ## Safety + /// Undefined behavior if the corresponding initialization function ever fails. + #[inline(always)] + pub unsafe fn new_unchecked() -> Self { + TrustedAllocInit { _priv: () } + } +} +unsafe impl AllocInitSafety for TrustedAllocInit { + const MIGHT_PANIC: bool = false; +} +impl Sealed for TrustedAllocInit {} + +/// Indicates that the initialization function is untrusted, +/// and could possibly fail via panicking. +pub struct UntrustedAllocInit { + _priv: (), +} +impl UntrustedAllocInit { + /// Create a new marker value indicating the initialization function is untrusted, + /// and could possibly fail. + /// + /// This is completely safe, + /// since no assumptions are made. + #[inline(always)] + pub fn new() -> Self { + UntrustedAllocInit { _priv: () } + } +} +unsafe impl AllocInitSafety for UntrustedAllocInit { + const MIGHT_PANIC: bool = true; +} +impl Sealed for UntrustedAllocInit {} + +/// The header for a garbage-collected object. +/// +/// The specific type of object header may not be known, +/// and could be either [array headers](GcArrayHeader) and [regular headers](GcRegularHeader). +/// +/// ## Safety +/// Any access to an object's header is extremely unsafe. +/// +/// It is usually done behind a [NonNull] pointer. +pub unsafe trait GcHeader { + /// The id of the collector. type Id: CollectorId; + + /// Return a reference to the object's collector id. + fn id(&self) -> Self::Id; + + /// Determine the specific kind of header. + fn kind(&self) -> GcHeaderKind<'_, Self::Id>; +} + +/// Indicates the specific type of header, +/// which can be used for casts. +pub enum GcHeaderKind<'a, Id: CollectorId> { + /// An [array header](GcArrayHeader). + Array(&'a Id::ArrayHeader), + /// A [regular header](GcRegularHeader) + Regular(&'a Id::RegularHeader), +} +impl<'a, Id: CollectorId> GcHeaderKind<'a, Id> { + /// Attempt to cast this header into an [array header](GcArrayHeader), + /// returning `None` if it fails. + #[inline] + pub fn as_array(&self) -> Option<&'_ Id::ArrayHeader> { + match *self { + Self::Array(arr) => Some(arr), + _ => None, + } + } + + /// Attempt to cast this header into an [regular object header](GcRegularHeader), + /// returning `None` if it fails. + #[inline] + pub fn as_regular(&self) -> Option<&'_ Id::RegularHeader> { + match *self { + Self::Regular(header) => Some(header), + _ => None, + } + } + + /// Unsafely assume this header is a [regular object header](GcRegularHeader). + /// + /// ## Safety + /// Triggers undefined behavior if the cast is incorrect. + #[inline] + pub unsafe fn assume_regular(&self) -> &'_ Id::RegularHeader { + self.as_regular().unwrap_unchecked() + } + + /// Unsafely assume this header is a [array header](GcArrayHeader). + /// + /// ## Safety + /// Triggers undefined behavior if the cast is incorrect. + #[inline] + pub unsafe fn assume_array(&self) -> &'_ Id::ArrayHeader { + self.as_array().unwrap_unchecked() + } +} + +/// The header for a regular garbage-collected object. +pub unsafe trait GcRegularHeader: GcHeader { + /// Convert a reference to an object header into a typed [`Gc`] pointer. + /// + /// ## Safety + /// Undefined behavior if the type or lifetime is incorrect. + unsafe fn to_gcptr<'gc, T: GcSafe<'gc, Self::Id>>(&self) -> Gc<'gc, T, Self::Id>; +} + +/// The header for a garbage-collected array. +pub unsafe trait GcArrayHeader: GcHeader { + /// Return the length of the array, + /// in terms of the elements. + fn len(&self) -> usize; } /// Uniquely identifies the collector in case there are /// multiple collectors. /// +/// This can be seen as a typed pointer to a [`GcSystem`]. +/// /// ## Safety /// To simply the typing, this contains no references to the /// lifetime of the associated [GcSystem]. /// -/// It's implicitly held and is unsafe to access. +/// A reference to the system is implicitly held and is unsafe to access. /// As long as the collector is valid, /// this id should be too. /// /// It should be safe to assume that a collector exists -/// if any of its pointers still do! +/// if any of its [`Gc`] pointers still do! pub unsafe trait CollectorId: Copy + Eq + Hash + Debug + NullTrace + TrustedDrop + 'static + for<'gc> GcSafe<'gc, Self> { /// The type of the garbage collector system type System: GcSystem; + /// The [context-specific](`GcContext`) state for a collector. + type ContextState: GcContextState; + /// The header for a garbage-collected array. + type ArrayHeader: GcArrayHeader; + /// The header for a regulalar [`Gc`] object + type RegularHeader: GcRegularHeader; - /// Get the runtime id of the collector that allocated the [Gc] + /// Determine the [regular header](GcRegularHeader) for the specified [Gc] object. + /// + /// ## Safety + /// Undefined behavior if the header is used incorrectly. /// - /// Assumes that `T: GcSafe<'gc, Self>`, although that can't be - /// proven at compile time. - fn from_gc_ptr<'a, 'gc, T>(gc: &'a Gc<'gc, T, Self>) -> &'a Self + /// The header is normally guaranteed to live for the `'gc` lifetime. + /// However, during a collection, it may not be valid for quite as long, + /// so the lifetime of the returned header is tied to a transient borrow. + /// User code should not be able to access a GC pointer during a collection, + /// so only unsafe code needs to worry about this. + unsafe fn determine_header<'a, 'gc, T>(gc: &'a Gc<'gc, T, Self>) -> &'a Self::RegularHeader where T: ?Sized, 'gc: 'a; @@ -60,5 +269,8 @@ pub unsafe trait CollectorId: /// /// ## Safety /// Undefined behavior if the associated collector no longer exists. - unsafe fn assume_valid_system(&self) -> &Self::System; + /// + /// The lifetime of the returned system must be correct, + /// and not used after the collector no longer exists. + unsafe fn assume_valid_system<'a>(self) -> &'a Self::System; } diff --git a/src/trace/gcptr.rs b/src/trace/gcptr.rs index 38018bc..3ab5e53 100644 --- a/src/trace/gcptr.rs +++ b/src/trace/gcptr.rs @@ -9,11 +9,12 @@ use core::ptr::NonNull; // nightly feature: Unsized coercion #[cfg(feature = "nightly")] -use std::marker::Unsize; +use core::marker::Unsize; #[cfg(feature = "nightly")] -use std::ops::CoerceUnsized; +use core::ops::CoerceUnsized; -use crate::system::CollectorId; +use crate::context::GcContext; +use crate::system::{CollectorId, GcHeader}; use crate::trace::barrier::GcDirectBarrier; use crate::trace::{GcRebrand, GcSafe, GcVisitor, Trace, TrustedDrop}; @@ -73,11 +74,11 @@ pub struct Gc<'gc, T: ?Sized, Id: CollectorId> { /// See the comments on 'Lifetime' for details. value: NonNull, /// Marker struct used to statically identify the collector's type, - /// and indicate that 'gc is a logical reference the system. + /// and indicate that 'gc is a logical reference the context. /// /// The runtime instance of this value can be /// computed from the pointer itself: `NonNull` -> `&CollectorId` - collector_id: PhantomData<&'gc Id::System>, + collector_id: PhantomData<&'gc GcContext>, } impl<'gc, T: GcSafe<'gc, Id> + ?Sized, Id: CollectorId> Gc<'gc, T, Id> { /// Create a GC pointer from a raw pointer @@ -101,35 +102,54 @@ impl<'gc, T: GcSafe<'gc, Id> + ?Sized, Id: CollectorId> Gc<'gc, T, Id> { /// Although it could be restricted to the lifetime of the [CollectorId] /// (in theory that may have an internal pointer) it will still live for '&self'. #[inline] - pub fn system(&self) -> &'_ Id::System { + pub fn system(self: &Self) -> &'_ Id::System { // This assumption is safe - see the docs - unsafe { self.collector_id().assume_valid_system() } + unsafe { Gc::id(*self).assume_valid_system() } } } impl<'gc, T: ?Sized, Id: CollectorId> Gc<'gc, T, Id> { /// The value of the underlying pointer #[inline(always)] - pub const fn value(&self) -> &'gc T { - unsafe { *(&self.value as *const NonNull as *const &'gc T) } + pub const fn value(this: Self) -> &'gc T { + unsafe { this.value.as_ref() } } + /// Cast this reference to a raw pointer /// /// ## Safety - /// It's undefined behavior to mutate the - /// value. + /// It's undefined behavior to mutate the value. /// The pointer is only valid as long as /// the reference is. + /// + /// Moving collectors may change the pointer during a collection. + #[inline(always)] + pub unsafe fn raw_ptr(this: Self) -> NonNull { + this.value + } + + /// Get a reference to the object's header. + /// + /// This is a ["regular" header](GcRegularHeader), not an array header. + /// + /// ## Safety + /// It is undefined behavior to use the header incorrectly. + /// + /// As described in [`CollectorId::determine_header`], + /// the header may not be valid for the full `'gc` lifetime + /// if a collection is in progress. + /// Normally, user code can't be called during a collection + /// and does not need to handle this scenario. #[inline] - pub unsafe fn as_raw_ptr(&self) -> *mut T { - self.value.as_ptr() as *const T as *mut T + pub unsafe fn header(this: &Self) -> &'_ Id::RegularHeader { + Id::determine_header(this) } /// Get a reference to the collector's id /// - /// The underlying collector it points to is not necessarily always valid + /// The underlying collector it points to is not necessarily always valid. #[inline] - pub fn collector_id(&self) -> &'_ Id { - Id::from_gc_ptr(self) + pub fn id(this: Self) -> Id { + unsafe { Gc::header(&this).id() } } } @@ -173,7 +193,7 @@ impl<'gc, T: GcSafe<'gc, Id> + ?Sized, Id: CollectorId> Deref for Gc<'gc, T, Id> #[inline(always)] fn deref(&self) -> &Self::Target { - self.value() + unsafe { &*self.value.as_ptr() } } } unsafe impl<'gc, O, V, Id> GcDirectBarrier<'gc, Gc<'gc, O, Id>> for Gc<'gc, V, Id> @@ -199,58 +219,58 @@ impl<'gc, T: ?Sized, Id: CollectorId> Clone for Gc<'gc, T, Id> { impl<'gc, T: GcSafe<'gc, Id> + Hash, Id: CollectorId> Hash for Gc<'gc, T, Id> { #[inline] fn hash(&self, state: &mut H) { - self.value().hash(state) + Gc::value(*self).hash(state) } } impl<'gc, T: GcSafe<'gc, Id> + PartialEq, Id: CollectorId> PartialEq for Gc<'gc, T, Id> { #[inline] fn eq(&self, other: &Self) -> bool { // NOTE: We compare by value, not identity - self.value() == other.value() + Gc::value(*self) == Gc::value(*other) } } impl<'gc, T: GcSafe<'gc, Id> + Eq, Id: CollectorId> Eq for Gc<'gc, T, Id> {} impl<'gc, T: GcSafe<'gc, Id> + PartialEq, Id: CollectorId> PartialEq for Gc<'gc, T, Id> { #[inline] fn eq(&self, other: &T) -> bool { - self.value() == other + Gc::value(*self) == other } } impl<'gc, T: GcSafe<'gc, Id> + PartialOrd, Id: CollectorId> PartialOrd for Gc<'gc, T, Id> { #[inline] fn partial_cmp(&self, other: &Self) -> Option { - self.value().partial_cmp(other.value()) + Gc::value(*self).partial_cmp(Gc::value(*other)) } } impl<'gc, T: GcSafe<'gc, Id> + PartialOrd, Id: CollectorId> PartialOrd for Gc<'gc, T, Id> { #[inline] fn partial_cmp(&self, other: &T) -> Option { - self.value().partial_cmp(other) + Gc::value(*self).partial_cmp(other) } } impl<'gc, T: GcSafe<'gc, Id> + Ord, Id: CollectorId> Ord for Gc<'gc, T, Id> { #[inline] fn cmp(&self, other: &Self) -> Ordering { - self.value().cmp(other) + Gc::value(*self).cmp(other) } } impl<'gc, T: ?Sized + GcSafe<'gc, Id> + Debug, Id: CollectorId> Debug for Gc<'gc, T, Id> { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { if !f.alternate() { // Pretend we're a newtype by default - f.debug_tuple("Gc").field(&self.value()).finish() + f.debug_tuple("Gc").field(&Gc::value(*self)).finish() } else { // Alternate spec reveals `collector_id` f.debug_struct("Gc") - .field("collector_id", &self.collector_id) - .field("value", &self.value()) + .field("collector_id", &Gc::id(*self)) + .field("value", &Gc::value(*self)) .finish() } } } impl<'gc, T: ?Sized + GcSafe<'gc, Id> + Display, Id: CollectorId> Display for Gc<'gc, T, Id> { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - Display::fmt(&self.value(), f) + Display::fmt(Gc::value(*self), f) } } From 443e626c7e5a2f5727dfe3df7eb05451e32d254a Mon Sep 17 00:00:00 2001 From: Techcable Date: Tue, 28 May 2024 14:30:29 -0700 Subject: [PATCH 33/35] Fix clippy lints --- src/system.rs | 6 ++++++ src/trace/gcptr.rs | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/system.rs b/src/system.rs index 162b123..fb6a32d 100644 --- a/src/system.rs +++ b/src/system.rs @@ -114,6 +114,7 @@ impl UntrustedAllocInit { /// This is completely safe, /// since no assumptions are made. #[inline(always)] + #[allow(clippy::new_without_default)] pub fn new() -> Self { UntrustedAllocInit { _priv: () } } @@ -205,6 +206,11 @@ pub unsafe trait GcArrayHeader: GcHeader { /// Return the length of the array, /// in terms of the elements. fn len(&self) -> usize; + /// Check if the array is empty. + #[inline] + fn is_empty(&self) -> bool { + self.len() == 0 + } } /// Uniquely identifies the collector in case there are diff --git a/src/trace/gcptr.rs b/src/trace/gcptr.rs index 3ab5e53..6458a50 100644 --- a/src/trace/gcptr.rs +++ b/src/trace/gcptr.rs @@ -98,13 +98,13 @@ impl<'gc, T: GcSafe<'gc, Id> + ?Sized, Id: CollectorId> Gc<'gc, T, Id> { /// /// ## Safety /// This is based on the assumption that a [GcSystem] must outlive - /// all of the pointers it owns. + /// all the pointers it owns. /// Although it could be restricted to the lifetime of the [CollectorId] /// (in theory that may have an internal pointer) it will still live for '&self'. #[inline] - pub fn system(self: &Self) -> &'_ Id::System { + pub fn system(this: &Self) -> &'_ Id::System { // This assumption is safe - see the docs - unsafe { Gc::id(*self).assume_valid_system() } + unsafe { Gc::id(*this).assume_valid_system() } } } impl<'gc, T: ?Sized, Id: CollectorId> Gc<'gc, T, Id> { From 6672aaa4d0187f7b5517d1914adcb7db3a4499ad Mon Sep 17 00:00:00 2001 From: Techcable Date: Thu, 20 Jun 2024 18:41:14 -0700 Subject: [PATCH 34/35] More work on GcHandle trait and GcContext --- Cargo.toml | 26 ++++++-- src/context.rs | 54 ++++++++++++++++ src/context/handle.rs | 19 ++++++ src/lib.rs | 1 + src/prelude.rs | 1 + src/system.rs | 54 ++++++++++++++-- src/system/threads.rs | 142 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 288 insertions(+), 9 deletions(-) create mode 100644 src/context/handle.rs create mode 100644 src/system/threads.rs diff --git a/Cargo.toml b/Cargo.toml index b0a75e7..519cdfa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,12 @@ edition.workspace = true readme = "README.md" [dependencies] +# NOTE: Require an implementation of the abort that doesn't panic +# +# This can currently be done by enabling `std` or `libc` +libabort = { version = "0.1.7", default-features = false, features = ["always-immediate-abort"] } +# This provides a fallback `Allocator` API for stable rust. +allocator-api2 = { version = "0.2.18", default-features = false, features = ["alloc"] } # Manually included tracing support for third party libraries # Providing support for these important libraries, # gives zerogc batteries included support. @@ -19,6 +25,8 @@ anyhow = { version = "1", optional = true } hashbrown = { version = "0.14.5", optional = true } # Used for macros zerogc-derive = { path = "libs/derive", version = "0.2.0-alpha.6" } +# Static assertions +static_assertions = "1.1" [dev-dependencies] serde_json = "1" @@ -42,11 +50,19 @@ default = ["std"] # Depend on the standard library (optional) # # This implements tracing for most standard library types. -std = ["alloc"] -# Depend on `extern crate alloc` in addition to the Rust `core` -# This is implied by using the standard library (feature="std") +std = ["alloc", "allocator-api2/std", "libabort/std"] +# Implement `Trace` for types in `extern crate alloc` in addition to the Rust `core`. +# This supports types like `alloc::box::Box` and `alloc::vec::Vec` # -# This implements `Trace` for `Box` and collections like `Vec` +# This is implied by using the standard library (feature="std") +# Regardless of the presence or absence of this feature, +# the implementation currently relies on the `alloc` crate (but this may change in the future). alloc = [] +# Enables support for thread-safe collectors. +# +# This requires the `std` feature to be enabled. +sync = [] # Use nightly features -nightly = ["zerogc-derive/nightly"] \ No newline at end of file +nightly = ["zerogc-derive/nightly"] +# Use the nightly `Allocator` api +nightly-allocator = ["allocator-api2/nightly"] \ No newline at end of file diff --git a/src/context.rs b/src/context.rs index 33c893e..f969f61 100644 --- a/src/context.rs +++ b/src/context.rs @@ -5,6 +5,8 @@ use crate::system::{GcContextState, TrustedAllocInit, UntrustedAllocInit}; use crate::{CollectorId, Gc, GcSafe}; +pub mod handle; + /// A single context for interfacing with a garbage collector. /// /// For any given collector, there should be only one of these per thread. @@ -35,6 +37,58 @@ impl GcContext { } } + /// Test if a safepoint needs to be invoked. + /// + /// This should return true if [`GcContext::is_gc_needed`] does, + /// but also should return true if other threads are + /// already blocked at a safepoint. + /// + /// In a single-threaded scenario, the two tests should be equivalent. + /// See docs for [`GcContext::is_gc_needed`] for why the two tests are seperate. + /// + /// This can be used to avoid the overhead of mutation. + /// For example, consider the following code: + /// ``` + /// # use zerogc::prelude::*; + /// + /// fn run(ctx: &GcContext) { + /// let mut index = 0; + /// const LIMIT: usize = 100_000; + /// let total = + /// while index < LIMIT { + /// { + /// let cached = expensive_pure_computation(ctx); + /// while index < LIMIT && !ctx.is_safepoint_needed() { + /// index % cached.len() + /// } + /// } + /// // a safepoint was deemed necessary, which would invalidate the `cached` object. + /// } + /// } + /// + /// fn expensive_pure_computation(ctx: &GcContext) -> Gc<'_, String, Id> + /// # { unreachable!() } + /// ``` + #[inline] + pub fn is_safepoint_needed(&self) -> bool { + self.state.is_safepoint_needed() + } + + /// Test if a garbage collection is needed. + /// + /// This is a weaker test than [`GcContext::is_safepoint_needed`], + /// which this does not check if other threads are blocked. + /// + /// This function is distinct from [`GcContext::is_safepoint_needed`] because user code + /// may already have ways to atomically signal other executing threads. + /// In that case, there is no need to check if other threads are blocked. + /// + /// In a single-threaded context, the two tests are equivalent. + #[inline] + pub fn is_gc_needed(&self) -> bool { + self.state.is_gc_needed() + } + /// Trigger a [`safepoint`](Self::safepoint) and force a garbage collection. /// /// This will block until all threads reach a safepoint. diff --git a/src/context/handle.rs b/src/context/handle.rs new file mode 100644 index 0000000..6b690dc --- /dev/null +++ b/src/context/handle.rs @@ -0,0 +1,19 @@ +//! Implements the [`GcHandle`] type. + +use crate::{CollectorId, GcSafe}; +use core::ptr::NonNull; + +type TODO = std::convert::Infailable; + +struct GcHandleBox, Id: CollectorId> { + value: T, + refcnt: TODO, // todo +} + +/// A thread-local `GcHandle`. +/// +/// This can be faster than a standard [`GcHandle`], +/// because it is `!Send`. +pub struct GcLocalHandle, Id: CollectorId> { + refcnt: TODO, +} diff --git a/src/lib.rs b/src/lib.rs index 7c9a3ac..0590575 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,7 @@ clippy::missing_safety_doc, // TODO: Add missing safety docs and make this #[deny(...)] )] #![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(feature = "nightly-allocator", feature(allocator_api))] //! Zero overhead tracing garbage collection for rust, //! by abusing the borrow checker. //! diff --git a/src/prelude.rs b/src/prelude.rs index b0afb2f..4f31ee8 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -6,6 +6,7 @@ //! collected program needs to use the API. // Basic collector types +pub use crate::context::GcContext; pub use crate::system::GcSystem; pub use crate::trace::{Gc, GcVisitor}; // Traits for user code to implement diff --git a/src/system.rs b/src/system.rs index fb6a32d..444bfff 100644 --- a/src/system.rs +++ b/src/system.rs @@ -1,11 +1,51 @@ //! Defines the [`GcSystem`] API for collector backends.d -use crate::sealed::Sealed; use core::fmt::Debug; use core::hash::Hash; +use core::marker::PhantomData; use crate::trace::{Gc, GcSafe, NullTrace, TrustedDrop}; +pub(crate) mod threads; + +/// Indicates whether a collector is thread-safe. +/// +/// Most of the functions and types in this trait are logically private. +/// Accessing them is not subject to semver guarantees. +pub unsafe trait GcThreadSafety: + threads::ThreadSafetyInternal + crate::sealed::Sealed +{ +} + +/// A collector which is not thread-safe. This is `!Send` and `!Sync`. +pub struct NotSyncCollector { + /// Indicates `!Send` and `!Sync` + marker: PhantomData<*const ()>, +} + +unsafe impl GcThreadSafety for NotSyncCollector {} +unsafe impl threads::ThreadSafetyInternal for NotSyncCollector { + type ReferenceCounter = self::threads::NotSyncReferenceCounter; +} +impl crate::sealed::Sealed for NotSyncCollector {} + +static_assertions::assert_not_impl_any!(NotSyncCollector: Send, Sync); + +/// A collector which is thread-safe. +/// +/// This is both `Send` and `Sync`. +pub struct SyncCollector { + _priv: (), +} + +unsafe impl GcThreadSafety for SyncCollector {} +unsafe impl threads::ThreadSafetyInternal for SyncCollector { + type ReferenceCounter = self::threads::AtomicReferenceCounter; +} +impl crate::sealed::Sealed for SyncCollector {} + +static_assertions::assert_impl_all!(SyncCollector: Send, Sync); + /// A garbage collector implementation, /// conforming to the zerogc API. /// @@ -41,9 +81,15 @@ pub unsafe trait GcContextState { /// Return a reference to the owning system. fn system(&self) -> &'_ ::System; - /// Indicate that . + /// Indicate that the collector has reached a safepoint. unsafe fn safepoint(&mut self); + /// Check if a safepoint is needed. + fn is_safepoint_needed(&self) -> bool; + + /// Check if a gc is needed. + fn is_gc_needed(&self) -> bool; + /// Unconditionally force a garbage collection unsafe fn force_collect(&mut self); @@ -100,7 +146,7 @@ impl TrustedAllocInit { unsafe impl AllocInitSafety for TrustedAllocInit { const MIGHT_PANIC: bool = false; } -impl Sealed for TrustedAllocInit {} +impl crate::sealed::Sealed for TrustedAllocInit {} /// Indicates that the initialization function is untrusted, /// and could possibly fail via panicking. @@ -122,7 +168,7 @@ impl UntrustedAllocInit { unsafe impl AllocInitSafety for UntrustedAllocInit { const MIGHT_PANIC: bool = true; } -impl Sealed for UntrustedAllocInit {} +impl crate::sealed::Sealed for UntrustedAllocInit {} /// The header for a garbage-collected object. /// diff --git a/src/system/threads.rs b/src/system/threads.rs new file mode 100644 index 0000000..064b2e7 --- /dev/null +++ b/src/system/threads.rs @@ -0,0 +1,142 @@ +//! Internal code to abstract over the thread-safety of a collector. + +use core::cell::Cell; +use core::num::NonZeroUsize; +use std::sync::Arc; + +/// Internal abstraction over the thread-safety of a collector. +/// +/// Publicly exposed via [`zerogc::system::GcThreadSafety`]. +/// +/// ## Safety +/// The correctness of this abstraction is relied upon for thread-safety. +pub unsafe trait ThreadSafetyInternal: crate::sealed::Sealed { + /// Potentially thread-safe reference counting. + type ReferenceCounter: ReferenceCounter; +} + +/// A potentially thread-safe reference count. +/// +/// ## Safety +/// The correctness of the reference-count is relied upon for memory safety. +pub unsafe trait ReferenceCounter: crate::sealed::Sealed { + /// The maximum reference count. + const MAX_COUNT: usize; + /// Increment the reference count by one. + /// + /// For memory safety reasons, + /// this method will implicitly abort on overflow + /// just like standard-library reference counts. + /// + /// ## Safety + /// Not technically unsafe, but incorrectly incrementing this + /// could trigger memory leaks or aborts. + /// + /// Undefined behavior to incref a dead object. + unsafe fn incref(&self) -> RefCountModification; + /// Decrement the reference count by one. + /// + /// ## Safety + /// Incorrectly decrementing the reference count + /// may result in use-after-free. + /// + /// Undefined behavior to incref an already dead object. + /// Undefined behavior on underflow. + unsafe fn decref(&self) -> RefCountModification; +} + +pub struct RefCountModification { + /// The previous reference count. + /// + /// Guaranteed to be nonzero, + /// since refcount operations can not be performed on dead objects. + pub old_count: NonZeroUsize, +} +impl RefCountModification { + #[inline] + pub unsafe fn handle_incref(old_count: usize) -> Self { + /* + * Overflow is possible due to `std::mem::forget`. + * If encountered, abort the program. + * This is the same behavior as the stdlib. + */ + if old_count >= Self::MAX_COUNT { + libabort::abort(); // TODO: Switch to `trap` which is faster? + } + debug_assert_ne!(old_count, 0, "UB: cannot incref dead objects"); + RefCountModification { + old_count: NonZeroUsize::new_unchecked(old_count), + } + } + + #[inline] + pub unsafe fn handle_decref(old_count: usize) -> Self { + debug_assert_ne!(old_count, 0, "UB: cannot decref already dead objects"); + RefCountModification { + old_count: NonZeroUsize::new_unchecked(old_count), + } + } +} + +/// A simple thread-unsafe reference count. +#[derive(Debug)] +pub struct NotSyncReferenceCounter { + count: Cell, +} +unsafe impl ReferenceCounter for NotSyncReferenceCounter { + const MAX_COUNT: usize = isize::MAX as usize; + + #[inline] + unsafe fn incref(&self) -> RefCountModification { + let old_count = self.count.get(); + let update = RefCountModification::handle_incref(old_count); + self.count.set(old_count.unchecked_add(1)); + update + } + + #[inline] + unsafe fn decref(&self) -> RefCountModification { + let old_count = self.count.get(); + let update = RefCountModification::handle_decref(old_count); + self.count.set(old_count.unchecked_sub(1)); + update + } +} +impl crate::sealed::Sealed for NotSyncReferenceCounter {} +// should be implied by Cell +static_assertions::assert_not_impl_any!(NotSyncReferenceCounter: Send, Sync); + +#[cfg(feature = "sync")] +pub use self::sync::AtomicReferenceCounter; + +#[cfg(feature = "sync")] +mod sync { + use crate::system::threads::{RefCountModification, ReferenceCounter}; + use core::sync::atomic::{AtomicUsize, Ordering}; + + /// An atomic reference-counter, similar to [`std::sync::Arc`]. + /// + /// TODO: Implement biased reference counting (separate crate?) + #[derive(Debug)] + pub struct AtomicReferenceCounter { + count: AtomicUsize, + } + unsafe impl ReferenceCounter for AtomicReferenceCounter { + const MAX_COUNT: usize = isize::MAX as usize; + + #[inline] + unsafe fn incref(&self) -> RefCountModification { + // NOTE: The stdlib uses `Relaxed` ordering. + let old_count = self.count.fetch_add(1, Ordering::Relaxed); + RefCountModification::handle_incref(old_count) + } + + #[inline] + unsafe fn decref(&self) -> RefCountModification { + // NTE: The stdlib uses `Release` ordering. + let old_count = self.count.fetch_sub(1, Ordering::Release); + RefCountModification::handle_decref(old_count) + } + } + impl crate::sealed::Sealed for AtomicReferenceCounter {} +} From c3d14721056b69d1cffc3fd80b1ff744a06e8a63 Mon Sep 17 00:00:00 2001 From: Techcable Date: Sun, 30 Jun 2024 03:53:12 -0700 Subject: [PATCH 35/35] Attempt to make GcHandle a trait Currently doesn't compile --- src/context/handle.rs | 105 +++++++++++++++++++++++++++---- src/system.rs | 53 ++++------------ src/system/threads.rs | 142 ------------------------------------------ 3 files changed, 105 insertions(+), 195 deletions(-) delete mode 100644 src/system/threads.rs diff --git a/src/context/handle.rs b/src/context/handle.rs index 6b690dc..799ebce 100644 --- a/src/context/handle.rs +++ b/src/context/handle.rs @@ -1,19 +1,100 @@ -//! Implements the [`GcHandle`] type. +//! Defines the [`GcSyncHandle`] and [`GcLocalHandle`] types. -use crate::{CollectorId, GcSafe}; -use core::ptr::NonNull; +use crate::system::CollectorIdSync; +use crate::{CollectorId, NullTrace, Trace}; +use core::fmt::{self, Debug}; +use core::marker::PhantomData; +use zerogc_derive::unsafe_gc_impl; -type TODO = std::convert::Infailable; +/// The underlying implementation of a [`GcHandle`] +/// +/// ## Safety +/// These types must be implemented correctly, +/// and preserve values across collections. +pub unsafe trait GcHandleImpl: NullTrace + 'static { + /// The id of the collector + type Id: CollectorId; + + /// Retrieve the underlying object header from this handle. + /// + /// ## Safety + /// The header cannot be used while collections are occurring. + /// + /// The underlying object must actually be a regular object (not an array). + unsafe fn resolve_regular_header(&self) -> ::RegularHeader; + + /// Create a duplicate of this handle + /// + /// This is roughly equivalent to [`Clone::clone`] for a [`Arc`](std::sync::Arc). + fn duplicate(&self) -> Self; +} + +macro_rules! common_handle_impl { + ($target:ident, Id: $bound:ident) => { + impl $target<[T], Id> { -struct GcHandleBox, Id: CollectorId> { - value: T, - refcnt: TODO, // todo + } + impl Clone for $target { + #[inline] + fn clone(&self) -> Self { + $target::from_impl(self.value.duplicate()) + } + } + impl Debug for $target { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct(stringify!($target)) + .finish_non_exhaustive() + } + } + unsafe_gc_impl!( + target => $target, + params => [T: Trace + ?Sized, ImplId: $bound], + null_trace => always, + NEEDS_TRACE => false, + NEEDS_DROP => core::mem::needs_drop::<$target>(), + collector_id => *, + trace_template => |self, visitor| { /* nop */ Ok(()) }, + ); + }; } -/// A thread-local `GcHandle`. +/// A thread-safe handle to a garbage collected value. /// -/// This can be faster than a standard [`GcHandle`], -/// because it is `!Send`. -pub struct GcLocalHandle, Id: CollectorId> { - refcnt: TODO, +/// This is analogous to a [`Arc`](std::sync::Arc) +pub struct GcSyncHandle { + value: Id::SyncHandle, +} +common_handle_impl!(GcSyncHandle, Id: CollectorIdSync); +impl GcSyncHandle { + #[inline] + pub(crate) fn from_impl(value: Id::SyncHandle) -> Self { + GcSyncHandle { value } + } +} +/// Implements `Sync` as long as `T: Sync` +unsafe impl Sync for GcSyncHandle {} +/// Implements `Send` as long as `T: Sync` +/// +/// The reason for requiring `T: Sync` is the same as [`Arc`](std::sync::Arc), +/// because these references can be duplicated. +unsafe impl Send for GcSyncHandle {} + +/// A thread-local handle to a garbage-collected value. +/// +/// This is analogous to a [`Rc`](std::rc::Rc), and cannot be sent across threads (is `!Send`). +/// However, it is often faster to create and clone. +pub struct GcLocalHandle { + value: Id::LocalHandle, + /// Indicates the type is never `Send` or `Sync` + marker: PhantomData<*mut T>, +} +common_handle_impl!(GcLocalHandle, Id: CollectorId); +impl GcLocalHandle { + #[inline] + pub(crate) fn from_impl(value: Id::SyncHandle) -> Self { + GcLocalHandle { + value, + marker: PhantomData, + } + } } diff --git a/src/system.rs b/src/system.rs index 444bfff..ee22862 100644 --- a/src/system.rs +++ b/src/system.rs @@ -3,49 +3,11 @@ use core::fmt::Debug; use core::hash::Hash; use core::marker::PhantomData; +use zerogc::context::handle::GcHandleImpl; +use zerogc::Trace; use crate::trace::{Gc, GcSafe, NullTrace, TrustedDrop}; -pub(crate) mod threads; - -/// Indicates whether a collector is thread-safe. -/// -/// Most of the functions and types in this trait are logically private. -/// Accessing them is not subject to semver guarantees. -pub unsafe trait GcThreadSafety: - threads::ThreadSafetyInternal + crate::sealed::Sealed -{ -} - -/// A collector which is not thread-safe. This is `!Send` and `!Sync`. -pub struct NotSyncCollector { - /// Indicates `!Send` and `!Sync` - marker: PhantomData<*const ()>, -} - -unsafe impl GcThreadSafety for NotSyncCollector {} -unsafe impl threads::ThreadSafetyInternal for NotSyncCollector { - type ReferenceCounter = self::threads::NotSyncReferenceCounter; -} -impl crate::sealed::Sealed for NotSyncCollector {} - -static_assertions::assert_not_impl_any!(NotSyncCollector: Send, Sync); - -/// A collector which is thread-safe. -/// -/// This is both `Send` and `Sync`. -pub struct SyncCollector { - _priv: (), -} - -unsafe impl GcThreadSafety for SyncCollector {} -unsafe impl threads::ThreadSafetyInternal for SyncCollector { - type ReferenceCounter = self::threads::AtomicReferenceCounter; -} -impl crate::sealed::Sealed for SyncCollector {} - -static_assertions::assert_impl_all!(SyncCollector: Send, Sync); - /// A garbage collector implementation, /// conforming to the zerogc API. /// @@ -252,6 +214,7 @@ pub unsafe trait GcArrayHeader: GcHeader { /// Return the length of the array, /// in terms of the elements. fn len(&self) -> usize; + /// Check if the array is empty. #[inline] fn is_empty(&self) -> bool { @@ -283,8 +246,11 @@ pub unsafe trait CollectorId: type ContextState: GcContextState; /// The header for a garbage-collected array. type ArrayHeader: GcArrayHeader; - /// The header for a regulalar [`Gc`] object + /// The header for a regular [`Gc`] object type RegularHeader: GcRegularHeader; + /// A thread-local handle to garbage-collected objects, + /// guaranteed to be preserved across collections. + type LocalHandle: GcHandleImpl; /// Determine the [regular header](GcRegularHeader) for the specified [Gc] object. /// @@ -326,3 +292,8 @@ pub unsafe trait CollectorId: /// and not used after the collector no longer exists. unsafe fn assume_valid_system<'a>(self) -> &'a Self::System; } + +/// A thread-safe [`CollectorId`] +pub trait CollectorIdSync: CollectorId + Sync { + type SyncHandle: GcHandleImpl + Sync; +} diff --git a/src/system/threads.rs b/src/system/threads.rs deleted file mode 100644 index 064b2e7..0000000 --- a/src/system/threads.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! Internal code to abstract over the thread-safety of a collector. - -use core::cell::Cell; -use core::num::NonZeroUsize; -use std::sync::Arc; - -/// Internal abstraction over the thread-safety of a collector. -/// -/// Publicly exposed via [`zerogc::system::GcThreadSafety`]. -/// -/// ## Safety -/// The correctness of this abstraction is relied upon for thread-safety. -pub unsafe trait ThreadSafetyInternal: crate::sealed::Sealed { - /// Potentially thread-safe reference counting. - type ReferenceCounter: ReferenceCounter; -} - -/// A potentially thread-safe reference count. -/// -/// ## Safety -/// The correctness of the reference-count is relied upon for memory safety. -pub unsafe trait ReferenceCounter: crate::sealed::Sealed { - /// The maximum reference count. - const MAX_COUNT: usize; - /// Increment the reference count by one. - /// - /// For memory safety reasons, - /// this method will implicitly abort on overflow - /// just like standard-library reference counts. - /// - /// ## Safety - /// Not technically unsafe, but incorrectly incrementing this - /// could trigger memory leaks or aborts. - /// - /// Undefined behavior to incref a dead object. - unsafe fn incref(&self) -> RefCountModification; - /// Decrement the reference count by one. - /// - /// ## Safety - /// Incorrectly decrementing the reference count - /// may result in use-after-free. - /// - /// Undefined behavior to incref an already dead object. - /// Undefined behavior on underflow. - unsafe fn decref(&self) -> RefCountModification; -} - -pub struct RefCountModification { - /// The previous reference count. - /// - /// Guaranteed to be nonzero, - /// since refcount operations can not be performed on dead objects. - pub old_count: NonZeroUsize, -} -impl RefCountModification { - #[inline] - pub unsafe fn handle_incref(old_count: usize) -> Self { - /* - * Overflow is possible due to `std::mem::forget`. - * If encountered, abort the program. - * This is the same behavior as the stdlib. - */ - if old_count >= Self::MAX_COUNT { - libabort::abort(); // TODO: Switch to `trap` which is faster? - } - debug_assert_ne!(old_count, 0, "UB: cannot incref dead objects"); - RefCountModification { - old_count: NonZeroUsize::new_unchecked(old_count), - } - } - - #[inline] - pub unsafe fn handle_decref(old_count: usize) -> Self { - debug_assert_ne!(old_count, 0, "UB: cannot decref already dead objects"); - RefCountModification { - old_count: NonZeroUsize::new_unchecked(old_count), - } - } -} - -/// A simple thread-unsafe reference count. -#[derive(Debug)] -pub struct NotSyncReferenceCounter { - count: Cell, -} -unsafe impl ReferenceCounter for NotSyncReferenceCounter { - const MAX_COUNT: usize = isize::MAX as usize; - - #[inline] - unsafe fn incref(&self) -> RefCountModification { - let old_count = self.count.get(); - let update = RefCountModification::handle_incref(old_count); - self.count.set(old_count.unchecked_add(1)); - update - } - - #[inline] - unsafe fn decref(&self) -> RefCountModification { - let old_count = self.count.get(); - let update = RefCountModification::handle_decref(old_count); - self.count.set(old_count.unchecked_sub(1)); - update - } -} -impl crate::sealed::Sealed for NotSyncReferenceCounter {} -// should be implied by Cell -static_assertions::assert_not_impl_any!(NotSyncReferenceCounter: Send, Sync); - -#[cfg(feature = "sync")] -pub use self::sync::AtomicReferenceCounter; - -#[cfg(feature = "sync")] -mod sync { - use crate::system::threads::{RefCountModification, ReferenceCounter}; - use core::sync::atomic::{AtomicUsize, Ordering}; - - /// An atomic reference-counter, similar to [`std::sync::Arc`]. - /// - /// TODO: Implement biased reference counting (separate crate?) - #[derive(Debug)] - pub struct AtomicReferenceCounter { - count: AtomicUsize, - } - unsafe impl ReferenceCounter for AtomicReferenceCounter { - const MAX_COUNT: usize = isize::MAX as usize; - - #[inline] - unsafe fn incref(&self) -> RefCountModification { - // NOTE: The stdlib uses `Relaxed` ordering. - let old_count = self.count.fetch_add(1, Ordering::Relaxed); - RefCountModification::handle_incref(old_count) - } - - #[inline] - unsafe fn decref(&self) -> RefCountModification { - // NTE: The stdlib uses `Release` ordering. - let old_count = self.count.fetch_sub(1, Ordering::Release); - RefCountModification::handle_decref(old_count) - } - } - impl crate::sealed::Sealed for AtomicReferenceCounter {} -}