diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs index c595bf830a3dc..f38254b241fc2 100644 --- a/compiler/rustc_data_structures/src/lib.rs +++ b/compiler/rustc_data_structures/src/lib.rs @@ -27,6 +27,9 @@ #![feature(thread_id_value)] #![feature(vec_into_raw_parts)] #![feature(get_mut_unchecked)] +#![feature(const_trait_impl)] +#![feature(const_ptr_as_ref)] +#![feature(const_mut_refs)] #![allow(rustc::default_hash_types)] #![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index 31323c21df009..6f5e4427cda2d 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -18,11 +18,19 @@ //! depending on the value of cfg!(parallel_compiler). use crate::owning_ref::{Erased, OwningRef}; +use std::cell::Cell; +use std::cell::UnsafeCell; use std::collections::HashMap; +use std::fmt::{Debug, Formatter}; use std::hash::{BuildHasher, Hash}; +use std::intrinsics::likely; +use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; +use std::ptr::NonNull; +use parking_lot::lock_api::RawMutex as _; +use parking_lot::RawMutex; pub use std::sync::atomic::Ordering; pub use std::sync::atomic::Ordering::SeqCst; @@ -30,6 +38,8 @@ pub use vec::AppendOnlyVec; mod vec; +static PARALLEL: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + cfg_if! { if #[cfg(not(parallel_compiler))] { pub auto trait Send {} @@ -172,15 +182,11 @@ cfg_if! { pub use std::cell::Ref as MappedReadGuard; pub use std::cell::RefMut as WriteGuard; pub use std::cell::RefMut as MappedWriteGuard; - pub use std::cell::RefMut as LockGuard; pub use std::cell::RefMut as MappedLockGuard; pub use std::cell::OnceCell; use std::cell::RefCell as InnerRwLock; - use std::cell::RefCell as InnerLock; - - use std::cell::Cell; #[derive(Debug)] pub struct WorkerLocal(OneThread); @@ -257,7 +263,6 @@ cfg_if! { pub use parking_lot::RwLockWriteGuard as WriteGuard; pub use parking_lot::MappedRwLockWriteGuard as MappedWriteGuard; - pub use parking_lot::MutexGuard as LockGuard; pub use parking_lot::MappedMutexGuard as MappedLockGuard; pub use std::sync::OnceLock as OnceCell; @@ -299,7 +304,6 @@ cfg_if! { } } - use parking_lot::Mutex as InnerLock; use parking_lot::RwLock as InnerRwLock; use std::thread; @@ -381,55 +385,106 @@ impl HashMapExt for HashMap } } -#[derive(Debug)] -pub struct Lock(InnerLock); +pub struct Lock { + single_thread: bool, + data: UnsafeCell, + borrow: Cell, + mutex: RawMutex, +} + +impl Debug for Lock { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self.try_lock() { + Some(guard) => f.debug_struct("Lock").field("data", &&*guard).finish(), + None => { + struct LockedPlaceholder; + impl Debug for LockedPlaceholder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str("") + } + } + + f.debug_struct("Lock").field("data", &LockedPlaceholder).finish() + } + } + } +} impl Lock { - #[inline(always)] - pub fn new(inner: T) -> Self { - Lock(InnerLock::new(inner)) + #[inline] + pub fn new(val: T) -> Self { + Lock { + single_thread: !PARALLEL.load(Ordering::Relaxed), + data: UnsafeCell::new(val), + borrow: Cell::new(false), + mutex: RawMutex::INIT, + } } - #[inline(always)] + #[inline] pub fn into_inner(self) -> T { - self.0.into_inner() + self.data.into_inner() } - #[inline(always)] + #[inline] pub fn get_mut(&mut self) -> &mut T { - self.0.get_mut() + self.data.get_mut() } - #[cfg(parallel_compiler)] - #[inline(always)] + #[inline] pub fn try_lock(&self) -> Option> { - self.0.try_lock() + // SAFETY: the `&mut T` is accessible as long as self exists. + if likely(self.single_thread) { + if self.borrow.get() { + None + } else { + self.borrow.set(true); + Some(LockGuard { + value: unsafe { NonNull::new_unchecked(self.data.get()) }, + lock: &self, + marker: PhantomData, + }) + } + } else { + if !self.mutex.try_lock() { + None + } else { + Some(LockGuard { + value: unsafe { NonNull::new_unchecked(self.data.get()) }, + lock: &self, + marker: PhantomData, + }) + } + } } - #[cfg(not(parallel_compiler))] - #[inline(always)] - pub fn try_lock(&self) -> Option> { - self.0.try_borrow_mut().ok() + #[inline(never)] + fn lock_mt(&self) -> LockGuard<'_, T> { + self.mutex.lock(); + LockGuard { + value: unsafe { NonNull::new_unchecked(self.data.get()) }, + lock: &self, + marker: PhantomData, + } } - #[cfg(parallel_compiler)] - #[inline(always)] + #[inline] #[track_caller] pub fn lock(&self) -> LockGuard<'_, T> { - if ERROR_CHECKING { - self.0.try_lock().expect("lock was already held") + // SAFETY: the `&mut T` is accessible as long as self exists. + if likely(self.single_thread) { + assert!(!self.borrow.get()); + self.borrow.set(true); + LockGuard { + value: unsafe { NonNull::new_unchecked(self.data.get()) }, + lock: &self, + marker: PhantomData, + } } else { - self.0.lock() + self.lock_mt() } } - #[cfg(not(parallel_compiler))] - #[inline(always)] - #[track_caller] - pub fn lock(&self) -> LockGuard<'_, T> { - self.0.borrow_mut() - } - #[inline(always)] #[track_caller] pub fn with_lock R, R>(&self, f: F) -> R { @@ -464,6 +519,47 @@ impl Clone for Lock { } } +// Just for speed test +unsafe impl std::marker::Send for Lock {} +unsafe impl std::marker::Sync for Lock {} + +pub struct LockGuard<'a, T> { + value: NonNull, + lock: &'a Lock, + marker: PhantomData<&'a mut T>, +} + +impl const Deref for LockGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + unsafe { self.value.as_ref() } + } +} + +impl const DerefMut for LockGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + unsafe { self.value.as_mut() } + } +} + +#[inline(never)] +unsafe fn unlock_mt(guard: &mut LockGuard<'_, T>) { + guard.lock.mutex.unlock() +} + +impl<'a, T> Drop for LockGuard<'a, T> { + #[inline] + fn drop(&mut self) { + if likely(self.lock.single_thread) { + debug_assert!(self.lock.borrow.get()); + self.lock.borrow.set(false); + } else { + unsafe { unlock_mt(self) } + } + } +} + #[derive(Debug, Default)] pub struct RwLock(InnerRwLock);