From 66208820895c3e32ce4bff3b4a4039da4dde1d4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sat, 31 Oct 2020 03:14:32 +0100 Subject: [PATCH 1/3] Use only one shard with a single thread --- compiler/rustc_data_structures/src/sharded.rs | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index 7ed70ba1e0fc7..c16b9cbf8b773 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -1,4 +1,6 @@ use crate::fx::{FxHashMap, FxHasher}; +#[cfg(parallel_compiler)] +use crate::sync::is_dyn_thread_safe; use crate::sync::{CacheAligned, Lock, LockGuard}; use std::borrow::Borrow; use std::collections::hash_map::RawEntryMut; @@ -18,6 +20,8 @@ pub const SHARDS: usize = 1 << SHARD_BITS; /// An array of cache-line aligned inner locked structures with convenience methods. pub struct Sharded { + #[cfg(parallel_compiler)] + mask: usize, shards: [CacheAligned>; SHARDS], } @@ -31,31 +35,53 @@ impl Default for Sharded { impl Sharded { #[inline] pub fn new(mut value: impl FnMut() -> T) -> Self { - Sharded { shards: [(); SHARDS].map(|()| CacheAligned(Lock::new(value()))) } + Sharded { + #[cfg(parallel_compiler)] + mask: if is_dyn_thread_safe() { SHARDS - 1 } else { 0 }, + shards: [(); SHARDS].map(|()| CacheAligned(Lock::new(value()))), + } + } + + #[inline(always)] + fn mask(&self) -> usize { + #[cfg(parallel_compiler)] + { + self.mask + } + #[cfg(not(parallel_compiler))] + { + 0 + } + } + + #[inline(always)] + fn count(&self) -> usize { + self.mask() + 1 } /// The shard is selected by hashing `val` with `FxHasher`. #[inline] pub fn get_shard_by_value(&self, val: &K) -> &Lock { - if SHARDS == 1 { &self.shards[0].0 } else { self.get_shard_by_hash(make_hash(val)) } + self.get_shard_by_hash(if SHARDS == 1 { 0 } else { make_hash(val) }) } #[inline] pub fn get_shard_by_hash(&self, hash: u64) -> &Lock { - &self.shards[get_shard_index_by_hash(hash)].0 + self.get_shard_by_index(get_shard_hash(hash)) } #[inline] pub fn get_shard_by_index(&self, i: usize) -> &Lock { - &self.shards[i].0 + // SAFETY: The index get ANDed with the mask, ensuring it is always inbounds. + unsafe { &self.shards.get_unchecked(i & self.mask()).0 } } pub fn lock_shards(&self) -> Vec> { - (0..SHARDS).map(|i| self.shards[i].0.lock()).collect() + (0..self.count()).map(|i| self.get_shard_by_index(i).lock()).collect() } pub fn try_lock_shards(&self) -> Option>> { - (0..SHARDS).map(|i| self.shards[i].0.try_lock()).collect() + (0..self.count()).map(|i| self.get_shard_by_index(i).try_lock()).collect() } } @@ -136,11 +162,9 @@ pub fn make_hash(val: &K) -> u64 { /// `hash` can be computed with any hasher, so long as that hasher is used /// consistently for each `Sharded` instance. #[inline] -#[allow(clippy::modulo_one)] -pub fn get_shard_index_by_hash(hash: u64) -> usize { +fn get_shard_hash(hash: u64) -> usize { let hash_len = mem::size_of::(); // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits. // hashbrown also uses the lowest bits, so we can't use those - let bits = (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize; - bits % SHARDS + (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize } From 5843858c015614c92515f317ce93796c885ca2cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sun, 28 May 2023 15:47:44 +0200 Subject: [PATCH 2/3] Don't access self.mask with a single shard --- compiler/rustc_data_structures/src/sharded.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index c16b9cbf8b773..9f5c97ae798e3 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -46,7 +46,7 @@ impl Sharded { fn mask(&self) -> usize { #[cfg(parallel_compiler)] { - self.mask + if SHARDS == 1 { 0 } else { self.mask } } #[cfg(not(parallel_compiler))] { From 8abafd085aab9b703711b49d02b66910c8c2d739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sun, 28 May 2023 15:54:52 +0200 Subject: [PATCH 3/3] Add some comments --- compiler/rustc_data_structures/src/sharded.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index 9f5c97ae798e3..40cbf14958e77 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -20,6 +20,9 @@ pub const SHARDS: usize = 1 << SHARD_BITS; /// An array of cache-line aligned inner locked structures with convenience methods. pub struct Sharded { + /// This mask is used to ensure that accesses are inbounds of `shards`. + /// When dynamic thread safety is off, this field is set to 0 causing only + /// a single shard to be used for greater cache efficiency. #[cfg(parallel_compiler)] mask: usize, shards: [CacheAligned>; SHARDS], @@ -56,6 +59,7 @@ impl Sharded { #[inline(always)] fn count(&self) -> usize { + // `self.mask` is always one below the used shard count self.mask() + 1 }