Skip to content

Commit deaf7eb

Browse files
a1phyra1phyr
committed
Specialize fold implementation of iterators
This provides 5-8% iteration speedups
1 parent 778e235 commit deaf7eb

File tree

4 files changed

+249
-6
lines changed

4 files changed

+249
-6
lines changed

‎src/map.rs

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2469,6 +2469,14 @@ impl<K, V, A: Allocator> Iterator for IntoKeys<K, V, A> {
24692469
fn size_hint(&self) -> (usize, Option<usize>) {
24702470
self.inner.size_hint()
24712471
}
2472+
#[inline]
2473+
fn fold<B, F>(self, init: B, mut f: F) -> B
2474+
where
2475+
Self: Sized,
2476+
F: FnMut(B, Self::Item) -> B,
2477+
{
2478+
self.inner.fold(init, |acc, (k, _)| f(acc, k))
2479+
}
24722480
}
24732481

24742482
impl<K, V, A: Allocator> ExactSizeIterator for IntoKeys<K, V, A> {
@@ -2531,6 +2539,14 @@ impl<K, V, A: Allocator> Iterator for IntoValues<K, V, A> {
25312539
fn size_hint(&self) -> (usize, Option<usize>) {
25322540
self.inner.size_hint()
25332541
}
2542+
#[inline]
2543+
fn fold<B, F>(self, init: B, mut f: F) -> B
2544+
where
2545+
Self: Sized,
2546+
F: FnMut(B, Self::Item) -> B,
2547+
{
2548+
self.inner.fold(init, |acc, (_, v)| f(acc, v))
2549+
}
25342550
}
25352551

25362552
impl<K, V, A: Allocator> ExactSizeIterator for IntoValues<K, V, A> {
@@ -4722,6 +4738,17 @@ impl<'a, K, V> Iterator for Iter<'a, K, V> {
47224738
fn size_hint(&self) -> (usize, Option<usize>) {
47234739
self.inner.size_hint()
47244740
}
4741+
#[cfg_attr(feature = "inline-more", inline)]
4742+
fn fold<B, F>(self, init: B, mut f: F) -> B
4743+
where
4744+
Self: Sized,
4745+
F: FnMut(B, Self::Item) -> B,
4746+
{
4747+
self.inner.fold(init, |acc, x| unsafe {
4748+
let (k, v) = x.as_ref();
4749+
f(acc, (k, v))
4750+
})
4751+
}
47254752
}
47264753
impl<K, V> ExactSizeIterator for Iter<'_, K, V> {
47274754
#[cfg_attr(feature = "inline-more", inline)]
@@ -4750,6 +4777,17 @@ impl<'a, K, V> Iterator for IterMut<'a, K, V> {
47504777
fn size_hint(&self) -> (usize, Option<usize>) {
47514778
self.inner.size_hint()
47524779
}
4780+
#[cfg_attr(feature = "inline-more", inline)]
4781+
fn fold<B, F>(self, init: B, mut f: F) -> B
4782+
where
4783+
Self: Sized,
4784+
F: FnMut(B, Self::Item) -> B,
4785+
{
4786+
self.inner.fold(init, |acc, x| unsafe {
4787+
let (k, v) = x.as_mut();
4788+
f(acc, (k, v))
4789+
})
4790+
}
47534791
}
47544792
impl<K, V> ExactSizeIterator for IterMut<'_, K, V> {
47554793
#[cfg_attr(feature = "inline-more", inline)]
@@ -4780,6 +4818,14 @@ impl<K, V, A: Allocator> Iterator for IntoIter<K, V, A> {
47804818
fn size_hint(&self) -> (usize, Option<usize>) {
47814819
self.inner.size_hint()
47824820
}
4821+
#[cfg_attr(feature = "inline-more", inline)]
4822+
fn fold<B, F>(self, init: B, f: F) -> B
4823+
where
4824+
Self: Sized,
4825+
F: FnMut(B, Self::Item) -> B,
4826+
{
4827+
self.inner.fold(init, f)
4828+
}
47834829
}
47844830
impl<K, V, A: Allocator> ExactSizeIterator for IntoIter<K, V, A> {
47854831
#[cfg_attr(feature = "inline-more", inline)]
@@ -4810,6 +4856,14 @@ impl<'a, K, V> Iterator for Keys<'a, K, V> {
48104856
fn size_hint(&self) -> (usize, Option<usize>) {
48114857
self.inner.size_hint()
48124858
}
4859+
#[cfg_attr(feature = "inline-more", inline)]
4860+
fn fold<B, F>(self, init: B, mut f: F) -> B
4861+
where
4862+
Self: Sized,
4863+
F: FnMut(B, Self::Item) -> B,
4864+
{
4865+
self.inner.fold(init, |acc, (k, _)| f(acc, k))
4866+
}
48134867
}
48144868
impl<K, V> ExactSizeIterator for Keys<'_, K, V> {
48154869
#[cfg_attr(feature = "inline-more", inline)]
@@ -4834,6 +4888,14 @@ impl<'a, K, V> Iterator for Values<'a, K, V> {
48344888
fn size_hint(&self) -> (usize, Option<usize>) {
48354889
self.inner.size_hint()
48364890
}
4891+
#[cfg_attr(feature = "inline-more", inline)]
4892+
fn fold<B, F>(self, init: B, mut f: F) -> B
4893+
where
4894+
Self: Sized,
4895+
F: FnMut(B, Self::Item) -> B,
4896+
{
4897+
self.inner.fold(init, |acc, (_, v)| f(acc, v))
4898+
}
48374899
}
48384900
impl<K, V> ExactSizeIterator for Values<'_, K, V> {
48394901
#[cfg_attr(feature = "inline-more", inline)]
@@ -4858,6 +4920,14 @@ impl<'a, K, V> Iterator for ValuesMut<'a, K, V> {
48584920
fn size_hint(&self) -> (usize, Option<usize>) {
48594921
self.inner.size_hint()
48604922
}
4923+
#[cfg_attr(feature = "inline-more", inline)]
4924+
fn fold<B, F>(self, init: B, mut f: F) -> B
4925+
where
4926+
Self: Sized,
4927+
F: FnMut(B, Self::Item) -> B,
4928+
{
4929+
self.inner.fold(init, |acc, (_, v)| f(acc, v))
4930+
}
48614931
}
48624932
impl<K, V> ExactSizeIterator for ValuesMut<'_, K, V> {
48634933
#[cfg_attr(feature = "inline-more", inline)]
@@ -4886,6 +4956,14 @@ impl<'a, K, V, A: Allocator> Iterator for Drain<'a, K, V, A> {
48864956
fn size_hint(&self) -> (usize, Option<usize>) {
48874957
self.inner.size_hint()
48884958
}
4959+
#[cfg_attr(feature = "inline-more", inline)]
4960+
fn fold<B, F>(self, init: B, f: F) -> B
4961+
where
4962+
Self: Sized,
4963+
F: FnMut(B, Self::Item) -> B,
4964+
{
4965+
self.inner.fold(init, f)
4966+
}
48894967
}
48904968
impl<K, V, A: Allocator> ExactSizeIterator for Drain<'_, K, V, A> {
48914969
#[cfg_attr(feature = "inline-more", inline)]

‎src/raw/mod.rs

Lines changed: 71 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,36 @@ use self::imp::Group;
5050

5151
// Branch prediction hint. This is currently only available on nightly but it
5252
// consistently improves performance by 10-15%.
53-
#[cfg(not(feature = "nightly"))]
54-
use core::convert::identity as likely;
55-
#[cfg(not(feature = "nightly"))]
56-
use core::convert::identity as unlikely;
53+
// #[cfg(not(feature = "nightly"))]
54+
// use core::convert::identity as likely;
55+
// #[cfg(not(feature = "nightly"))]
56+
// use core::convert::identity as unlikely;
5757
#[cfg(feature = "nightly")]
5858
use core::intrinsics::{likely, unlikely};
5959

60+
#[cfg(not(feature = "nightly"))]
61+
#[inline]
62+
#[cold]
63+
fn cold() {}
64+
65+
#[cfg(not(feature = "nightly"))]
66+
#[inline]
67+
fn likely(b: bool) -> bool {
68+
if !b {
69+
cold()
70+
}
71+
b
72+
}
73+
#[cfg(not(feature = "nightly"))]
74+
#[cold]
75+
#[inline]
76+
fn unlikely(b: bool) -> bool {
77+
if b {
78+
cold()
79+
}
80+
b
81+
}
82+
6083
// Use strict provenance functions if available.
6184
#[cfg(feature = "nightly")]
6285
use core::ptr::invalid_mut;
@@ -3846,6 +3869,41 @@ impl<T> RawIterRange<T> {
38463869
self.next_ctrl = self.next_ctrl.add(Group::WIDTH);
38473870
}
38483871
}
3872+
3873+
/// # Safety
3874+
/// The provided `n` value must match the actual number of items
3875+
#[allow(clippy::while_let_on_iterator)]
3876+
#[cfg_attr(feature = "inline-more", inline)]
3877+
unsafe fn fold_impl<F, B>(mut self, mut n: usize, mut acc: B, mut f: F) -> B
3878+
where
3879+
F: FnMut(B, Bucket<T>) -> B,
3880+
{
3881+
if n == 0 {
3882+
return acc;
3883+
}
3884+
3885+
loop {
3886+
while let Some(index) = self.current_group.next() {
3887+
debug_assert!(n != 0);
3888+
let bucket = self.data.next_n(index);
3889+
acc = f(acc, bucket);
3890+
n -= 1;
3891+
}
3892+
3893+
if n == 0 {
3894+
return acc;
3895+
}
3896+
3897+
// We might read past self.end up to the next group boundary,
3898+
// but this is fine because it only occurs on tables smaller
3899+
// than the group size where the trailing control bytes are all
3900+
// EMPTY. On larger tables self.end is guaranteed to be aligned
3901+
// to the group size (since tables are power-of-two sized).
3902+
self.current_group = Group::load_aligned(self.next_ctrl).match_full().into_iter();
3903+
self.data = self.data.next_n(Group::WIDTH);
3904+
self.next_ctrl = self.next_ctrl.add(Group::WIDTH);
3905+
}
3906+
}
38493907
}
38503908

38513909
// We make raw iterators unconditionally Send and Sync, and let the PhantomData
@@ -4069,6 +4127,15 @@ impl<T> Iterator for RawIter<T> {
40694127
fn size_hint(&self) -> (usize, Option<usize>) {
40704128
(self.items, Some(self.items))
40714129
}
4130+
4131+
#[inline]
4132+
fn fold<B, F>(self, init: B, f: F) -> B
4133+
where
4134+
Self: Sized,
4135+
F: FnMut(B, Self::Item) -> B,
4136+
{
4137+
unsafe { self.iter.fold_impl(self.items, init, f) }
4138+
}
40724139
}
40734140

40744141
impl<T> ExactSizeIterator for RawIter<T> {}

‎src/set.rs

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1696,6 +1696,14 @@ impl<'a, K> Iterator for Iter<'a, K> {
16961696
fn size_hint(&self) -> (usize, Option<usize>) {
16971697
self.iter.size_hint()
16981698
}
1699+
#[cfg_attr(feature = "inline-more", inline)]
1700+
fn fold<B, F>(self, init: B, f: F) -> B
1701+
where
1702+
Self: Sized,
1703+
F: FnMut(B, Self::Item) -> B,
1704+
{
1705+
self.iter.fold(init, f)
1706+
}
16991707
}
17001708
impl<'a, K> ExactSizeIterator for Iter<'a, K> {
17011709
#[cfg_attr(feature = "inline-more", inline)]
@@ -1726,6 +1734,14 @@ impl<K, A: Allocator> Iterator for IntoIter<K, A> {
17261734
fn size_hint(&self) -> (usize, Option<usize>) {
17271735
self.iter.size_hint()
17281736
}
1737+
#[cfg_attr(feature = "inline-more", inline)]
1738+
fn fold<B, F>(self, init: B, mut f: F) -> B
1739+
where
1740+
Self: Sized,
1741+
F: FnMut(B, Self::Item) -> B,
1742+
{
1743+
self.iter.fold(init, |acc, (k, ())| f(acc, k))
1744+
}
17291745
}
17301746
impl<K, A: Allocator> ExactSizeIterator for IntoIter<K, A> {
17311747
#[cfg_attr(feature = "inline-more", inline)]
@@ -1757,6 +1773,14 @@ impl<K, A: Allocator> Iterator for Drain<'_, K, A> {
17571773
fn size_hint(&self) -> (usize, Option<usize>) {
17581774
self.iter.size_hint()
17591775
}
1776+
#[cfg_attr(feature = "inline-more", inline)]
1777+
fn fold<B, F>(self, init: B, mut f: F) -> B
1778+
where
1779+
Self: Sized,
1780+
F: FnMut(B, Self::Item) -> B,
1781+
{
1782+
self.iter.fold(init, |acc, (k, ())| f(acc, k))
1783+
}
17601784
}
17611785
impl<K, A: Allocator> ExactSizeIterator for Drain<'_, K, A> {
17621786
#[cfg_attr(feature = "inline-more", inline)]
@@ -1827,6 +1851,20 @@ where
18271851
let (_, upper) = self.iter.size_hint();
18281852
(0, upper)
18291853
}
1854+
#[cfg_attr(feature = "inline-more", inline)]
1855+
fn fold<B, F>(self, init: B, mut f: F) -> B
1856+
where
1857+
Self: Sized,
1858+
F: FnMut(B, Self::Item) -> B,
1859+
{
1860+
self.iter.fold(init, |acc, elt| {
1861+
if self.other.contains(elt) {
1862+
f(acc, elt)
1863+
} else {
1864+
acc
1865+
}
1866+
})
1867+
}
18301868
}
18311869

18321870
impl<T, S, A> fmt::Debug for Intersection<'_, T, S, A>
@@ -1881,6 +1919,20 @@ where
18811919
let (_, upper) = self.iter.size_hint();
18821920
(0, upper)
18831921
}
1922+
#[cfg_attr(feature = "inline-more", inline)]
1923+
fn fold<B, F>(self, init: B, mut f: F) -> B
1924+
where
1925+
Self: Sized,
1926+
F: FnMut(B, Self::Item) -> B,
1927+
{
1928+
self.iter.fold(init, |acc, elt| {
1929+
if self.other.contains(elt) {
1930+
acc
1931+
} else {
1932+
f(acc, elt)
1933+
}
1934+
})
1935+
}
18841936
}
18851937

18861938
impl<T, S, A> FusedIterator for Difference<'_, T, S, A>
@@ -1927,6 +1979,14 @@ where
19271979
fn size_hint(&self) -> (usize, Option<usize>) {
19281980
self.iter.size_hint()
19291981
}
1982+
#[cfg_attr(feature = "inline-more", inline)]
1983+
fn fold<B, F>(self, init: B, f: F) -> B
1984+
where
1985+
Self: Sized,
1986+
F: FnMut(B, Self::Item) -> B,
1987+
{
1988+
self.iter.fold(init, f)
1989+
}
19301990
}
19311991

19321992
impl<T, S, A> FusedIterator for SymmetricDifference<'_, T, S, A>
@@ -1992,6 +2052,14 @@ where
19922052
fn size_hint(&self) -> (usize, Option<usize>) {
19932053
self.iter.size_hint()
19942054
}
2055+
#[cfg_attr(feature = "inline-more", inline)]
2056+
fn fold<B, F>(self, init: B, f: F) -> B
2057+
where
2058+
Self: Sized,
2059+
F: FnMut(B, Self::Item) -> B,
2060+
{
2061+
self.iter.fold(init, f)
2062+
}
19952063
}
19962064

19972065
/// A view into a single entry in a set, which may either be vacant or occupied.

0 commit comments

Comments
 (0)