Skip to content

Commit 291465a

Browse files
borsbors
committed
Auto merge of #480 - a1phyr:iter_fold, r=Amanieu
Specialize `fold` implementation of iterators This provides 5-8% iteration speedups when using internal iteration. Similar work could probably be done with `try_fold` but this requires `nightly` to implement. Maybe this is worth running a rustc perf run with this before merge ?
2 parents 2e138e9 + 798ba9f commit 291465a

File tree

4 files changed

+270
-2
lines changed

4 files changed

+270
-2
lines changed

‎src/map.rs

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2469,6 +2469,14 @@ impl<K, V, A: Allocator> Iterator for IntoKeys<K, V, A> {
24692469
fn size_hint(&self) -> (usize, Option<usize>) {
24702470
self.inner.size_hint()
24712471
}
2472+
#[inline]
2473+
fn fold<B, F>(self, init: B, mut f: F) -> B
2474+
where
2475+
Self: Sized,
2476+
F: FnMut(B, Self::Item) -> B,
2477+
{
2478+
self.inner.fold(init, |acc, (k, _)| f(acc, k))
2479+
}
24722480
}
24732481

24742482
impl<K, V, A: Allocator> ExactSizeIterator for IntoKeys<K, V, A> {
@@ -2531,6 +2539,14 @@ impl<K, V, A: Allocator> Iterator for IntoValues<K, V, A> {
25312539
fn size_hint(&self) -> (usize, Option<usize>) {
25322540
self.inner.size_hint()
25332541
}
2542+
#[inline]
2543+
fn fold<B, F>(self, init: B, mut f: F) -> B
2544+
where
2545+
Self: Sized,
2546+
F: FnMut(B, Self::Item) -> B,
2547+
{
2548+
self.inner.fold(init, |acc, (_, v)| f(acc, v))
2549+
}
25342550
}
25352551

25362552
impl<K, V, A: Allocator> ExactSizeIterator for IntoValues<K, V, A> {
@@ -4722,6 +4738,17 @@ impl<'a, K, V> Iterator for Iter<'a, K, V> {
47224738
fn size_hint(&self) -> (usize, Option<usize>) {
47234739
self.inner.size_hint()
47244740
}
4741+
#[cfg_attr(feature = "inline-more", inline)]
4742+
fn fold<B, F>(self, init: B, mut f: F) -> B
4743+
where
4744+
Self: Sized,
4745+
F: FnMut(B, Self::Item) -> B,
4746+
{
4747+
self.inner.fold(init, |acc, x| unsafe {
4748+
let (k, v) = x.as_ref();
4749+
f(acc, (k, v))
4750+
})
4751+
}
47254752
}
47264753
impl<K, V> ExactSizeIterator for Iter<'_, K, V> {
47274754
#[cfg_attr(feature = "inline-more", inline)]
@@ -4750,6 +4777,17 @@ impl<'a, K, V> Iterator for IterMut<'a, K, V> {
47504777
fn size_hint(&self) -> (usize, Option<usize>) {
47514778
self.inner.size_hint()
47524779
}
4780+
#[cfg_attr(feature = "inline-more", inline)]
4781+
fn fold<B, F>(self, init: B, mut f: F) -> B
4782+
where
4783+
Self: Sized,
4784+
F: FnMut(B, Self::Item) -> B,
4785+
{
4786+
self.inner.fold(init, |acc, x| unsafe {
4787+
let (k, v) = x.as_mut();
4788+
f(acc, (k, v))
4789+
})
4790+
}
47534791
}
47544792
impl<K, V> ExactSizeIterator for IterMut<'_, K, V> {
47554793
#[cfg_attr(feature = "inline-more", inline)]
@@ -4780,6 +4818,14 @@ impl<K, V, A: Allocator> Iterator for IntoIter<K, V, A> {
47804818
fn size_hint(&self) -> (usize, Option<usize>) {
47814819
self.inner.size_hint()
47824820
}
4821+
#[cfg_attr(feature = "inline-more", inline)]
4822+
fn fold<B, F>(self, init: B, f: F) -> B
4823+
where
4824+
Self: Sized,
4825+
F: FnMut(B, Self::Item) -> B,
4826+
{
4827+
self.inner.fold(init, f)
4828+
}
47834829
}
47844830
impl<K, V, A: Allocator> ExactSizeIterator for IntoIter<K, V, A> {
47854831
#[cfg_attr(feature = "inline-more", inline)]
@@ -4810,6 +4856,14 @@ impl<'a, K, V> Iterator for Keys<'a, K, V> {
48104856
fn size_hint(&self) -> (usize, Option<usize>) {
48114857
self.inner.size_hint()
48124858
}
4859+
#[cfg_attr(feature = "inline-more", inline)]
4860+
fn fold<B, F>(self, init: B, mut f: F) -> B
4861+
where
4862+
Self: Sized,
4863+
F: FnMut(B, Self::Item) -> B,
4864+
{
4865+
self.inner.fold(init, |acc, (k, _)| f(acc, k))
4866+
}
48134867
}
48144868
impl<K, V> ExactSizeIterator for Keys<'_, K, V> {
48154869
#[cfg_attr(feature = "inline-more", inline)]
@@ -4834,6 +4888,14 @@ impl<'a, K, V> Iterator for Values<'a, K, V> {
48344888
fn size_hint(&self) -> (usize, Option<usize>) {
48354889
self.inner.size_hint()
48364890
}
4891+
#[cfg_attr(feature = "inline-more", inline)]
4892+
fn fold<B, F>(self, init: B, mut f: F) -> B
4893+
where
4894+
Self: Sized,
4895+
F: FnMut(B, Self::Item) -> B,
4896+
{
4897+
self.inner.fold(init, |acc, (_, v)| f(acc, v))
4898+
}
48374899
}
48384900
impl<K, V> ExactSizeIterator for Values<'_, K, V> {
48394901
#[cfg_attr(feature = "inline-more", inline)]
@@ -4858,6 +4920,14 @@ impl<'a, K, V> Iterator for ValuesMut<'a, K, V> {
48584920
fn size_hint(&self) -> (usize, Option<usize>) {
48594921
self.inner.size_hint()
48604922
}
4923+
#[cfg_attr(feature = "inline-more", inline)]
4924+
fn fold<B, F>(self, init: B, mut f: F) -> B
4925+
where
4926+
Self: Sized,
4927+
F: FnMut(B, Self::Item) -> B,
4928+
{
4929+
self.inner.fold(init, |acc, (_, v)| f(acc, v))
4930+
}
48614931
}
48624932
impl<K, V> ExactSizeIterator for ValuesMut<'_, K, V> {
48634933
#[cfg_attr(feature = "inline-more", inline)]
@@ -4886,6 +4956,14 @@ impl<'a, K, V, A: Allocator> Iterator for Drain<'a, K, V, A> {
48864956
fn size_hint(&self) -> (usize, Option<usize>) {
48874957
self.inner.size_hint()
48884958
}
4959+
#[cfg_attr(feature = "inline-more", inline)]
4960+
fn fold<B, F>(self, init: B, f: F) -> B
4961+
where
4962+
Self: Sized,
4963+
F: FnMut(B, Self::Item) -> B,
4964+
{
4965+
self.inner.fold(init, f)
4966+
}
48894967
}
48904968
impl<K, V, A: Allocator> ExactSizeIterator for Drain<'_, K, V, A> {
48914969
#[cfg_attr(feature = "inline-more", inline)]

‎src/raw/mod.rs

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3844,6 +3844,85 @@ impl<T> RawIterRange<T> {
38443844
self.next_ctrl = self.next_ctrl.add(Group::WIDTH);
38453845
}
38463846
}
3847+
3848+
/// Folds every element into an accumulator by applying an operation,
3849+
/// returning the final result.
3850+
///
3851+
/// `fold_impl()` takes three arguments: the number of items remaining in
3852+
/// the iterator, an initial value, and a closure with two arguments: an
3853+
/// 'accumulator', and an element. The closure returns the value that the
3854+
/// accumulator should have for the next iteration.
3855+
///
3856+
/// The initial value is the value the accumulator will have on the first call.
3857+
///
3858+
/// After applying this closure to every element of the iterator, `fold_impl()`
3859+
/// returns the accumulator.
3860+
///
3861+
/// # Safety
3862+
///
3863+
/// If any of the following conditions are violated, the result is
3864+
/// [`Undefined Behavior`]:
3865+
///
3866+
/// * The [`RawTableInner`] / [`RawTable`] must be alive and not moved,
3867+
/// i.e. table outlives the `RawIterRange`;
3868+
///
3869+
/// * The provided `n` value must match the actual number of items
3870+
/// in the table.
3871+
///
3872+
/// [`Undefined Behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
3873+
#[allow(clippy::while_let_on_iterator)]
3874+
#[cfg_attr(feature = "inline-more", inline)]
3875+
unsafe fn fold_impl<F, B>(mut self, mut n: usize, mut acc: B, mut f: F) -> B
3876+
where
3877+
F: FnMut(B, Bucket<T>) -> B,
3878+
{
3879+
loop {
3880+
while let Some(index) = self.current_group.next() {
3881+
// The returned `index` will always be in the range `0..Group::WIDTH`,
3882+
// so that calling `self.data.next_n(index)` is safe (see detailed explanation below).
3883+
debug_assert!(n != 0);
3884+
let bucket = self.data.next_n(index);
3885+
acc = f(acc, bucket);
3886+
n -= 1;
3887+
}
3888+
3889+
if n == 0 {
3890+
return acc;
3891+
}
3892+
3893+
// SAFETY: The caller of this function ensures that:
3894+
//
3895+
// 1. The provided `n` value matches the actual number of items in the table;
3896+
// 2. The table is alive and did not moved.
3897+
//
3898+
// Taking the above into account, we always stay within the bounds, because:
3899+
//
3900+
// 1. For tables smaller than the group width (self.buckets() <= Group::WIDTH),
3901+
// we will never end up in the given branch, since we should have already
3902+
// yielded all the elements of the table.
3903+
//
3904+
// 2. For tables larger than the group width. The the number of buckets is a
3905+
// power of two (2 ^ n), Group::WIDTH is also power of two (2 ^ k). Sinse
3906+
// `(2 ^ n) > (2 ^ k)`, than `(2 ^ n) % (2 ^ k) = 0`. As we start from the
3907+
// the start of the array of control bytes, and never try to iterate after
3908+
// getting all the elements, the last `self.current_group` will read bytes
3909+
// from the `self.buckets() - Group::WIDTH` index. We know also that
3910+
// `self.current_group.next()` will always retun indices within the range
3911+
// `0..Group::WIDTH`.
3912+
//
3913+
// Knowing all of the above and taking into account that we are synchronizing
3914+
// the `self.data` index with the index we used to read the `self.current_group`,
3915+
// the subsequent `self.data.next_n(index)` will always return a bucket with
3916+
// an index number less than `self.buckets()`.
3917+
//
3918+
// The last `self.next_ctrl`, whose index would be `self.buckets()`, will never
3919+
// actually be read, since we should have already yielded all the elements of
3920+
// the table.
3921+
self.current_group = Group::load_aligned(self.next_ctrl).match_full().into_iter();
3922+
self.data = self.data.next_n(Group::WIDTH);
3923+
self.next_ctrl = self.next_ctrl.add(Group::WIDTH);
3924+
}
3925+
}
38473926
}
38483927

38493928
// We make raw iterators unconditionally Send and Sync, and let the PhantomData
@@ -4067,6 +4146,15 @@ impl<T> Iterator for RawIter<T> {
40674146
fn size_hint(&self) -> (usize, Option<usize>) {
40684147
(self.items, Some(self.items))
40694148
}
4149+
4150+
#[inline]
4151+
fn fold<B, F>(self, init: B, f: F) -> B
4152+
where
4153+
Self: Sized,
4154+
F: FnMut(B, Self::Item) -> B,
4155+
{
4156+
unsafe { self.iter.fold_impl(self.items, init, f) }
4157+
}
40704158
}
40714159

40724160
impl<T> ExactSizeIterator for RawIter<T> {}

‎src/set.rs

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1696,6 +1696,14 @@ impl<'a, K> Iterator for Iter<'a, K> {
16961696
fn size_hint(&self) -> (usize, Option<usize>) {
16971697
self.iter.size_hint()
16981698
}
1699+
#[cfg_attr(feature = "inline-more", inline)]
1700+
fn fold<B, F>(self, init: B, f: F) -> B
1701+
where
1702+
Self: Sized,
1703+
F: FnMut(B, Self::Item) -> B,
1704+
{
1705+
self.iter.fold(init, f)
1706+
}
16991707
}
17001708
impl<'a, K> ExactSizeIterator for Iter<'a, K> {
17011709
#[cfg_attr(feature = "inline-more", inline)]
@@ -1726,6 +1734,14 @@ impl<K, A: Allocator> Iterator for IntoIter<K, A> {
17261734
fn size_hint(&self) -> (usize, Option<usize>) {
17271735
self.iter.size_hint()
17281736
}
1737+
#[cfg_attr(feature = "inline-more", inline)]
1738+
fn fold<B, F>(self, init: B, mut f: F) -> B
1739+
where
1740+
Self: Sized,
1741+
F: FnMut(B, Self::Item) -> B,
1742+
{
1743+
self.iter.fold(init, |acc, (k, ())| f(acc, k))
1744+
}
17291745
}
17301746
impl<K, A: Allocator> ExactSizeIterator for IntoIter<K, A> {
17311747
#[cfg_attr(feature = "inline-more", inline)]
@@ -1757,6 +1773,14 @@ impl<K, A: Allocator> Iterator for Drain<'_, K, A> {
17571773
fn size_hint(&self) -> (usize, Option<usize>) {
17581774
self.iter.size_hint()
17591775
}
1776+
#[cfg_attr(feature = "inline-more", inline)]
1777+
fn fold<B, F>(self, init: B, mut f: F) -> B
1778+
where
1779+
Self: Sized,
1780+
F: FnMut(B, Self::Item) -> B,
1781+
{
1782+
self.iter.fold(init, |acc, (k, ())| f(acc, k))
1783+
}
17601784
}
17611785
impl<K, A: Allocator> ExactSizeIterator for Drain<'_, K, A> {
17621786
#[cfg_attr(feature = "inline-more", inline)]
@@ -1827,6 +1851,20 @@ where
18271851
let (_, upper) = self.iter.size_hint();
18281852
(0, upper)
18291853
}
1854+
#[cfg_attr(feature = "inline-more", inline)]
1855+
fn fold<B, F>(self, init: B, mut f: F) -> B
1856+
where
1857+
Self: Sized,
1858+
F: FnMut(B, Self::Item) -> B,
1859+
{
1860+
self.iter.fold(init, |acc, elt| {
1861+
if self.other.contains(elt) {
1862+
f(acc, elt)
1863+
} else {
1864+
acc
1865+
}
1866+
})
1867+
}
18301868
}
18311869

18321870
impl<T, S, A> fmt::Debug for Intersection<'_, T, S, A>
@@ -1881,6 +1919,20 @@ where
18811919
let (_, upper) = self.iter.size_hint();
18821920
(0, upper)
18831921
}
1922+
#[cfg_attr(feature = "inline-more", inline)]
1923+
fn fold<B, F>(self, init: B, mut f: F) -> B
1924+
where
1925+
Self: Sized,
1926+
F: FnMut(B, Self::Item) -> B,
1927+
{
1928+
self.iter.fold(init, |acc, elt| {
1929+
if self.other.contains(elt) {
1930+
acc
1931+
} else {
1932+
f(acc, elt)
1933+
}
1934+
})
1935+
}
18841936
}
18851937

18861938
impl<T, S, A> FusedIterator for Difference<'_, T, S, A>
@@ -1927,6 +1979,14 @@ where
19271979
fn size_hint(&self) -> (usize, Option<usize>) {
19281980
self.iter.size_hint()
19291981
}
1982+
#[cfg_attr(feature = "inline-more", inline)]
1983+
fn fold<B, F>(self, init: B, f: F) -> B
1984+
where
1985+
Self: Sized,
1986+
F: FnMut(B, Self::Item) -> B,
1987+
{
1988+
self.iter.fold(init, f)
1989+
}
19301990
}
19311991

19321992
impl<T, S, A> FusedIterator for SymmetricDifference<'_, T, S, A>
@@ -1992,6 +2052,14 @@ where
19922052
fn size_hint(&self) -> (usize, Option<usize>) {
19932053
self.iter.size_hint()
19942054
}
2055+
#[cfg_attr(feature = "inline-more", inline)]
2056+
fn fold<B, F>(self, init: B, f: F) -> B
2057+
where
2058+
Self: Sized,
2059+
F: FnMut(B, Self::Item) -> B,
2060+
{
2061+
self.iter.fold(init, f)
2062+
}
19952063
}
19962064

19972065
/// A view into a single entry in a set, which may either be vacant or occupied.

0 commit comments

Comments
 (0)