diff --git a/benches/bench.rs b/benches/bench.rs index 987bef9..2386400 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -80,7 +80,7 @@ impl Vector for SmallVec { } fn from_elems(val: &[T]) -> Self { - SmallVec::from_slice(val) + SmallVec::from(val) } fn extend_from_slice(&mut self, other: &[T]) { @@ -278,17 +278,6 @@ fn gen_from_elem>(n: usize, b: &mut Bencher) { }); } -#[bench] -fn bench_insert_from_slice(b: &mut Bencher) { - let v: Vec = (0..SPILLED_SIZE as _).collect(); - b.iter(|| { - let mut vec = SmallVec::::new(); - vec.insert_from_slice(0, &v); - vec.insert_from_slice(0, &v); - vec - }); -} - #[bench] fn bench_macro_from_list(b: &mut Bencher) { b.iter(|| { diff --git a/fuzz/fuzz_targets/smallvec_ops.rs b/fuzz/fuzz_targets/smallvec_ops.rs index 10747a0..991793d 100644 --- a/fuzz/fuzz_targets/smallvec_ops.rs +++ b/fuzz/fuzz_targets/smallvec_ops.rs @@ -139,7 +139,7 @@ fn do_test(data: &[u8]) -> SmallVec { } } 22 => { - v = SmallVec::from_slice(data); + v = SmallVec::from(data); } 23 => { @@ -149,10 +149,10 @@ fn do_test(data: &[u8]) -> SmallVec { } 24 => { - if v.len() < CAP_GROWTH { - let insert_pos = next_usize!(bytes) % (v.len() + 1); - v.insert_from_slice(insert_pos, data); - } + let a = next_usize!(bytes) % (v.len() + 1); + let b = next_usize!(bytes) % (v.len() + 1); + let (start, end) = (a.min(b), a.max(b)); + v.extend_from_within(start..end); } 25 => { @@ -161,7 +161,7 @@ fn do_test(data: &[u8]) -> SmallVec { } } 26 => { - v = SmallVec::from_elem(next_u8!(bytes), next_usize!(bytes)); + v = smallvec::from_elem(next_u8!(bytes), next_usize!(bytes)); } _ => panic!("booo"), } diff --git a/src/lib.rs b/src/lib.rs index a2d1b7d..42a60ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -60,7 +60,7 @@ #![no_std] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(feature = "specialization", allow(incomplete_features))] -#![cfg_attr(feature = "specialization", feature(specialization))] +#![cfg_attr(feature = "specialization", feature(specialization, trusted_len))] #![cfg_attr(feature = "may_dangle", feature(dropck_eyepatch))] #[doc(hidden)] @@ -144,6 +144,41 @@ fn infallible(result: Result) -> T { } } +#[inline] +/// A local copy of [`core::slice::range`]. The latter function is unstable +/// and thus cannot be used yet. +fn slice_range(range: R, bounds: core::ops::RangeTo) -> core::ops::Range +where + R: core::ops::RangeBounds, +{ + let len = bounds.end; + + let start = match range.start_bound() { + core::ops::Bound::Included(&start) => start, + core::ops::Bound::Excluded(start) => start + .checked_add(1) + .unwrap_or_else(|| panic!("attempted to index slice from after maximum usize")), + core::ops::Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + core::ops::Bound::Included(end) => end + .checked_add(1) + .unwrap_or_else(|| panic!("attempted to index slice up to maximum usize")), + core::ops::Bound::Excluded(&end) => end, + core::ops::Bound::Unbounded => len, + }; + + if start > end { + panic!("slice index starts at {start} but ends at {end}"); + } + if end > len { + panic!("range end index {end} out of range for slice of length {len}"); + } + + core::ops::Range { start, end } +} + impl RawSmallVec { #[inline] const fn is_zst() -> bool { @@ -306,6 +341,13 @@ pub struct SmallVec { unsafe impl Send for SmallVec {} unsafe impl Sync for SmallVec {} +impl Default for SmallVec { + #[inline] + fn default() -> Self { + Self::new() + } +} + /// An iterator that removes the items from a `SmallVec` and yields them by value. /// /// Returned from [`SmallVec::drain`][1]. @@ -592,9 +634,7 @@ where ::Item: Debug, { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_tuple("Splice") - .field(&self.drain) - .finish() + f.debug_tuple("Splice").field(&self.drain).finish() } } @@ -626,7 +666,7 @@ impl Drop for Splice<'_, I, N> { // Which means we can replace the slice::Iter with pointers that won't point to deallocated // memory, so that Drain::drop is still allowed to call iter.len(), otherwise it would break // the ptr.sub_ptr contract. - self.drain.iter = (&[]).iter(); + self.drain.iter = [].iter(); unsafe { if self.drain.tail_len == 0 { @@ -1047,22 +1087,8 @@ impl SmallVec { where R: core::ops::RangeBounds, { - use core::ops::Bound::*; - let len = self.len(); - let start = match range.start_bound() { - Included(&n) => n, - Excluded(&n) => n.checked_add(1).expect("Range start out of bounds"), - Unbounded => 0, - }; - let end = match range.end_bound() { - Included(&n) => n.checked_add(1).expect("Range end out of bounds"), - Excluded(&n) => n, - Unbounded => len, - }; - - assert!(start <= end); - assert!(end <= len); + let core::ops::Range { start, end } = slice_range(range, ..len); unsafe { // SAFETY: `start <= len` @@ -1104,7 +1130,7 @@ impl SmallVec { /// # use smallvec::SmallVec; /// # use std::cmp::min; /// # let some_predicate = |x: &mut i32| { *x == 2 || *x == 3 || *x == 6 }; - /// # let mut vec: SmallVec = SmallVec::from_slice(&[1i32, 2, 3, 4, 5, 6]); + /// # let mut vec: SmallVec = SmallVec::from(&[1i32, 2, 3, 4, 5, 6]); /// # let range = 1..4; /// let mut i = 0; /// while i < min(vec.len(), range.end) { @@ -1116,7 +1142,7 @@ impl SmallVec { /// } /// } /// - /// # assert_eq!(vec, SmallVec::::from_slice(&[1i32, 4, 5])); + /// # assert_eq!(vec, SmallVec::::from(&[1i32, 4, 5])); /// ``` /// /// But `extract_if` is easier to use. `extract_if` is also more efficient, @@ -1135,22 +1161,22 @@ impl SmallVec { /// /// ``` /// # use smallvec::SmallVec; - /// let mut numbers: SmallVec = SmallVec::from_slice(&[1i32, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 15]); + /// let mut numbers: SmallVec = SmallVec::from(&[1i32, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 15]); /// /// let evens = numbers.extract_if(.., |x| *x % 2 == 0).collect::>(); /// let odds = numbers; /// - /// assert_eq!(evens, SmallVec::::from_slice(&[2i32, 4, 6, 8, 14])); - /// assert_eq!(odds, SmallVec::::from_slice(&[1i32, 3, 5, 9, 11, 13, 15])); + /// assert_eq!(evens, SmallVec::::from(&[2i32, 4, 6, 8, 14])); + /// assert_eq!(odds, SmallVec::::from(&[1i32, 3, 5, 9, 11, 13, 15])); /// ``` /// /// Using the range argument to only process a part of the vector: /// /// ``` /// # use smallvec::SmallVec; - /// let mut items: SmallVec = SmallVec::from_slice(&[0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 2, 1, 2]); + /// let mut items: SmallVec = SmallVec::from(&[0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 2, 1, 2]); /// let ones = items.extract_if(7.., |x| *x == 1).collect::>(); - /// assert_eq!(items, SmallVec::::from_slice(&[0, 0, 0, 0, 0, 0, 0, 2, 2, 2])); + /// assert_eq!(items, SmallVec::::from(&[0, 0, 0, 0, 0, 0, 0, 2, 2, 2])); /// assert_eq!(ones.len(), 3); /// ``` pub fn extract_if(&mut self, range: R, filter: F) -> ExtractIf<'_, T, N, F> @@ -1159,36 +1185,7 @@ impl SmallVec { R: core::ops::RangeBounds, { let old_len = self.len(); - // This line can be used instead once `core::slice::range` is stable. - //let core::ops::Range { start, end } = core::slice::range(range, ..old_len); - let (start, end) = { - let len = old_len; - - let start = match range.start_bound() { - core::ops::Bound::Included(&start) => start, - core::ops::Bound::Excluded(start) => { - start.checked_add(1).unwrap_or_else(|| panic!("attempted to index slice from after maximum usize")) - } - core::ops::Bound::Unbounded => 0, - }; - - let end = match range.end_bound() { - core::ops::Bound::Included(end) => { - end.checked_add(1).unwrap_or_else(|| panic!("attempted to index slice up to maximum usize")) - } - core::ops::Bound::Excluded(&end) => end, - core::ops::Bound::Unbounded => len, - }; - - if start > end { - panic!("slice index starts at {start} but ends at {end}"); - } - if end > len { - panic!("range end index {end} out of range for slice of length {len}"); - } - - (start, end) - }; + let core::ops::Range { start, end } = slice_range(range, ..old_len); // Guard against us getting leaked (leak amplification) unsafe { @@ -1773,139 +1770,127 @@ impl SmallVec { _marker: PhantomData, } } +} - fn extend_impl>(&mut self, iter: I) { - let mut iter = iter.fuse(); - let (lower_bound, _) = iter.size_hint(); - self.reserve(lower_bound); - let mut capacity = self.capacity(); - let mut ptr = self.as_mut_ptr(); - unsafe { - loop { - let mut len = self.len(); - // SAFETY: ptr is valid for `capacity - len` writes - ptr = ptr.add(len); - let mut guard = DropGuard { ptr, len: 0 }; - iter.by_ref().take(capacity - len).for_each(|item| { - ptr.add(guard.len).write(item); - guard.len += 1; - }); - len += guard.len; - core::mem::forget(guard); - self.set_len(len); - // At this point we either consumed all capacity or the iterator is exhausted (fused) - if let Some(item) = iter.next() { - self.push(item); - } else { - return; - } - // SAFETY: The push above would have spilled it - let (heap_ptr, heap_capacity) = self.raw.heap; - ptr = heap_ptr.as_ptr(); - capacity = heap_capacity; - } +impl SmallVec { + #[inline] + pub fn resize(&mut self, len: usize, value: T) { + let old_len = self.len(); + if len > old_len { + self.extend(core::iter::repeat_n(value, len - old_len)); + } else { + self.truncate(len); } } -} -impl Default for SmallVec { #[inline] - fn default() -> Self { - Self::new() + pub fn extend_from_slice(&mut self, other: &[T]) { + self.extend(other.iter()) } -} -impl SmallVec { - #[inline] - pub fn from_slice(slice: &[T]) -> Self { - let len = slice.len(); - if len <= Self::inline_size() { - let mut this = Self::new(); - unsafe { - let ptr = this.raw.as_mut_ptr_inline(); - copy_nonoverlapping(slice.as_ptr(), ptr, len); - this.set_len(len); + pub fn extend_from_within(&mut self, src: R) + where + R: core::ops::RangeBounds, + { + let src = slice_range(src, ..self.len()); + self.reserve(src.len()); + + // SAFETY: The call to `reserve` ensures that the capacity is large enough. + // The range is within bounds through the use of `core::slice::range`. + unsafe { + #[cfg(feature = "specialization")] + { + >::spec_extend_from_within(self, src); } - this - } else { - let mut this = Vec::with_capacity(len); - unsafe { - let ptr = this.as_mut_ptr(); - copy_nonoverlapping(slice.as_ptr(), ptr, len); - this.set_len(len); + + #[cfg(not(feature = "specialization"))] + { + self.extend_from_within_fallback(src); } - Self::from_vec(this) } } #[inline] - pub fn insert_from_slice(&mut self, index: usize, slice: &[T]) { - let len = self.len(); - let other_len = slice.len(); - assert!(index <= len); - self.reserve(other_len); + pub fn extend_from_slice_copy(&mut self, other: &[T]) + where + T: Copy + { + + let len = other.len(); + let src = other.as_ptr(); + + let l = self.len(); + self.reserve(len); + + // SAFETY: Additional memory has been reserved, + // therefore the pointer access is valid. unsafe { - let base_ptr = self.as_mut_ptr(); - let ith_ptr = base_ptr.add(index); - let shifted_ptr = base_ptr.add(index + other_len); - // elements at `index + other_len..len + other_len` are now initialized - copy(ith_ptr, shifted_ptr, len - index); - // elements at `index..index + other_len` are now initialized - copy_nonoverlapping(slice.as_ptr(), ith_ptr, other_len); - - // SAFETY: all the elements are initialized - self.set_len(len + other_len); + let dst = self.as_mut_ptr().add(l); + copy_nonoverlapping(src, dst, len); + self.set_len(l + len); } } - #[inline] - pub fn extend_from_slice(&mut self, slice: &[T]) { - let len = self.len(); - let other_len = slice.len(); - self.reserve(other_len); - // SAFETY: see above + pub fn extend_from_within_copy(&mut self, src: R) + where + R: core::ops::RangeBounds, + T: Copy + { + let src = slice_range(src, ..self.len()); + let core::ops::Range { start, end } = src; + let len = end - start; + self.reserve(len); + + // SAFETY: The call to `reserve` ensures that the capacity is large enough. + // The range is within bounds through the use of `core::slice::range`. unsafe { - let base_ptr = self.as_mut_ptr(); - let end_ptr = base_ptr.add(len); - copy_nonoverlapping(slice.as_ptr(), end_ptr, other_len); - self.set_len(len + other_len); + let l = self.len(); + let ptr = self.as_mut_ptr(); + copy_nonoverlapping(ptr.add(start), ptr.add(l), len); + self.set_len(l + len); } } -} -impl SmallVec { - #[inline] - pub fn resize(&mut self, len: usize, value: T) { - let old_len = self.len(); - if len > old_len { - self.extend(core::iter::repeat_n(value, len - old_len)); - } else { - self.truncate(len); + pub fn insert_from_slice_copy(&mut self, index: usize, other: &[T]) + where + T: Copy + { + let l = self.len(); + let len = other.len(); + assert!(index <= l); + self.reserve(len); + unsafe { + let base_ptr = self.as_mut_ptr(); + let ith_ptr = base_ptr.add(index); + let shifted_ptr = base_ptr.add(index + len); + // elements at `index + other_len..len + other_len` are now initialized + copy(ith_ptr, shifted_ptr, l - index); + // elements at `index..index + other_len` are now initialized + copy_nonoverlapping(other.as_ptr(), ith_ptr, len); + + // SAFETY: all the elements are initialized + self.set_len(l + len); } } - #[inline] - pub fn from_elem(elem: T, n: usize) -> Self { - if n > Self::inline_size() { - Self::from_vec(vec![elem; n]) - } else { - let mut v = Self::new(); - - unsafe { - let ptr = v.raw.as_mut_ptr_inline(); - let mut guard = DropGuard { ptr, len: 0 }; + /// A function for creating [`SmallVec`] values out of slices + /// for types with the [`Copy`] trait. + pub fn from_slice_copy(slice: &[T]) -> Self + where + T: Copy + { + let src = slice.as_ptr(); + let len = slice.len(); + let mut result = Self::with_capacity(len); - // SAFETY: `n <= Self::inline_size()` so we can write `n` elements - for i in 0..n { - guard.len = i; - ptr.add(i).write(elem.clone()); - } - core::mem::forget(guard); - // SAFETY: we just initialized `n` elements in the vector - v.set_len(n); - } - v + // SAFETY: By using `with_capacity`, the pointer will point to valid memory. + unsafe { + let dst = result.as_mut_ptr(); + copy_nonoverlapping(src, dst, len); + result.set_len(len); } + + result } } @@ -1922,13 +1907,6 @@ impl Drop for DropGuard { } } -impl Extend for SmallVec { - #[inline] - fn extend>(&mut self, iterable: I) { - self.extend_impl(iterable.into_iter()); - } -} - struct DropDealloc { ptr: NonNull, size_bytes: usize, @@ -2035,48 +2013,539 @@ impl core::ops::DerefMut for SmallVec { } } -impl core::iter::FromIterator for SmallVec { - #[inline] - fn from_iter>(iterable: I) -> Self { - let mut vec = Self::new(); - vec.extend_impl(iterable.into_iter()); - vec +/// This function is used in the [`smallvec`] macro. +/// It is recommended to use the macro instead of using thís function. +#[doc(hidden)] +#[track_caller] +pub fn from_elem(elem: T, n: usize) -> SmallVec { + if n > SmallVec::::inline_size() { + // Standard Rust vectors are already specialized. + SmallVec::::from_vec(vec![elem; n]) + } else { + #[cfg(feature = "specialization")] + { + // SAFETY: The precondition is checked in the initial comparison above. + unsafe { as spec_traits::SpecFromElem>::spec_from_elem(elem, n) } + } + + #[cfg(not(feature = "specialization"))] + { + // SAFETY: The precondition is checked in the initial comparison above. + unsafe { SmallVec::::from_elem_fallback(elem, n) } + } } } #[cfg(feature = "specialization")] -trait SpecFrom { - type Element; - fn spec_from(slice: &[Self::Element]) -> Self; +mod spec_traits { + use super::*; + + /// A trait for specializing the implementation of [`from_elem`]. + /// + /// [`from_elem`]: crate::from_elem + pub(crate) trait SpecFromElem { + /// Creates a `Smallvec` value where `elem` is repeated `n` times. + /// This will use the inline storage, not the heap. + /// + /// # Safety + /// + /// The caller must ensure that `n <= Self::inline_size()`. + unsafe fn spec_from_elem(elem: T, n: usize) -> Self; + } + + impl SpecFromElem for SmallVec { + #[inline] + default unsafe fn spec_from_elem(elem: T, n: usize) -> Self { + // SAFETY: Safety conditions are identical. + unsafe { SmallVec::from_elem_fallback(elem, n) } + } + } + + impl SpecFromElem for SmallVec { + unsafe fn spec_from_elem(elem: T, n: usize) -> Self { + let mut result = Self::new(); + + if n > 0 { + let ptr = result.raw.as_mut_ptr_inline(); + + // SAFETY: The caller ensures that the first `n` + // is smaller than the inline size. + unsafe { + for i in 0..n { + ptr.add(i).write(elem); + } + } + } + + // SAFETY: The first `n` elements of the vector + // have been initialized in the loop above. + unsafe { + result.set_len(n); + } + + result + } + } + + /// A trait for specializing the implementations of [`Extend`] and [`extend_from_slice`]. + /// + /// [`extend_from_slice`]: crate::SmallVec::extend_from_slice + pub(crate) trait SpecExtend { + fn spec_extend(&mut self, iter: I); + } + + impl SpecExtend for SmallVec + where + I: Iterator, + { + #[inline] + default fn spec_extend(&mut self, iter: I) { + self.extend_fallback(iter); + } + } + + impl SpecExtend for SmallVec + where + I: core::iter::TrustedLen, + { + fn spec_extend(&mut self, iter: I) { + let (_, Some(additional)) = iter.size_hint() else { + panic!("capacity overflow") + }; + self.reserve(additional); + + // SAFETY: A `TrustedLen` iterator provides accurate information + // about its size, which was used to reserve additional memory. + // This ensures that the access operations inside the loop always + // operate on valid memory. + unsafe { + let len = self.len(); + let ptr = self.as_mut_ptr().add(len); + let mut guard = DropGuard { ptr, len: 0 }; + + for x in iter { + ptr.add(guard.len).write(x); + guard.len += 1; + } + + // The elements have been initialized in the loop above. + self.set_len(len + guard.len); + core::mem::forget(guard); + } + } + } + + impl SpecExtend> for SmallVec { + fn spec_extend(&mut self, mut iter: IntoIter) { + let slice = iter.as_slice(); + let len = slice.len(); + let old_len = self.len(); + + self.reserve(len); + + // SAFETY: Additional memory has been reserved above. + // Therefore, the copy operates on valid memory. + unsafe { + let dst = self.as_mut_ptr().add(old_len); + let src = slice.as_ptr(); + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + self.set_len(old_len + len); + } + + // Mark the iterator as fully consumed. + iter.begin = iter.end.value(Self::is_zst()); + } + } + + impl<'a, T: 'a, const N: usize, I> SpecExtend<&'a T, I> for SmallVec + where + I: Iterator, + T: Clone, + { + #[inline] + default fn spec_extend(&mut self, iterator: I) { + self.spec_extend(iterator.cloned()) + } + } + + impl<'a, T: 'a, const N: usize> SpecExtend<&'a T, core::slice::Iter<'a, T>> for SmallVec + where + T: Copy, + { + fn spec_extend(&mut self, iter: core::slice::Iter<'a, T>) { + let slice = iter.as_slice(); + let len = slice.len(); + let old_len = self.len(); + + self.reserve(len); + + // SAFETY: Additional memory has been reserved above. + // Therefore, the copy operates on valid memory. + unsafe { + let dst = self.as_mut_ptr().add(old_len); + let src = slice.as_ptr(); + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + self.set_len(old_len + len); + } + } + } + + /// A trait for specializing the implementation of [`extend_from_within`]. + /// + /// [`extend_from_within`]: crate::SmallVec::extend_from_within + pub(crate) trait SpecExtendFromWithin { + /// Main worker for [`extend_from_within`]. + /// + /// # Safety + /// + /// * The length of the vector is larger than or equal to `src.len()`. + /// * The spare capacity of the vector is larger than or equal to `src.len()`. + /// + /// [`extend_from_within`]: SmallVec::extend_from_within + unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range); + } + + impl SpecExtendFromWithin for SmallVec { + default unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { + // SAFETY: Safety conditions are identical. + unsafe { + self.extend_from_within_fallback(src); + } + } + } + + impl SpecExtendFromWithin for SmallVec { + unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { + let old_len = self.len(); + + let start = src.start; + let len = src.len(); + + // SAFETY: The caller ensures that the vector has spare capacity + // for at least `src.len()` elements. This is alse the amount of memory + // accessed when the data is copied. + unsafe { + let ptr = self.as_mut_ptr(); + let dst = ptr.add(old_len); + let src = ptr.add(start); + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + self.set_len(old_len + len); + } + } + } + + /// A trait for specializing the implementation of [`FromIterator`]. + /// + /// [`clone_from`]: Clone::clone_from + pub(crate) trait SpecFromIterator { + fn spec_from_iter(iter: I) -> Self; + } + + impl SpecFromIterator for SmallVec + where + I: Iterator, + { + #[inline] + default fn spec_from_iter(iter: I) -> Self { + Self::from_iter_fallback(iter) + } + } + + impl SpecFromIterator for SmallVec + where + I: core::iter::TrustedLen, + { + fn spec_from_iter(iter: I) -> Self { + let mut v = match iter.size_hint() { + (_, Some(upper)) => SmallVec::with_capacity(upper), + // TrustedLen contract guarantees that `size_hint() == (_, None)` means that there + // are more than `usize::MAX` elements. + // Since the previous branch would eagerly panic if the capacity is too large + // (via `with_capacity`) we do the same here. + _ => panic!("capacity overflow"), + }; + // Reuse the extend specialization for TrustedLen. + v.spec_extend(iter); + v + } + } + + /// A trait for specializing the implementation of [`clone_from`]. + /// + /// [`clone_from`]: Clone::clone_from + pub(crate) trait SpecCloneFrom { + fn spec_clone_from(&mut self, source: &[T]); + } + + impl SpecCloneFrom for SmallVec { + #[inline] + default fn spec_clone_from(&mut self, source: &[T]) { + self.clone_from_fallback(source); + } + } + + impl SpecCloneFrom for SmallVec { + fn spec_clone_from(&mut self, source: &[T]) { + self.clear(); + self.extend_from_slice(source); + } + } + + /// A trait for specializing the implementation of [`From`] + /// with the source type being slices. + pub(crate) trait SpecFromSlice { + /// Creates a `SmallVec` value based on the contents of `slice`. + /// This will use the inline storage, not the heap. + /// + /// # Safety + /// + /// The caller must ensure that `slice.len() <= Self::inline_size()`. + unsafe fn spec_from(slice: &[T]) -> Self; + } + + impl SpecFromSlice for SmallVec { + default unsafe fn spec_from(slice: &[T]) -> Self { + // SAFETY: Safety conditions are identical. + unsafe { Self::from_slice_fallback(slice) } + } + } + + impl SpecFromSlice for SmallVec { + unsafe fn spec_from(slice: &[T]) -> Self { + let mut v = Self::new(); + + let src = slice.as_ptr(); + let len = slice.len(); + let dst = v.as_mut_ptr(); + + // SAFETY: The caller ensures that the slice length is smaller + // than or equal to the inline length. + unsafe { + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + v.set_len(len); + } + + v + } + } } -#[cfg(feature = "specialization")] -impl SpecFrom for SmallVec { - type Element = T; +/// Fallback functions for various specialized methods. These are kept in +/// a separate implementation block for easy access whenever specialization is disabled. +impl SmallVec { + /// Creates a `Smallvec` value where `elem` is repeated `n` times. + /// This will use the inline storage, not the heap. + /// + /// # Safety + /// + /// The caller must ensure that `n <= Self::inline_size()`. + unsafe fn from_elem_fallback(elem: T, n: usize) -> Self + where + T: Clone, + { + let mut result = Self::new(); + + if n > 0 { + let ptr = result.raw.as_mut_ptr_inline(); + let mut guard = DropGuard { ptr, len: 0 }; + + // SAFETY: The caller ensures that the first `n` + // is smaller than the inline size. + unsafe { + for i in 0..(n - 1) { + ptr.add(i).write(elem.clone()); + guard.len += 1; + } + core::mem::forget(guard); + ptr.add(n - 1).write(elem); + } + } + + // SAFETY: The first `n` elements of the vector + // have been initialized in the loop above. + unsafe { + result.set_len(n); + } + + result + } + + fn extend_fallback(&mut self, iter: I) + where + I: IntoIterator, + { + let iter = iter.into_iter(); + let (size, _) = iter.size_hint(); + self.reserve(size); + for x in iter { + self.push(x); + } + } + + /// Main worker for [`extend_from_within`]. + /// + /// # Safety + /// + /// * The length of the vector is larger than or equal to `src.len()`. + /// * The spare capacity of the vector is larger than or equal to `src.len()`. + /// + /// [`extend_from_within`]: SmallVec::extend_from_within + unsafe fn extend_from_within_fallback(&mut self, src: core::ops::Range) + where + T: Clone, + { + let old_len = self.len(); + + let start = src.start; + let len = src.len(); + + // SAFETY: The caller ensures that the vector has spare capacity + // for at least `src.len()` elements. This implies that the loop + // operates on valid memory. + unsafe { + let ptr = self.as_mut_ptr(); + let dst = ptr.add(old_len); + let src = ptr.add(start); + + let mut guard = DropGuard { ptr: dst, len: 0 }; + for i in 0..len { + let val = (*src.add(i)).clone(); + dst.add(i).write(val); + guard.len += 1; + } + core::mem::forget(guard); + } + + // SAFETY: The elements were initialized in the loop above. + unsafe { + self.set_len(old_len + len); + } + } + + fn from_iter_fallback(iter: I) -> Self + where + I: Iterator, + { + let (size, _) = iter.size_hint(); + let mut v = Self::with_capacity(size); + for x in iter { + v.push(x); + } + v + } + + fn clone_from_fallback(&mut self, source: &[T]) + where + T: Clone, + { + // Inspired from `impl Clone for Vec`. + + // Drop anything that will not be overwritten. + self.truncate(source.len()); + + // SAFETY: self.len <= other.len due to the truncate above, so the + // slices here are always in-bounds. + let (init, tail) = unsafe { source.split_at_unchecked(self.len()) }; - default fn spec_from(slice: &[Self::Element]) -> Self { - slice.iter().cloned().collect() + // Reuse the contained values' allocations/resources. + self.clone_from_slice(init); + self.extend(tail.iter().cloned()); + } + + /// Creates a `SmallVec` value based on the contents of `slice`. + /// This will use the inline storage, not the heap. + /// + /// # Safety + /// + /// The caller must ensure that `slice.len() <= Self::inline_size()`. + unsafe fn from_slice_fallback(slice: &[T]) -> Self + where + T: Clone, + { + let mut v = Self::new(); + + let src = slice.as_ptr(); + let len = slice.len(); + let dst = v.as_mut_ptr(); + + // SAFETY: The caller ensures that the slice length is smaller + // than or equal to the inline length. + unsafe { + let mut guard = DropGuard { ptr: dst, len: 0 }; + for i in 0..len { + let val = (*src.add(i)).clone(); + dst.add(i).write(val); + guard.len += 1; + } + core::mem::forget(guard); + } + + // SAFETY: The elements were initialized in the loop above. + unsafe { + v.set_len(len); + } + + v } } -#[cfg(feature = "specialization")] -impl SpecFrom for SmallVec { - fn spec_from(slice: &[Self::Element]) -> Self { - Self::from_slice(slice) +impl From<&[T]> for SmallVec { + #[inline] + fn from(slice: &[T]) -> Self { + if slice.len() > Self::inline_size() { + // Standard Rust vectors are already specialized. + Self::from_vec(Vec::from(slice)) + } else { + // SAFETY: The precondition is checked in the initial comparison above. + unsafe { + #[cfg(feature = "specialization")] + { + >::spec_from(slice) + } + + #[cfg(not(feature = "specialization"))] + { + Self::from_slice_fallback(slice) + } + } + } } } -#[cfg(feature = "specialization")] -impl<'a, T: Clone, const N: usize> From<&'a [T]> for SmallVec { - fn from(slice: &'a [T]) -> Self { - ::spec_from(slice) +impl From<&mut [T]> for SmallVec { + #[inline] + fn from(slice: &mut [T]) -> Self { + Self::from(slice as &[T]) } } -#[cfg(not(feature = "specialization"))] -impl<'a, T: Clone, const N: usize> From<&'a [T]> for SmallVec { - fn from(slice: &'a [T]) -> Self { - slice.iter().cloned().collect() +impl From<&[T; M]> for SmallVec { + #[inline] + fn from(slice: &[T; M]) -> Self { + Self::from(slice as &[T]) + } +} + +impl From<&mut [T; M]> for SmallVec { + #[inline] + fn from(slice: &mut [T; M]) -> Self { + Self::from(slice as &[T]) } } @@ -2084,7 +2553,7 @@ impl From<[T; M]> for SmallVec { fn from(array: [T; M]) -> Self { if M > N { // If M > N, we'd have to heap allocate anyway, - // so delegate for Vec for the allocation + // so delegate for Vec for the allocation. Self::from(Vec::from(array)) } else { // M <= N @@ -2100,6 +2569,7 @@ impl From<[T; M]> for SmallVec { } } } + impl From> for SmallVec { fn from(array: Vec) -> Self { Self::from_vec(array) @@ -2112,20 +2582,17 @@ impl Clone for SmallVec { SmallVec::from(self.as_slice()) } + #[inline] fn clone_from(&mut self, source: &Self) { - // Inspired from `impl Clone for Vec`. - - // drop anything that will not be overwritten - self.truncate(source.len()); - - // SAFETY: self.len <= other.len due to the truncate above, so the - // slices here are always in-bounds. - let init = unsafe { source.get_unchecked(..self.len()) }; - let tail = unsafe { source.get_unchecked(self.len()..) }; + #[cfg(feature = "specialization")] + { + >::spec_clone_from(self, source); + } - // reuse the contained values' allocations/resources. - self.clone_from_slice(init); - self.extend(tail.iter().cloned()); + #[cfg(not(feature = "specialization"))] + { + self.clone_from_fallback(&*source); + } } } @@ -2136,6 +2603,51 @@ impl Clone for IntoIter { } } +impl Extend for SmallVec { + #[inline] + fn extend>(&mut self, iter: I) { + #[cfg(feature = "specialization")] + { + spec_traits::SpecExtend::::spec_extend(self, iter.into_iter()); + } + + #[cfg(not(feature = "specialization"))] + { + self.extend_fallback(iter); + } + } +} + +impl<'a, T: Clone + 'a, const N: usize> Extend<&'a T> for SmallVec { + #[inline] + fn extend>(&mut self, iter: I) { + #[cfg(feature = "specialization")] + { + spec_traits::SpecExtend::<&'a T, _>::spec_extend(self, iter.into_iter()); + } + + #[cfg(not(feature = "specialization"))] + { + self.extend_fallback(iter.into_iter().cloned()); + } + } +} + +impl core::iter::FromIterator for SmallVec { + #[inline] + fn from_iter>(iter: I) -> Self { + #[cfg(feature = "specialization")] + { + spec_traits::SpecFromIterator::::spec_from_iter(iter.into_iter()) + } + + #[cfg(not(feature = "specialization"))] + { + Self::from_iter_fallback(iter.into_iter()) + } + } +} + #[macro_export] macro_rules! smallvec { // count helper: transform any expression into 1 @@ -2144,12 +2656,12 @@ macro_rules! smallvec { $crate::SmallVec::new() ); ($elem:expr; $n:expr) => ({ - $crate::SmallVec::from_elem($elem, $n) + $crate::from_elem($elem, $n) }); ($($x:expr),+$(,)?) => ({ - let count = 0usize $(+ $crate::smallvec!(@one $x))+; + const COUNT: usize = 0usize $(+ $crate::smallvec!(@one $x))+; let mut vec = $crate::SmallVec::new(); - if count <= vec.capacity() { + if COUNT <= vec.capacity() { $(vec.push($x);)* vec } else { diff --git a/src/tests.rs b/src/tests.rs index 18665b4..2082f96 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -374,30 +374,26 @@ fn drain_overflow() { } #[test] -fn test_insert_from_slice() { +fn test_extend_from_slice() { let mut v: SmallVec = SmallVec::new(); for x in 0..4 { v.push(x); } assert_eq!(v.len(), 4); - v.insert_from_slice(1, &[5, 6]); + v.extend_from_slice(&[5, 6]); assert_eq!( &v.iter().map(|v| *v).collect::>(), - &[0, 5, 6, 1, 2, 3] + &[0, 1, 2, 3, 5, 6] ); } #[test] -fn test_extend_from_slice() { - let mut v: SmallVec = SmallVec::new(); - for x in 0..4 { - v.push(x); - } - assert_eq!(v.len(), 4); - v.extend_from_slice(&[5, 6]); +fn test_extend_from_within() { + let mut v: SmallVec = smallvec![0, 1, 2, 3]; + v.extend_from_within(1..3); assert_eq!( &v.iter().map(|v| *v).collect::>(), - &[0, 1, 2, 3, 5, 6] + &[0, 1, 2, 3, 1, 2], ); } @@ -590,9 +586,9 @@ fn test_from() { #[test] fn test_from_slice() { - assert_eq!(&SmallVec::::from_slice(&[1][..])[..], [1]); + assert_eq!(&SmallVec::::from(&[1][..])[..], [1]); assert_eq!( - &SmallVec::::from_slice(&[1, 2, 3][..])[..], + &SmallVec::::from(&[1, 2, 3][..])[..], [1, 2, 3] ); } @@ -723,7 +719,7 @@ fn test_from_vec() { #[test] fn test_retain() { // Test inline data storage - let mut sv: SmallVec = SmallVec::from_slice(&[1, 2, 3, 3, 4]); + let mut sv: SmallVec = SmallVec::from(&[1, 2, 3, 3, 4]); sv.retain(|&i| i != 3); assert_eq!(sv.pop(), Some(4)); assert_eq!(sv.pop(), Some(2)); @@ -731,7 +727,7 @@ fn test_retain() { assert_eq!(sv.pop(), None); // Test spilled data storage - let mut sv: SmallVec = SmallVec::from_slice(&[1, 2, 3, 3, 4]); + let mut sv: SmallVec = SmallVec::from(&[1, 2, 3, 3, 4]); sv.retain(|&i| i != 3); assert_eq!(sv.pop(), Some(4)); assert_eq!(sv.pop(), Some(2)); @@ -757,7 +753,7 @@ fn test_retain() { #[test] fn test_dedup() { - let mut dupes: SmallVec = SmallVec::from_slice(&[1, 1, 2, 3, 3]); + let mut dupes: SmallVec = SmallVec::from(&[1, 1, 2, 3, 3]); dupes.dedup(); assert_eq!(&*dupes, &[1, 2, 3]); @@ -765,11 +761,11 @@ fn test_dedup() { empty.dedup(); assert!(empty.is_empty()); - let mut all_ones: SmallVec = SmallVec::from_slice(&[1, 1, 1, 1, 1]); + let mut all_ones: SmallVec = SmallVec::from(&[1, 1, 1, 1, 1]); all_ones.dedup(); assert_eq!(all_ones.len(), 1); - let mut no_dupes: SmallVec = SmallVec::from_slice(&[1, 2, 3, 4, 5]); + let mut no_dupes: SmallVec = SmallVec::from(&[1, 2, 3, 4, 5]); no_dupes.dedup(); assert_eq!(no_dupes.len(), 5); } @@ -940,8 +936,8 @@ fn test_extract_if() { let b: SmallVec = a.extract_if(1..9, |x| *x % 3 == 0).collect(); - assert_eq!(a, SmallVec::::from_slice(&[0, 1u8, 2, 4, 5, 7, 8, 0])); - assert_eq!(b, SmallVec::::from_slice(&[3u8, 6])); + assert_eq!(a, SmallVec::::from(&[0, 1u8, 2, 4, 5, 7, 8, 0])); + assert_eq!(b, SmallVec::::from(&[3u8, 6])); } /// This assortment of tests, in combination with miri, verifies we handle UB on fishy arguments