From 58431aff0a1e1bccc785dc2e200ee1ab10efb82b Mon Sep 17 00:00:00 2001 From: Pedro Nobre Date: Thu, 3 Sep 2026 20:41:26 +0100 Subject: [PATCH 1/2] refactor: modularized `mod spec_traits` into a `specialization.rs` file --- src/conversions.rs | 2 +- src/lib.rs | 326 ++---------------------------------------- src/specialization.rs | 313 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 324 insertions(+), 317 deletions(-) create mode 100644 src/specialization.rs diff --git a/src/conversions.rs b/src/conversions.rs index 5a387e1..4883bb2 100644 --- a/src/conversions.rs +++ b/src/conversions.rs @@ -19,7 +19,7 @@ impl From<&[T]> for SmallVec { unsafe { #[cfg(feature = "specialization")] { - >::spec_from(slice) + >::spec_from(slice) } #[cfg(not(feature = "specialization"))] diff --git a/src/lib.rs b/src/lib.rs index ba2302a..1e010aa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -84,6 +84,8 @@ mod rayon; mod references; #[cfg(feature = "serde")] mod serde; +#[cfg(feature = "specialization")] +mod specialization; mod taggedlen; #[cfg(feature = "bytes")] @@ -1880,7 +1882,9 @@ impl SmallVec { unsafe { #[cfg(feature = "specialization")] { - >::spec_extend_from_within(self, src); + >::spec_extend_from_within( + self, src + ); } #[cfg(not(feature = "specialization"))] @@ -2088,7 +2092,7 @@ pub fn from_elem(elem: T, n: usize) -> SmallVec { // SAFETY: The precondition is checked in the initial comparison // above. - unsafe { as spec_traits::SpecFromElem>::spec_from_elem(elem, n) } + unsafe { as specialization::SpecFromElem>::spec_from_elem(elem, n) } } #[cfg(not(feature = "specialization"))] @@ -2100,316 +2104,6 @@ pub fn from_elem(elem: T, n: usize) -> SmallVec } } -#[cfg(feature = "specialization")] -mod spec_traits { - use super::*; - - /// A trait for specializing the implementation of [`from_elem`]. - /// - /// [`from_elem`]: crate::from_elem - pub(crate) trait SpecFromElem { - /// Creates a `Smallvec` value where `elem` is repeated `n` times. - /// This will use the inline storage, not the heap. - /// - /// # Safety - /// - /// The caller must ensure that `n <= Self::inline_size()`. - unsafe fn spec_from_elem(elem: T, n: usize) -> Self; - } - - impl SpecFromElem for SmallVec { - #[inline] - default unsafe fn spec_from_elem(elem: T, n: usize) -> Self { - // SAFETY: Safety conditions are identical. - unsafe { SmallVec::from_elem_fallback(elem, n) } - } - } - - impl SpecFromElem for SmallVec { - unsafe fn spec_from_elem(elem: T, n: usize) -> Self { - let mut result = Self::new(); - - if n > 0 { - let ptr = result.raw.as_mut_ptr_inline(); - - // SAFETY: The caller ensures that the first `n` - // is smaller than the inline size. - unsafe { - for i in 0..n { - ptr.add(i).write(elem); - } - } - } - - // SAFETY: The first `n` elements of the vector - // have been initialized in the loop above. - unsafe { - result.set_len(n); - } - - result - } - } - - /// A trait for specializing the implementations of [`Extend`] and - /// [`extend_from_slice`]. - /// - /// [`extend_from_slice`]: crate::SmallVec::extend_from_slice - pub(crate) trait SpecExtend { - fn spec_extend(&mut self, iter: I); - } - - impl SpecExtend for SmallVec - where I: Iterator - { - #[inline] - default fn spec_extend(&mut self, iter: I) { - self.extend_fallback(iter); - } - } - - impl SpecExtend for SmallVec - where I: core::iter::TrustedLen - { - fn spec_extend(&mut self, iter: I) { - let (_, Some(additional)) = iter.size_hint() else { - panic!("capacity overflow") - }; - self.reserve(additional); - - // SAFETY: A `TrustedLen` iterator provides accurate information - // about its size, which was used to reserve additional memory. - // This ensures that the access operations inside the loop always - // operate on valid memory. - unsafe { - let len = self.len(); - let ptr = self.as_mut_ptr().add(len); - let mut guard = DropGuard { - ptr, - len: 0 - }; - - for x in iter { - ptr.add(guard.len).write(x); - guard.len += 1; - } - - // The elements have been initialized in the loop above. - self.set_len(len + guard.len); - core::mem::forget(guard); - } - } - } - - impl SpecExtend> for SmallVec { - fn spec_extend(&mut self, mut iter: IntoIter) { - let slice = iter.as_slice(); - let len = slice.len(); - let old_len = self.len(); - - self.reserve(len); - - // SAFETY: Additional memory has been reserved above. - // Therefore, the copy operates on valid memory. - unsafe { - let dst = self.as_mut_ptr().add(old_len); - let src = slice.as_ptr(); - copy_nonoverlapping(src, dst, len); - } - - // SAFETY: The elements were initialized above. - unsafe { - self.set_len(old_len + len); - } - - // Mark the iterator as fully consumed. - iter.begin = iter.end.value(); - } - } - - impl<'a, T: 'a, const N: usize, I> SpecExtend<&'a T, I> for SmallVec - where - I: Iterator, - T: Clone - { - #[inline] - default fn spec_extend(&mut self, iterator: I) { - self.spec_extend(iterator.cloned()) - } - } - - impl<'a, T: 'a, const N: usize> SpecExtend<&'a T, core::slice::Iter<'a, T>> for SmallVec - where T: Copy - { - fn spec_extend(&mut self, iter: core::slice::Iter<'a, T>) { - let slice = iter.as_slice(); - let len = slice.len(); - let old_len = self.len(); - - self.reserve(len); - - // SAFETY: Additional memory has been reserved above. - // Therefore, the copy operates on valid memory. - unsafe { - let dst = self.as_mut_ptr().add(old_len); - let src = slice.as_ptr(); - copy_nonoverlapping(src, dst, len); - } - - // SAFETY: The elements were initialized above. - unsafe { - self.set_len(old_len + len); - } - } - } - - /// A trait for specializing the implementation of [`extend_from_within`]. - /// - /// [`extend_from_within`]: crate::SmallVec::extend_from_within - pub(crate) trait SpecExtendFromWithin { - /// Main worker for [`extend_from_within`]. - /// - /// # Safety - /// - /// * The length of the vector is larger than or equal to `src.len()`. - /// * The spare capacity of the vector is larger than or equal to - /// `src.len()`. - /// - /// [`extend_from_within`]: SmallVec::extend_from_within - unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range); - } - - impl SpecExtendFromWithin for SmallVec { - default unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { - // SAFETY: Safety conditions are identical. - unsafe { - self.extend_from_within_fallback(src); - } - } - } - - impl SpecExtendFromWithin for SmallVec { - unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { - let old_len = self.len(); - - let start = src.start; - let len = src.len(); - - // SAFETY: The caller ensures that the vector has spare capacity - // for at least `src.len()` elements. This is also the amount of - // memory accessed when the data is copied. - unsafe { - let ptr = self.as_mut_ptr(); - let dst = ptr.add(old_len); - let src = ptr.add(start); - copy_nonoverlapping(src, dst, len); - } - - // SAFETY: The elements were initialized above. - unsafe { - self.set_len(old_len + len); - } - } - } - - /// A trait for specializing the implementation of [`FromIterator`]. - /// - /// [`clone_from`]: Clone::clone_from - pub(crate) trait SpecFromIterator { - fn spec_from_iter(iter: I) -> Self; - } - - impl SpecFromIterator for SmallVec - where I: Iterator - { - #[inline] - default fn spec_from_iter(iter: I) -> Self { - Self::from_iter_fallback(iter) - } - } - - impl SpecFromIterator for SmallVec - where I: core::iter::TrustedLen - { - fn spec_from_iter(iter: I) -> Self { - let mut v = match iter.size_hint() { - (_, Some(upper)) => SmallVec::with_capacity(upper), - // TrustedLen contract guarantees that `size_hint() == (_, None)` means that there - // are more than `usize::MAX` elements. - // Since the previous branch would eagerly panic if the capacity is too large - // (via `with_capacity`) we do the same here. - _ => panic!("capacity overflow") - }; - // Reuse the extend specialization for TrustedLen. - v.spec_extend(iter); - v - } - } - - /// A trait for specializing the implementation of [`clone_from`]. - /// - /// [`clone_from`]: Clone::clone_from - pub(crate) trait SpecCloneFrom { - fn spec_clone_from(&mut self, source: &[T]); - } - - impl SpecCloneFrom for SmallVec { - #[inline] - default fn spec_clone_from(&mut self, source: &[T]) { - self.clone_from_fallback(source); - } - } - - impl SpecCloneFrom for SmallVec { - fn spec_clone_from(&mut self, source: &[T]) { - self.clear(); - self.extend_from_slice(source); - } - } - - /// A trait for specializing the implementation of [`From`] - /// with the source type being slices. - pub(crate) trait SpecFromSlice { - /// Creates a `SmallVec` value based on the contents of `slice`. - /// This will use the inline storage, not the heap. - /// - /// # Safety - /// - /// The caller must ensure that `slice.len() <= Self::inline_size()`. - unsafe fn spec_from(slice: &[T]) -> Self; - } - - impl SpecFromSlice for SmallVec { - default unsafe fn spec_from(slice: &[T]) -> Self { - // SAFETY: Safety conditions are identical. - unsafe { Self::from_slice_fallback(slice) } - } - } - - impl SpecFromSlice for SmallVec { - unsafe fn spec_from(slice: &[T]) -> Self { - let mut v = Self::new(); - - let src = slice.as_ptr(); - let len = slice.len(); - let dst = v.as_mut_ptr(); - - // SAFETY: The caller ensures that the slice length is smaller - // than or equal to the inline length. - unsafe { - copy_nonoverlapping(src, dst, len); - } - - // SAFETY: The elements were initialized above. - unsafe { - v.set_len(len); - } - - v - } - } -} - /// Fallback functions for various specialized methods. These are kept in /// a separate implementation block for easy access whenever specialization is /// disabled. @@ -2620,7 +2314,7 @@ impl Clone for SmallVec { fn clone_from(&mut self, source: &Self) { #[cfg(feature = "specialization")] { - >::spec_clone_from(self, source); + >::spec_clone_from(self, source); } #[cfg(not(feature = "specialization"))] @@ -2642,7 +2336,7 @@ impl Extend for SmallVec { fn extend>(&mut self, iter: I) { #[cfg(feature = "specialization")] { - spec_traits::SpecExtend::::spec_extend(self, iter.into_iter()); + specialization::SpecExtend::::spec_extend(self, iter.into_iter()); } #[cfg(not(feature = "specialization"))] @@ -2657,7 +2351,7 @@ impl<'a, T: Clone + 'a, const N: usize> Extend<&'a T> for SmallVec { fn extend>(&mut self, iter: I) { #[cfg(feature = "specialization")] { - spec_traits::SpecExtend::<&'a T, _>::spec_extend(self, iter.into_iter()); + specialization::SpecExtend::<&'a T, _>::spec_extend(self, iter.into_iter()); } #[cfg(not(feature = "specialization"))] @@ -2672,7 +2366,7 @@ impl core::iter::FromIterator for SmallVec { fn from_iter>(iter: I) -> Self { #[cfg(feature = "specialization")] { - spec_traits::SpecFromIterator::::spec_from_iter(iter.into_iter()) + specialization::SpecFromIterator::::spec_from_iter(iter.into_iter()) } #[cfg(not(feature = "specialization"))] diff --git a/src/specialization.rs b/src/specialization.rs new file mode 100644 index 0000000..b197b51 --- /dev/null +++ b/src/specialization.rs @@ -0,0 +1,313 @@ +use { + crate::{ + DropGuard, + IntoIter, + SmallVec + }, + core::ptr::copy_nonoverlapping +}; + +/// A trait for specializing the implementation of [`from_elem`]. +/// +/// [`from_elem`]: crate::from_elem +pub(crate) trait SpecFromElem { + /// Creates a `Smallvec` value where `elem` is repeated `n` times. + /// This will use the inline storage, not the heap. + /// + /// # Safety + /// + /// The caller must ensure that `n <= Self::inline_size()`. + unsafe fn spec_from_elem(elem: T, n: usize) -> Self; +} + +impl SpecFromElem for SmallVec { + #[inline] + default unsafe fn spec_from_elem(elem: T, n: usize) -> Self { + // SAFETY: Safety conditions are identical. + unsafe { SmallVec::from_elem_fallback(elem, n) } + } +} + +impl SpecFromElem for SmallVec { + unsafe fn spec_from_elem(elem: T, n: usize) -> Self { + let mut result = Self::new(); + + if n > 0 { + let ptr = result.raw.as_mut_ptr_inline(); + + // SAFETY: The caller ensures that the first `n` + // is smaller than the inline size. + unsafe { + for i in 0..n { + ptr.add(i).write(elem); + } + } + } + + // SAFETY: The first `n` elements of the vector + // have been initialized in the loop above. + unsafe { + result.set_len(n); + } + + result + } +} + +/// A trait for specializing the implementations of [`Extend`] and +/// [`extend_from_slice`]. +/// +/// [`extend_from_slice`]: crate::SmallVec::extend_from_slice +pub(crate) trait SpecExtend { + fn spec_extend(&mut self, iter: I); +} + +impl SpecExtend for SmallVec +where I: Iterator +{ + #[inline] + default fn spec_extend(&mut self, iter: I) { + self.extend_fallback(iter); + } +} + +impl SpecExtend for SmallVec +where I: core::iter::TrustedLen +{ + fn spec_extend(&mut self, iter: I) { + let (_, Some(additional)) = iter.size_hint() else { + panic!("capacity overflow") + }; + self.reserve(additional); + + // SAFETY: A `TrustedLen` iterator provides accurate information + // about its size, which was used to reserve additional memory. + // This ensures that the access operations inside the loop always + // operate on valid memory. + unsafe { + let len = self.len(); + let ptr = self.as_mut_ptr().add(len); + let mut guard = DropGuard { + ptr, + len: 0 + }; + + for x in iter { + ptr.add(guard.len).write(x); + guard.len += 1; + } + + // The elements have been initialized in the loop above. + self.set_len(len + guard.len); + core::mem::forget(guard); + } + } +} + +impl SpecExtend> for SmallVec { + fn spec_extend(&mut self, mut iter: IntoIter) { + let slice = iter.as_slice(); + let len = slice.len(); + let old_len = self.len(); + + self.reserve(len); + + // SAFETY: Additional memory has been reserved above. + // Therefore, the copy operates on valid memory. + unsafe { + let dst = self.as_mut_ptr().add(old_len); + let src = slice.as_ptr(); + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + self.set_len(old_len + len); + } + + // Mark the iterator as fully consumed. + iter.begin = iter.end.value(); + } +} + +impl<'a, T: 'a, const N: usize, I> SpecExtend<&'a T, I> for SmallVec +where + I: Iterator, + T: Clone +{ + #[inline] + default fn spec_extend(&mut self, iterator: I) { + self.spec_extend(iterator.cloned()) + } +} + +impl<'a, T: 'a, const N: usize> SpecExtend<&'a T, core::slice::Iter<'a, T>> for SmallVec +where T: Copy +{ + fn spec_extend(&mut self, iter: core::slice::Iter<'a, T>) { + let slice = iter.as_slice(); + let len = slice.len(); + let old_len = self.len(); + + self.reserve(len); + + // SAFETY: Additional memory has been reserved above. + // Therefore, the copy operates on valid memory. + unsafe { + let dst = self.as_mut_ptr().add(old_len); + let src = slice.as_ptr(); + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + self.set_len(old_len + len); + } + } +} + +/// A trait for specializing the implementation of [`extend_from_within`]. +/// +/// [`extend_from_within`]: crate::SmallVec::extend_from_within +pub(crate) trait SpecExtendFromWithin { + /// Main worker for [`extend_from_within`]. + /// + /// # Safety + /// + /// * The length of the vector is larger than or equal to `src.len()`. + /// * The spare capacity of the vector is larger than or equal to + /// `src.len()`. + /// + /// [`extend_from_within`]: SmallVec::extend_from_within + unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range); +} + +impl SpecExtendFromWithin for SmallVec { + default unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { + // SAFETY: Safety conditions are identical. + unsafe { + self.extend_from_within_fallback(src); + } + } +} + +impl SpecExtendFromWithin for SmallVec { + unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { + let old_len = self.len(); + + let start = src.start; + let len = src.len(); + + // SAFETY: The caller ensures that the vector has spare capacity + // for at least `src.len()` elements. This is also the amount of + // memory accessed when the data is copied. + unsafe { + let ptr = self.as_mut_ptr(); + let dst = ptr.add(old_len); + let src = ptr.add(start); + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + self.set_len(old_len + len); + } + } +} + +/// A trait for specializing the implementation of [`FromIterator`]. +/// +/// [`clone_from`]: Clone::clone_from +pub(crate) trait SpecFromIterator { + fn spec_from_iter(iter: I) -> Self; +} + +impl SpecFromIterator for SmallVec +where I: Iterator +{ + #[inline] + default fn spec_from_iter(iter: I) -> Self { + Self::from_iter_fallback(iter) + } +} + +impl SpecFromIterator for SmallVec +where I: core::iter::TrustedLen +{ + fn spec_from_iter(iter: I) -> Self { + let mut v = match iter.size_hint() { + (_, Some(upper)) => SmallVec::with_capacity(upper), + // TrustedLen contract guarantees that `size_hint() == (_, None)` means that there + // are more than `usize::MAX` elements. + // Since the previous branch would eagerly panic if the capacity is too large + // (via `with_capacity`) we do the same here. + _ => panic!("capacity overflow") + }; + // Reuse the extend specialization for TrustedLen. + v.spec_extend(iter); + v + } +} + +/// A trait for specializing the implementation of [`clone_from`]. +/// +/// [`clone_from`]: Clone::clone_from +pub(crate) trait SpecCloneFrom { + fn spec_clone_from(&mut self, source: &[T]); +} + +impl SpecCloneFrom for SmallVec { + #[inline] + default fn spec_clone_from(&mut self, source: &[T]) { + self.clone_from_fallback(source); + } +} + +impl SpecCloneFrom for SmallVec { + fn spec_clone_from(&mut self, source: &[T]) { + self.clear(); + self.extend_from_slice(source); + } +} + +/// A trait for specializing the implementation of [`From`] +/// with the source type being slices. +pub(crate) trait SpecFromSlice { + /// Creates a `SmallVec` value based on the contents of `slice`. + /// This will use the inline storage, not the heap. + /// + /// # Safety + /// + /// The caller must ensure that `slice.len() <= Self::inline_size()`. + unsafe fn spec_from(slice: &[T]) -> Self; +} + +impl SpecFromSlice for SmallVec { + default unsafe fn spec_from(slice: &[T]) -> Self { + // SAFETY: Safety conditions are identical. + unsafe { Self::from_slice_fallback(slice) } + } +} + +impl SpecFromSlice for SmallVec { + unsafe fn spec_from(slice: &[T]) -> Self { + let mut v = Self::new(); + + let src = slice.as_ptr(); + let len = slice.len(); + let dst = v.as_mut_ptr(); + + // SAFETY: The caller ensures that the slice length is smaller + // than or equal to the inline length. + unsafe { + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + v.set_len(len); + } + + v + } +} From ac74c846a43fb21c7117663428e41ad73448bdf0 Mon Sep 17 00:00:00 2001 From: Pedro Nobre Date: Thu, 3 Sep 2026 20:49:55 +0100 Subject: [PATCH 2/2] refactor: switched all `pub(crate)` instances for an unambiguous `pub` inside of `specialization.rs` --- src/specialization.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/specialization.rs b/src/specialization.rs index b197b51..f726225 100644 --- a/src/specialization.rs +++ b/src/specialization.rs @@ -10,7 +10,7 @@ use { /// A trait for specializing the implementation of [`from_elem`]. /// /// [`from_elem`]: crate::from_elem -pub(crate) trait SpecFromElem { +pub trait SpecFromElem { /// Creates a `Smallvec` value where `elem` is repeated `n` times. /// This will use the inline storage, not the heap. /// @@ -58,7 +58,7 @@ impl SpecFromElem for SmallVec { /// [`extend_from_slice`]. /// /// [`extend_from_slice`]: crate::SmallVec::extend_from_slice -pub(crate) trait SpecExtend { +pub trait SpecExtend { fn spec_extend(&mut self, iter: I); } @@ -169,7 +169,7 @@ where T: Copy /// A trait for specializing the implementation of [`extend_from_within`]. /// /// [`extend_from_within`]: crate::SmallVec::extend_from_within -pub(crate) trait SpecExtendFromWithin { +pub trait SpecExtendFromWithin { /// Main worker for [`extend_from_within`]. /// /// # Safety @@ -218,7 +218,7 @@ impl SpecExtendFromWithin for SmallVec { /// A trait for specializing the implementation of [`FromIterator`]. /// /// [`clone_from`]: Clone::clone_from -pub(crate) trait SpecFromIterator { +pub trait SpecFromIterator { fn spec_from_iter(iter: I) -> Self; } @@ -252,7 +252,7 @@ where I: core::iter::TrustedLen /// A trait for specializing the implementation of [`clone_from`]. /// /// [`clone_from`]: Clone::clone_from -pub(crate) trait SpecCloneFrom { +pub trait SpecCloneFrom { fn spec_clone_from(&mut self, source: &[T]); } @@ -272,7 +272,7 @@ impl SpecCloneFrom for SmallVec { /// A trait for specializing the implementation of [`From`] /// with the source type being slices. -pub(crate) trait SpecFromSlice { +pub trait SpecFromSlice { /// Creates a `SmallVec` value based on the contents of `slice`. /// This will use the inline storage, not the heap. ///