perf: increment/decrement len without reading on heap flag

src/lib.rs

@@ -1138,10 +1138,11 @@ impl<T, const N: usize> SmallVec<T, N> {
             // We have to do this so that Miri doesn't report a "Stacked
             // Borrows" rule violation. See PR/406
 
-            let new_len = len + 1;
-            debug_assert!(new_len <= self.capacity());
-            let on_heap = self.len.on_heap();
-            self.len = TaggedLen::new(new_len, on_heap);
+            debug_assert!(len + 1 <= self.capacity());
+            // SAFETY: we have wrote the value to the address already
+            unsafe {
+                self.len.increment();
+            }
         }
 
         // SAFETY: `ptr` is aligned, non-null and points to the element
@@ -1157,8 +1158,11 @@ impl<T, const N: usize> SmallVec<T, N> {
             return None;
         }
         let new_len = len - 1;
-        // SAFETY: new_len < len since len is non-zero
-        unsafe { self.set_len(new_len) };
+        // SAFETY: new_len < len since len is non-zero and
+        // we are returning ownership of the current value.
+        unsafe {
+            self.len.decrement();
+        }
         // SAFETY: this element was initialized and we just gave up ownership of
         // it, so we can give it away
         let value = unsafe { self.as_mut_ptr().add(new_len).read() };
@@ -1457,10 +1461,11 @@ impl<T, const N: usize> SmallVec<T, N> {
             // We have to do this so that Miri doesn't report a "Stacked
             // Borrows" rule violation. See PR/406
 
-            let new_len = len + 1;
-            debug_assert!(new_len <= self.capacity());
-            let on_heap = self.len.on_heap();
-            self.len = TaggedLen::new(new_len, on_heap);
+            debug_assert!(len + 1 <= self.capacity());
+            // SAFETY: we have wrote the value to the address already
+            unsafe {
+                self.len.increment();
+            }
         }
 
         // SAFETY: `ptr` is aligned, non-null and points to the element

src/taggedlen.rs

@@ -54,4 +54,26 @@ impl<T> TaggedLen<T> {
     pub const fn value(self) -> usize {
         if Self::IS_ZST { self.0 } else { self.0 >> 1 }
     }
+
+    /// Returns the same tag with the length increased by one.
+    ///
+    /// This increases the length without rereading the `on heap` flag.
+    #[inline]
+    pub const unsafe fn increment(&mut self) {
+        self.0 += if Self::IS_ZST {
+            1
+        } else {
+            debug_assert!(self.value() + 1 < isize::MAX as usize);
+            0b10
+        }
+    }
+
+    /// Returns the same tag with the length decreased by one.
+    ///
+    /// This decreases the length without rereading the `on heap` flag.
+    #[inline]
+    pub const unsafe fn decrement(&mut self) {
+        debug_assert!(self.value() > 0);
+        self.0 -= if Self::IS_ZST { 1 } else { 0b10 };
+    }
 }