speed up mem::swap

src/libcore/mem.rs

@@ -109,7 +109,7 @@ pub use intrinsics::transmute;
 /// [`Clone`][clone]. You need the value's destructor to run only once,
 /// because a double `free` is undefined behavior.
 ///
-/// An example is the definition of [`mem::swap`][swap] in this module:
+/// An example is a possible implementation of [`mem::swap`][swap]:
 ///
 /// ```
 /// use std::mem;
@@ -499,18 +499,81 @@ pub unsafe fn uninitialized<T>() -> T {
 #[stable(feature = "rust1", since = "1.0.0")]
 pub fn swap<T>(x: &mut T, y: &mut T) {
     unsafe {
-        // Give ourselves some scratch space to work with
-        let mut t: T = uninitialized();
-
-        // Perform the swap, `&mut` pointers never alias
-        ptr::copy_nonoverlapping(&*x, &mut t, 1);
-        ptr::copy_nonoverlapping(&*y, x, 1);
-        ptr::copy_nonoverlapping(&t, y, 1);
-
-        // y and t now point to the same thing, but we need to completely
-        // forget `t` because we do not want to run the destructor for `T`
-        // on its value, which is still owned somewhere outside this function.
-        forget(t);
+        // The approach here is to utilize simd to swap x & y efficiently. Testing reveals
+        // that swapping either 32 bytes or 64 bytes at a time is most efficient for intel
+        // Haswell E processors. LLVM is more able to optimize if we give a struct a
+        // #[repr(simd)], even if we don't actually use this struct directly.
+        #[repr(simd)]
+        struct Block(u64, u64, u64, u64);
+        struct UnalignedBlock(u64, u64, u64, u64);
+
+        let block_size = size_of::<Block>();
+
+        // Get raw pointers to the bytes of x & y for easier manipulation
+        let x = x as *mut T as *mut u8;
+        let y = y as *mut T as *mut u8;
+
+        // Loop through x & y, copying them `Block` at a time
+        // The optimizer should unroll the loop fully for most types
+        // N.B. We can't use a for loop as the `range` impl calls `mem::swap` recursively
+        let len = size_of::<T>() as isize;
+        let mut i = 0;
+        while i + block_size as isize <= len {
+            // Create some uninitialized memory as scratch space
+            // Declaring `t` here avoids aligning the stack when this loop is unused
+            let mut t: Block = uninitialized();
+            let t = &mut t as *mut _ as *mut u8;
+
+            // Swap a block of bytes of x & y, using t as a temporary buffer
+            // This should be optimized into efficient SIMD operations where available
+            ptr::copy_nonoverlapping(x.offset(i), t, block_size);
+            ptr::copy_nonoverlapping(y.offset(i), x.offset(i), block_size);
+            ptr::copy_nonoverlapping(t, y.offset(i), block_size);
+            i += block_size as isize;
+        }
+
+
+        if i < len {
+            // Swap any remaining bytes, using aligned types to copy
+            // where appropriate (this information is lost by conversion
+            // to *mut u8, so restore it manually here)
+            let mut t: UnalignedBlock = uninitialized();
+            let rem = (len - i) as usize;
+
+            if align_of::<T>() % 8 == 0 && len % 8 == 0 {
+                let t = &mut t as *mut _ as *mut u64;
+                let x = x.offset(i) as *mut u64;
+                let y = y.offset(i) as *mut u64;
+
+                ptr::copy_nonoverlapping(x, t, rem / 8);
+                ptr::copy_nonoverlapping(y, x, rem / 8);
+                ptr::copy_nonoverlapping(t, y, rem / 8);
+            } else if align_of::<T>() % 4 == 0 && len % 4 == 0 {
+                let t = &mut t as *mut _ as *mut u32;
+                let x = x.offset(i) as *mut u32;
+                let y = y.offset(i) as *mut u32;
+
+                ptr::copy_nonoverlapping(x, t, rem / 4);
+                ptr::copy_nonoverlapping(y, x, rem / 4);
+                ptr::copy_nonoverlapping(t, y, rem / 4);
+            } else if align_of::<T>() % 2 == 0 && len % 2 == 0 {
+                let t = &mut t as *mut _ as *mut u16;
+                let x = x.offset(i) as *mut u16;
+                let y = y.offset(i) as *mut u16;
+
+                ptr::copy_nonoverlapping(x, t, rem / 2);
+                ptr::copy_nonoverlapping(y, x, rem / 2);
+                ptr::copy_nonoverlapping(t, y, rem / 2);
+            } else {
+                let t = &mut t as *mut _ as *mut u8;
+                let x = x.offset(i);
+                let y = y.offset(i);
+
+                ptr::copy_nonoverlapping(x, t, rem);
+                ptr::copy_nonoverlapping(y, x, rem);
+                ptr::copy_nonoverlapping(t, y, rem);
+            }
+        }
     }
 }