From ae723b4c8c3f435f0e0162665726447ed3c3e850 Mon Sep 17 00:00:00 2001 From: xtqqczze <45661989+xtqqczze@users.noreply.github.com> Date: Wed, 28 Feb 2024 01:07:27 +0000 Subject: [PATCH] Replace unaligned casts in `SpanHelpers.Memmove` Fix https://github.com/dotnet/runtime/issues/83709 IL size is reduced by ~140 bytes. --- .../src/System/SpanHelpers.ByteMemOps.cs | 140 +++++++++--------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index 54294592099256..5c24a5aff819f2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -58,56 +58,56 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) // Copy bytes which are multiples of 16 and leave the remainder for MCPY01 to handle. Debug.Assert(len > 16 && len <= 64); #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref dest) = Unsafe.As(ref src); // [0,16] + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); #elif TARGET_64BIT - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); // [0,16] + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 8))); #else - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); - Unsafe.As(ref Unsafe.Add(ref dest, 12)) = Unsafe.As(ref Unsafe.Add(ref src, 12)); // [0,16] + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 4))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 12), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 12))); #endif if (len <= 32) goto MCPY01; #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); // [0,32] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 16))); #elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); // [0,32] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 24))); #else - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 20)) = Unsafe.As(ref Unsafe.Add(ref src, 20)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); - Unsafe.As(ref Unsafe.Add(ref dest, 28)) = Unsafe.As(ref Unsafe.Add(ref src, 28)); // [0,32] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 20), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 20))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 24))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 28), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 28))); #endif if (len <= 48) goto MCPY01; #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); // [0,48] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 32))); #elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); // [0,48] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 32))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 40))); #else - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 36)) = Unsafe.As(ref Unsafe.Add(ref src, 36)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); - Unsafe.As(ref Unsafe.Add(ref dest, 44)) = Unsafe.As(ref Unsafe.Add(ref src, 44)); // [0,48] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 32))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 36), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 36))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 40))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 44), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 44))); #endif MCPY01: // Unconditionally copy the last 16 bytes using destEnd and srcEnd and return. Debug.Assert(len > 16 && len <= 64); #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); #elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); #else - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -12)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -12)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -12), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -12))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -4))); #endif return; @@ -117,13 +117,13 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) goto MCPY03; Debug.Assert(len >= 8 && len <= 16); #if TARGET_64BIT - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); #else - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 4))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -4))); #endif return; @@ -132,8 +132,8 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) if ((len & 4) == 0) goto MCPY04; Debug.Assert(len >= 4 && len < 8); - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -4))); return; MCPY04: @@ -144,7 +144,7 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) dest = src; if ((len & 2) == 0) return; - Unsafe.As(ref Unsafe.Add(ref destEnd, -2)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -2)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -2), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -2))); return; MCPY05: @@ -163,7 +163,7 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) // dest is more important to align than src because an unaligned store is more expensive // than an unaligned load. nuint misalignedElements = 64 - (nuint)Unsafe.AsPointer(ref dest) & 63; - Unsafe.As(ref dest) = Unsafe.As(ref src); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); src = ref Unsafe.Add(ref src, misalignedElements); dest = ref Unsafe.Add(ref dest, misalignedElements); len -= misalignedElements; @@ -177,33 +177,33 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) MCPY06: #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref dest) = Unsafe.As(ref src); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); #elif TARGET_64BIT - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); - Unsafe.As(ref Unsafe.Add(ref dest, 48)) = Unsafe.As(ref Unsafe.Add(ref src, 48)); - Unsafe.As(ref Unsafe.Add(ref dest, 56)) = Unsafe.As(ref Unsafe.Add(ref src, 56)); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 24))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 32))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 40))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 48), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 48))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 56), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 56))); #else - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); - Unsafe.As(ref Unsafe.Add(ref dest, 12)) = Unsafe.As(ref Unsafe.Add(ref src, 12)); - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 20)) = Unsafe.As(ref Unsafe.Add(ref src, 20)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); - Unsafe.As(ref Unsafe.Add(ref dest, 28)) = Unsafe.As(ref Unsafe.Add(ref src, 28)); - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 36)) = Unsafe.As(ref Unsafe.Add(ref src, 36)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); - Unsafe.As(ref Unsafe.Add(ref dest, 44)) = Unsafe.As(ref Unsafe.Add(ref src, 44)); - Unsafe.As(ref Unsafe.Add(ref dest, 48)) = Unsafe.As(ref Unsafe.Add(ref src, 48)); - Unsafe.As(ref Unsafe.Add(ref dest, 52)) = Unsafe.As(ref Unsafe.Add(ref src, 52)); - Unsafe.As(ref Unsafe.Add(ref dest, 56)) = Unsafe.As(ref Unsafe.Add(ref src, 56)); - Unsafe.As(ref Unsafe.Add(ref dest, 60)) = Unsafe.As(ref Unsafe.Add(ref src, 60)); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 4))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 12), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 12))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 20), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 20))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 24))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 28), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 28))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 32))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 36), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 36))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 40))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 44), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 44))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 48), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 48))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 52), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 52))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 56), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 56))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 60), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 60))); #endif dest = ref Unsafe.Add(ref dest, 64); src = ref Unsafe.Add(ref src, 64); @@ -215,15 +215,15 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) if (len > 16) goto MCPY00; #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); #elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); #else - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -12)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -12)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -12), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -12))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -4))); #endif return;