Skip to content

Commit

Permalink
Remove Vector<T> fallbacks (#88101)
Browse files Browse the repository at this point in the history
  • Loading branch information
@fanyang-mono
fanyang-mono authored Jun 28, 2023
1 parent f623089 commit 5ec1f15
Show file tree
Hide file tree
Showing 2 changed files with 0 additions and 230 deletions.
145 changes: 0 additions & 145 deletions src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,12 +462,6 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace)
// Avx2 branch also operates on Sse2 sizes, so check is combined.
lengthToExamine = UnalignedCountVector128(searchSpace);
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
lengthToExamine = UnalignedCountVector(searchSpace);
}
#endif

SequentialScan:
while (lengthToExamine >= 8)
Expand Down Expand Up @@ -740,34 +734,6 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace)
}
}
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
if (offset < (nuint)(uint)Length)
{
lengthToExamine = GetByteVectorSpanLength(offset, Length);

while (lengthToExamine > offset)
{
Vector<byte> matches = Vector.Equals(Vector<byte>.Zero, Vector.Load(searchSpace + offset));
if (Vector<byte>.Zero.Equals(matches))
{
offset += (nuint)Vector<byte>.Count;
continue;
}

// Find offset of first match and add to current offset
return (int)offset + LocateFirstFoundByte(matches);
}

if (offset < (nuint)(uint)Length)
{
lengthToExamine = ((nuint)(uint)Length - offset);
goto SequentialScan;
}
}
}
#endif

ThrowMustBeNullTerminatedString();
Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
Expand Down Expand Up @@ -945,36 +911,6 @@ public static unsafe bool SequenceEqual(ref byte first, ref byte second, nuint l
goto NotEqual;
}
}
#if MONO
else if (Vector.IsHardwareAccelerated && length >= (nuint)Vector<byte>.Count)
{
nuint offset = 0;
nuint lengthToExamine = length - (nuint)Vector<byte>.Count;
// Unsigned, so it shouldn't have overflowed larger than length (rather than negative)
Debug.Assert(lengthToExamine < length);
if (lengthToExamine > 0)
{
do
{
if (LoadVector(ref first, offset) != LoadVector(ref second, offset))
{
goto NotEqual;
}
offset += (nuint)Vector<byte>.Count;
} while (lengthToExamine > offset);
}

// Do final compare as Vector<byte>.Count from end rather than start
if (LoadVector(ref first, lengthToExamine) == LoadVector(ref second, lengthToExamine))
{
// C# compiler inverts this test, making the outer goto the conditional jmp.
goto Equal;
}

// This becomes a conditional jmp forward to not favor it.
goto NotEqual;
}
#endif

#if TARGET_64BIT
if (Vector128.IsHardwareAccelerated)
Expand Down Expand Up @@ -1184,24 +1120,6 @@ public static unsafe int SequenceCompareTo(ref byte first, int firstLength, ref
goto BytewiseCheck;
}
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
if (lengthToExamine > (nuint)Vector<byte>.Count)
{
lengthToExamine -= (nuint)Vector<byte>.Count;
while (lengthToExamine > offset)
{
if (LoadVector(ref first, offset) != LoadVector(ref second, offset))
{
goto BytewiseCheck;
}
offset += (nuint)Vector<byte>.Count;
}
goto BytewiseCheck;
}
}
#endif

if (lengthToExamine > (nuint)sizeof(nuint))
{
Expand Down Expand Up @@ -1323,69 +1241,6 @@ public static nuint CommonPrefixLength(ref byte first, ref byte second, nuint le
return i + uint.TrailingZeroCount(mask);
}

#if MONO
// Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundByte(Vector<byte> match)
{
var vector64 = Vector.AsVectorUInt64(match);
ulong candidate = 0;
int i = 0;
// Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001
for (; i < Vector<ulong>.Count; i++)
{
candidate = vector64[i];
if (candidate != 0)
{
break;
}
}

// Single LEA instruction with jitted const (using function result)
return i * 8 + LocateFirstFoundByte(candidate);
}

// Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateLastFoundByte(Vector<byte> match)
{
var vector64 = Vector.AsVectorUInt64(match);
ulong candidate = 0;
int i = Vector<ulong>.Count - 1;

// This pattern is only unrolled by the Jit if the limit is Vector<T>.Count
// As such, we need a dummy iteration variable for that condition to be satisfied
for (int j = 0; j < Vector<ulong>.Count; j++)
{
candidate = vector64[i];
if (candidate != 0)
{
break;
}

i--;
}

// Single LEA instruction with jitted const (using function result)
return i * 8 + LocateLastFoundByte(candidate);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector<byte> LoadVector(ref byte start, nuint offset)
=> Unsafe.ReadUnaligned<Vector<byte>>(ref Unsafe.AddByteOffset(ref start, offset));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint GetByteVectorSpanLength(nuint offset, int length)
=> (nuint)(uint)((length - (int)offset) & ~(Vector<byte>.Count - 1));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nuint UnalignedCountVector(byte* searchSpace)
{
nint unaligned = (nint)searchSpace & (Vector<byte>.Count - 1);
return (nuint)((Vector<byte>.Count - unaligned) & (Vector<byte>.Count - 1));
}
#endif

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundByte(ulong match)
=> BitOperations.TrailingZeroCount(match) >> 3;
Expand Down
85 changes: 0 additions & 85 deletions src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -546,13 +546,6 @@ public static unsafe int IndexOfNullCharacter(char* searchSpace)
// Needs to be double length to allow us to align the data first.
lengthToExamine = UnalignedCountVector128(searchSpace);
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
// Needs to be double length to allow us to align the data first.
lengthToExamine = UnalignedCountVector(searchSpace);
}
#endif

SequentialScan:
// In the non-vector case lengthToExamine is the total length.
Expand Down Expand Up @@ -850,42 +843,6 @@ public static unsafe int IndexOfNullCharacter(char* searchSpace)
}
}
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
if (offset < length)
{
Debug.Assert(length - offset >= Vector<ushort>.Count);

lengthToExamine = GetCharVectorSpanLength(offset, length);

if (lengthToExamine > 0)
{
do
{
Debug.Assert(lengthToExamine >= Vector<ushort>.Count);

var matches = Vector.Equals(Vector<ushort>.Zero, *(Vector<ushort>*)(searchSpace + (nuint)offset));
if (Vector<ushort>.Zero.Equals(matches))
{
offset += Vector<ushort>.Count;
lengthToExamine -= Vector<ushort>.Count;
continue;
}

// Find offset of first match
return (int)(offset + LocateFirstFoundChar(matches));
} while (lengthToExamine > 0);
}

if (offset < length)
{
lengthToExamine = length - offset;
goto SequentialScan;
}
}
}
#endif

ThrowMustBeNullTerminatedString();
Found3:
Expand All @@ -898,48 +855,6 @@ public static unsafe int IndexOfNullCharacter(char* searchSpace)
return (int)(offset);
}

#if MONO
// Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundChar(Vector<ushort> match)
{
var vector64 = Vector.AsVectorUInt64(match);
ulong candidate = 0;
int i = 0;
// Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001
for (; i < Vector<ulong>.Count; i++)
{
candidate = vector64[i];
if (candidate != 0)
{
break;
}
}

// Single LEA instruction with jitted const (using function result)
return i * 4 + LocateFirstFoundChar(candidate);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector<ushort> LoadVector(ref char start, nint offset)
=> Unsafe.ReadUnaligned<Vector<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, offset)));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector<ushort> LoadVector(ref char start, nuint offset)
=> Unsafe.ReadUnaligned<Vector<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, (nint)offset)));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nint GetCharVectorSpanLength(nint offset, nint length)
=> (length - offset) & ~(Vector<ushort>.Count - 1);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nint UnalignedCountVector(char* searchSpace)
{
const int ElementsPerByte = sizeof(ushort) / sizeof(byte);
return (nint)(uint)(-(int)searchSpace / ElementsPerByte) & (Vector<ushort>.Count - 1);
}
#endif

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundChar(ulong match)
=> BitOperations.TrailingZeroCount(match) >> 4;
Expand Down

0 comments on commit 5ec1f15

Please sign in to comment.