From b2d5f0f06d6113b5c9b5bba56690c7166d1cd0ae Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 30 Nov 2022 06:26:04 +0100 Subject: [PATCH 1/4] Add PackedIndexOf for chars --- .../System.Private.CoreLib.Shared.projitems | 10 +- .../src/System/Globalization/Ordinal.cs | 4 +- ...OfAny1Value.cs => IndexOfAny1ByteValue.cs} | 20 +- .../IndexOfAnyValues/IndexOfAny1CharValue.cs | 49 ++ ...Any2Values.cs => IndexOfAny2ByteValues.cs} | 20 +- .../IndexOfAnyValues/IndexOfAny2CharValues.cs | 51 ++ ...ange.cs => IndexOfAnyByteValuesInRange.cs} | 35 +- .../IndexOfAnyCharValuesInRange.cs | 67 ++ .../IndexOfAnyValues/IndexOfAnyValues.cs | 44 +- .../src/System/MemoryExtensions.cs | 14 +- .../src/System/SpanHelpers.Char.Packed.cs | 574 ++++++++++++++++++ .../src/System/SpanHelpers.T.cs | 81 ++- .../src/System/String.Searching.cs | 4 +- 13 files changed, 884 insertions(+), 89 deletions(-) rename src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/{IndexOfAny1Value.cs => IndexOfAny1ByteValue.cs} (61%) create mode 100644 src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs rename src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/{IndexOfAny2Values.cs => IndexOfAny2ByteValues.cs} (62%) create mode 100644 src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs rename src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/{IndexOfAnyValuesInRange.cs => IndexOfAnyByteValuesInRange.cs} (52%) create mode 100644 src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.Packed.cs diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 2c5113d9f7238..d4056302ff0ce 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -421,8 +421,10 @@ - - + + + + @@ -430,12 +432,13 @@ + - + @@ -1028,6 +1031,7 @@ + diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index 47678886ffa21..0ed1bcc8c5bdb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -339,7 +339,9 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan source, ReadOnly { // Do a quick search for the first element of "value". int relativeIndex = isLetter ? - SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : + SpanHelpers.PackedIndexOfIsSupported + ? SpanHelpers.PackedIndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) + : SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength); if (relativeIndex < 0) { diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1Value.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1ByteValue.cs similarity index 61% rename from src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1Value.cs rename to src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1ByteValue.cs index 565af9cf42c70..3dd3a6aa4fcaf 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1Value.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1ByteValue.cs @@ -2,42 +2,40 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; namespace System.Buffers { - internal sealed class IndexOfAny1Value : IndexOfAnyValues - where T : struct, INumber + internal sealed class IndexOfAny1ByteValue : IndexOfAnyValues { - private readonly T _e0; + private readonly byte _e0; - public IndexOfAny1Value(ReadOnlySpan values) + public IndexOfAny1ByteValue(ReadOnlySpan values) { Debug.Assert(values.Length == 1); _e0 = values[0]; } - internal override T[] GetValues() => new[] { _e0 }; + internal override byte[] GetValues() => new[] { _e0 }; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(T value) => + internal override bool ContainsCore(byte value) => value == _e0; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => + internal override int IndexOfAny(ReadOnlySpan span) => span.IndexOf(_e0); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => + internal override int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(_e0); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => + internal override int LastIndexOfAny(ReadOnlySpan span) => span.LastIndexOf(_e0); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => span.LastIndexOfAnyExcept(_e0); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs new file mode 100644 index 0000000000000..7579de199e77b --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + internal sealed class IndexOfAny1CharValue : IndexOfAnyValues + where TShouldUsePacked : struct, IndexOfAnyValues.IRuntimeConst + { + private char _e0; + + public IndexOfAny1CharValue(char value) => + _e0 = value; + + internal override char[] GetValues() => new[] { _e0 }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + TShouldUsePacked.Value + ? SpanHelpers.PackedIndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length) + : SpanHelpers.NonPackedIndexOfValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + TShouldUsePacked.Value + ? SpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length) + : SpanHelpers.NonPackedIndexOfValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOf(_e0); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2Values.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2ByteValues.cs similarity index 62% rename from src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2Values.cs rename to src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2ByteValues.cs index cb89fcfbc497e..76f8db819156f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2Values.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2ByteValues.cs @@ -2,42 +2,40 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; namespace System.Buffers { - internal sealed class IndexOfAny2Values : IndexOfAnyValues - where T : struct, INumber + internal sealed class IndexOfAny2ByteValues : IndexOfAnyValues { - private readonly T _e0, _e1; + private readonly byte _e0, _e1; - public IndexOfAny2Values(ReadOnlySpan values) + public IndexOfAny2ByteValues(ReadOnlySpan values) { Debug.Assert(values.Length == 2); (_e0, _e1) = (values[0], values[1]); } - internal override T[] GetValues() => new[] { _e0, _e1 }; + internal override byte[] GetValues() => new[] { _e0, _e1 }; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(T value) => + internal override bool ContainsCore(byte value) => value == _e0 || value == _e1; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => + internal override int IndexOfAny(ReadOnlySpan span) => span.IndexOfAny(_e0, _e1); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => + internal override int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(_e0, _e1); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => + internal override int LastIndexOfAny(ReadOnlySpan span) => span.LastIndexOfAny(_e0, _e1); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => span.LastIndexOfAnyExcept(_e0, _e1); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs new file mode 100644 index 0000000000000..9ca43deb7f522 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + internal sealed class IndexOfAny2CharValue : IndexOfAnyValues + where TShouldUsePacked : struct, IndexOfAnyValues.IRuntimeConst + { + private char _e0, _e1; + + public IndexOfAny2CharValue(char value0, char value1) => + (_e0, _e1) = (value0, value1); + + internal override char[] GetValues() => new[] { _e0 }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0 || value == _e1; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + TShouldUsePacked.Value + ? SpanHelpers.PackedIndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) + : SpanHelpers.NonPackedIndexOfAnyValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + Unsafe.As(ref _e1), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + TShouldUsePacked.Value + ? SpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) + : SpanHelpers.NonPackedIndexOfAnyValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + Unsafe.As(ref _e1), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAny(_e0, _e1); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0, _e1); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValuesInRange.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyByteValuesInRange.cs similarity index 52% rename from src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValuesInRange.cs rename to src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyByteValuesInRange.cs index c9c4f1a1f499a..4561a2170ec78 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValuesInRange.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyByteValuesInRange.cs @@ -1,58 +1,53 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; namespace System.Buffers { - internal sealed class IndexOfAnyValuesInRange : IndexOfAnyValues - where T : struct, INumber + internal sealed class IndexOfAnyByteValuesInRange : IndexOfAnyValues { - private readonly T _lowInclusive, _highInclusive; + private readonly byte _lowInclusive, _highInclusive; private readonly uint _lowUint, _highMinusLow; - public IndexOfAnyValuesInRange(T lowInclusive, T highInclusive) + public IndexOfAnyByteValuesInRange(byte lowInclusive, byte highInclusive) { - Debug.Assert(lowInclusive is byte or char); (_lowInclusive, _highInclusive) = (lowInclusive, highInclusive); - _lowUint = uint.CreateChecked(lowInclusive); - _highMinusLow = uint.CreateChecked(highInclusive - lowInclusive); + _lowUint = lowInclusive; + _highMinusLow = (uint)(highInclusive - lowInclusive); } - internal override T[] GetValues() + internal override byte[] GetValues() { - T[] values = new T[_highMinusLow + 1]; + byte[] values = new byte[_highMinusLow + 1]; - T element = _lowInclusive; + int low = _lowInclusive; for (int i = 0; i < values.Length; i++) { - values[i] = element; - element += T.One; + values[i] = (byte)(low + i); } return values; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(T value) => - uint.CreateChecked(value) - _lowUint <= _highMinusLow; + internal override bool ContainsCore(byte value) => + value - _lowUint <= _highMinusLow; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => + internal override int IndexOfAny(ReadOnlySpan span) => span.IndexOfAnyInRange(_lowInclusive, _highInclusive); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => + internal override int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExceptInRange(_lowInclusive, _highInclusive); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => + internal override int LastIndexOfAny(ReadOnlySpan span) => span.LastIndexOfAnyInRange(_lowInclusive, _highInclusive); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => span.LastIndexOfAnyExceptInRange(_lowInclusive, _highInclusive); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs new file mode 100644 index 0000000000000..11fe35bd9d865 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + internal sealed class IndexOfAnyCharValuesInRange : IndexOfAnyValues + where TShouldUsePacked : struct, IndexOfAnyValues.IRuntimeConst + { + private char _lowInclusive, _rangeInclusive, _highInclusive; + private readonly uint _lowUint, _highMinusLow; + + public IndexOfAnyCharValuesInRange(char lowInclusive, char highInclusive) + { + (_lowInclusive, _rangeInclusive, _highInclusive) = (lowInclusive, (char)(highInclusive - lowInclusive), highInclusive); + _lowUint = lowInclusive; + _highMinusLow = (uint)(highInclusive - lowInclusive); + } + + internal override char[] GetValues() + { + char[] values = new char[_rangeInclusive + 1]; + + int low = _lowInclusive; + for (int i = 0; i < values.Length; i++) + { + values[i] = (char)(low + i); + } + + return values; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value - _lowUint <= _highMinusLow; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + TShouldUsePacked.Value + ? SpanHelpers.PackedIndexOfAnyInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) + : SpanHelpers.NonPackedIndexOfAnyInRangeUnsignedNumber>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _lowInclusive), + Unsafe.As(ref _highInclusive), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + TShouldUsePacked.Value + ? SpanHelpers.PackedIndexOfAnyExceptInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) + : SpanHelpers.NonPackedIndexOfAnyInRangeUnsignedNumber>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _lowInclusive), + Unsafe.As(ref _highInclusive), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAnyInRange(_lowInclusive, _highInclusive); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExceptInRange(_lowInclusive, _highInclusive); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs index 8ca2624750027..bccdd87d21428 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs @@ -8,6 +8,8 @@ using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; +#pragma warning disable 8500 // address of managed types + namespace System.Buffers { /// @@ -31,7 +33,7 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) if (values.Length == 1) { - return new IndexOfAny1Value(values); + return new IndexOfAny1ByteValue(values); } // IndexOfAnyValuesInRange is slower than IndexOfAny1Value, but faster than IndexOfAny2Values @@ -45,7 +47,7 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) Debug.Assert(values.Length is 2 or 3 or 4 or 5); return values.Length switch { - 2 => new IndexOfAny2Values(values), + 2 => new IndexOfAny2ByteValues(values), 3 => new IndexOfAny3Values(values), 4 => new IndexOfAny4Values(values), _ => new IndexOfAny5Values(values), @@ -77,7 +79,10 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) if (values.Length == 1) { - return new IndexOfAny1Value(values); + char value = values[0]; + return SpanHelpers.CanUsePackedIndexOf(value) + ? new IndexOfAny1CharValue(value) + : new IndexOfAny1CharValue(value); } // IndexOfAnyValuesInRange is slower than IndexOfAny1Value, but faster than IndexOfAny2Values @@ -88,7 +93,11 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) if (values.Length == 2) { - return new IndexOfAny2Values(values); + char value0 = values[0]; + char value1 = values[1]; + return SpanHelpers.CanUsePackedIndexOf(value0) && SpanHelpers.CanUsePackedIndexOf(value1) + ? new IndexOfAny2CharValue(value0, value1) + : new IndexOfAny2CharValue(value0, value1); } if (values.Length == 3) @@ -130,7 +139,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), return new IndexOfAnyCharValuesProbabilistic(values); } - private static IndexOfAnyValues? TryGetSingleRange(ReadOnlySpan values, out T maxInclusive) + private static unsafe IndexOfAnyValues? TryGetSingleRange(ReadOnlySpan values, out T maxInclusive) where T : struct, INumber, IMinMaxValue { T min = T.MaxValue; @@ -165,7 +174,30 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), return null; } - return (IndexOfAnyValues)(object)new IndexOfAnyValuesInRange(min, max); + if (typeof(T) == typeof(byte)) + { + return (IndexOfAnyValues)(object)new IndexOfAnyByteValuesInRange(byte.CreateChecked(min), byte.CreateChecked(max)); + } + + Debug.Assert(typeof(T) == typeof(char)); + return (IndexOfAnyValues)(object)(SpanHelpers.CanUsePackedIndexOf(min) && SpanHelpers.CanUsePackedIndexOf(max) + ? new IndexOfAnyCharValuesInRange(*(char*)&min, *(char*)&max) + : new IndexOfAnyCharValuesInRange(*(char*)&min, *(char*)&max)); + } + + internal interface IRuntimeConst + { + static abstract bool Value { get; } + } + + private readonly struct TrueConst : IRuntimeConst + { + public static bool Value => true; + } + + private readonly struct FalseConst : IRuntimeConst + { + public static bool Value => false; } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index 23c3e86ef0ad0..595b6969960dd 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -558,7 +558,7 @@ public static int IndexOfAnyExcept(this Span span, IndexOfAnyValues val [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe int IndexOfAnyExcept(this ReadOnlySpan span, T value) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -609,7 +609,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe int IndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -673,7 +673,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int IndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1, T value2, T value3) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -864,7 +864,7 @@ public static int LastIndexOfAnyExcept(this Span span, IndexOfAnyValues [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe int LastIndexOfAnyExcept(this ReadOnlySpan span, T value) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -915,7 +915,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe int LastIndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -979,7 +979,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int LastIndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1, T value2, T value3) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -3067,7 +3067,7 @@ public static void Sort(this Span keys, Span items, [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe void Replace(this Span span, T oldValue, T newValue) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { nuint length = (uint)span.Length; diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.Packed.cs new file mode 100644 index 0000000000000..02d76e66077cb --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.Packed.cs @@ -0,0 +1,574 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers.Binary; +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; + +#pragma warning disable 8500 // sizeof of managed types + +namespace System +{ + internal static partial class SpanHelpers // .Char.Packed + { + internal static bool PackedIndexOfIsSupported => Sse2.IsSupported || AdvSimd.IsSupported; + + // Not all values can benefit from packing the searchSpace. See comments in PackSources below. + // On X86, the values must be in the [1, 254] range. + // On ARM, the values must be in the [0, 254] range. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe bool CanUsePackedIndexOf(T value) => + PackedIndexOfIsSupported && + RuntimeHelpers.IsBitwiseEquatable() && + sizeof(T) == sizeof(ushort) && + (Sse2.IsSupported + ? *(ushort*)&value - 1u < 254u + : *(ushort*)&value < 255u); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int PackedIndexOf(ref char searchSpace, char value, int length) => + PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int PackedIndexOfAnyExcept(ref char searchSpace, char value, int length) => + PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int PackedIndexOfAny(ref char searchSpace, char value0, char value1, int length) => + PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int PackedIndexOfAnyExcept(ref char searchSpace, char value0, char value1, int length) => + PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int PackedIndexOfAnyInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => + PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int PackedIndexOfAnyExceptInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => + PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + + private static int PackedIndexOf(ref short searchSpace, short value, int length) + where TNegator : struct, INegator + { + Debug.Assert(CanUsePackedIndexOf(value)); + + if (length < Vector128.Count) + { + nuint offset = 0; + + if (length >= 4) + { + length -= 4; + + if (TNegator.NegateIfNeeded(searchSpace == value)) return 0; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 1) == value)) return 1; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 2) == value)) return 2; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 3) == value)) return 3; + + offset = 4; + } + + while (length > 0) + { + length -= 1; + + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset; + + offset += 1; + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 packedValue = Vector256.Create((byte)value); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + else + { + Vector128 packedValue = Vector128.Create((byte)value); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + } + + return -1; + } + + private static int PackedIndexOfAny(ref short searchSpace, short value0, short value1, int length) + where TNegator : struct, INegator + { + Debug.Assert(CanUsePackedIndexOf(value0)); + Debug.Assert(CanUsePackedIndexOf(value1)); + + if (length < Vector128.Count) + { + nuint offset = 0; + short lookUp; + + if (length >= 4) + { + length -= 4; + + lookUp = searchSpace; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 0; + lookUp = Unsafe.Add(ref searchSpace, 1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 1; + lookUp = Unsafe.Add(ref searchSpace, 2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 2; + lookUp = Unsafe.Add(ref searchSpace, 3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 3; + + offset = 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset; + + offset += 1; + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 packedValue0 = Vector256.Create((byte)value0); + Vector256 packedValue1 = Vector256.Create((byte)value1); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + else + { + Vector128 packedValue0 = Vector128.Create((byte)value0); + Vector128 packedValue1 = Vector128.Create((byte)value1); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + } + + return -1; + } + + private static int PackedIndexOfAnyInRange(ref short searchSpace, short lowInclusive, short rangeInclusive, int length) + where TNegator : struct, INegator + { + Debug.Assert(CanUsePackedIndexOf(lowInclusive)); + Debug.Assert(CanUsePackedIndexOf((short)(lowInclusive + rangeInclusive))); + Debug.Assert(rangeInclusive >= 0); + + if (length < Vector128.Count) + { + uint lowInclusiveUint = (uint)lowInclusive; + uint rangeInclusiveUint = (uint)rangeInclusive; + for (int i = 0; i < length; i++) + { + uint current = (uint)Unsafe.Add(ref searchSpace, i); + if (TNegator.NegateIfNeeded((current - lowInclusiveUint) <= rangeInclusiveUint)) + { + return i; + } + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 lowVector = Vector256.Create((byte)lowInclusive); + Vector256 rangeVector = Vector256.Create((byte)rangeInclusive); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.LessThanOrEqual(packedSource - lowVector, rangeVector); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.LessThanOrEqual(packedSource - lowVector, rangeVector); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + else + { + Vector128 lowVector = Vector128.Create((byte)lowInclusive); + Vector128 rangeVector = Vector128.Create((byte)rangeInclusive); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.LessThanOrEqual(packedSource - lowVector, rangeVector); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.LessThanOrEqual(packedSource - lowVector, rangeVector); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 PackSources(Vector256 source0, Vector256 source1) + { + Debug.Assert(Avx2.IsSupported); + // Pack two vectors of characters into bytes. While the type is Vector256, these are really UInt16 characters. + // X86: Downcast every character using saturation. + // - Values <= 32767 result in min(value, 255). + // - Values > 32767 result in 0. Because of this we can't accept needles that contain 0. + return Avx2.PackUnsignedSaturate(source0, source1).AsByte(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 PackSources(Vector128 source0, Vector128 source1) + { + // Pack two vectors of characters into bytes. While the type is Vector128, these are really UInt16 characters. + // X86: Downcast every character using saturation. + // - Values <= 32767 result in min(value, 255). + // - Values > 32767 result in 0. Because of this we can't accept needles that contain 0. + // ARM64: Do narrowing saturation over unsigned values. + // - All values result in min(value, 255) + return Sse2.IsSupported + ? Sse2.PackUnsignedSaturate(source0, source1).AsByte() + : AdvSimd.ExtractNarrowingSaturateUpper(AdvSimd.ExtractNarrowingSaturateLower(source0.AsUInt16()), source1.AsUInt16()); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 NegateIfNeeded(Vector128 result) + where TNegator : struct, INegator => + typeof(TNegator) == typeof(DontNegate) ? result : ~result; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 NegateIfNeeded(Vector256 result) + where TNegator : struct, INegator => + typeof(TNegator) == typeof(DontNegate) ? result : ~result; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeFirstIndex(ref short searchSpace, ref short current, Vector128 equals) + { + uint notEqualsElements = equals.ExtractMostSignificantBits(); + int index = BitOperations.TrailingZeroCount(notEqualsElements); + return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / sizeof(short)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeFirstIndex(ref short searchSpace, ref short current, Vector256 equals) + { + uint notEqualsElements = FixUpPackedVector256Mask(equals.ExtractMostSignificantBits()); + int index = BitOperations.TrailingZeroCount(notEqualsElements); + return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / sizeof(short)); + } + +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeFirstIndexOverlapped(ref short searchSpace, ref short current0, ref short current1, Vector128 equals) + { + uint notEqualsElements = equals.ExtractMostSignificantBits(); + int offsetInVector = BitOperations.TrailingZeroCount(notEqualsElements); + if (offsetInVector >= Vector128.Count) + { + // We matched within the second vector + current0 = ref current1; + offsetInVector -= Vector128.Count; + } + return offsetInVector + (int)(Unsafe.ByteOffset(ref searchSpace, ref current0) / sizeof(short)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeFirstIndexOverlapped(ref short searchSpace, ref short current0, ref short current1, Vector256 equals) + { + uint notEqualsElements = FixUpPackedVector256Mask(equals.ExtractMostSignificantBits()); + int offsetInVector = BitOperations.TrailingZeroCount(notEqualsElements); + if (offsetInVector >= Vector256.Count) + { + // We matched within the second vector + current0 = ref current1; + offsetInVector -= Vector256.Count; + } + return offsetInVector + (int)(Unsafe.ByteOffset(ref searchSpace, ref current0) / sizeof(short)); + } +#pragma warning restore IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint FixUpPackedVector256Mask(uint mask) + { + Debug.Assert(Avx2.IsSupported); + // Avx2.PackUnsignedSaturate(Vector256.Create((short)1), Vector256.Create((short)2)) will result in + // 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 + // We want to swap the X and Y bits + // 1, 1, 1, 1, 1, 1, 1, 1, X, X, X, X, X, X, X, X, Y, Y, Y, Y, Y, Y, Y, Y, 2, 2, 2, 2, 2, 2, 2, 2 + const uint CorrectPositionsMask = 0xFF0000FF; + + return (mask & CorrectPositionsMask) | BinaryPrimitives.ReverseEndianness(mask & ~CorrectPositionsMask); + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 66bf0618cab3a..7f222ec0f238a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -1303,32 +1303,6 @@ public static int SequenceCompareTo(ref T first, int firstLength, ref T secon return firstLength.CompareTo(secondLength); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe bool CanVectorizeAndBenefit(int length) where T : IEquatable? - { - if (Vector128.IsHardwareAccelerated && RuntimeHelpers.IsBitwiseEquatable()) - { - if (sizeof(T) == sizeof(byte)) - { - return length >= Vector128.Count; - } - else if (sizeof(T) == sizeof(short)) - { - return length >= Vector128.Count; - } - else if (sizeof(T) == sizeof(int)) - { - return length >= Vector128.Count; - } - else if (sizeof(T) == sizeof(long)) - { - return length >= Vector128.Count; - } - } - - return false; - } - [MethodImpl(MethodImplOptions.AggressiveOptimization)] internal static bool ContainsValueType(ref T searchSpace, T value, int length) where T : struct, INumber { @@ -1458,8 +1432,23 @@ internal static int IndexOfValueType(ref T searchSpace, T value, int length) internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value, int length) where T : struct, INumber => IndexOfValueType>(ref searchSpace, value, length); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe int IndexOfValueType(ref TValue searchSpace, TValue value, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + if (CanUsePackedIndexOf(value)) + { + return typeof(TNegator) == typeof(DontNegate) + ? PackedIndexOf(ref Unsafe.As(ref searchSpace), *(char*)&value, length) + : PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value, length); + } + + return NonPackedIndexOfValueType(ref searchSpace, value, length); + } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static int IndexOfValueType(ref TValue searchSpace, TValue value, int length) + internal static int NonPackedIndexOfValueType(ref TValue searchSpace, TValue value, int length) where TValue : struct, INumber where TNegator : struct, INegator { @@ -1600,9 +1589,24 @@ internal static int IndexOfAnyValueType(ref T searchSpace, T value0, T value1 internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value0, T value1, int length) where T : struct, INumber => IndexOfAnyValueType>(ref searchSpace, value0, value1, length); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + if (CanUsePackedIndexOf(value0) && CanUsePackedIndexOf(value1)) + { + return typeof(TNegator) == typeof(DontNegate) + ? PackedIndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length) + : PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length); + } + + return NonPackedIndexOfAnyValueType(ref searchSpace, value0, value1, length); + } + // having INumber constraint here allows to use == operator and get better perf compared to .Equals [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, int length) + internal static int NonPackedIndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, int length) where TValue : struct, INumber where TNegator : struct, INegator { @@ -3050,7 +3054,26 @@ internal static int IndexOfAnyExceptInRangeUnsignedNumber(ref T searchSpace, where T : struct, IUnsignedNumber, IComparisonOperators => IndexOfAnyInRangeUnsignedNumber>(ref searchSpace, lowInclusive, highInclusive, length); - private static int IndexOfAnyInRangeUnsignedNumber(ref T searchSpace, T lowInclusive, T highInclusive, int length) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe int IndexOfAnyInRangeUnsignedNumber(ref T searchSpace, T lowInclusive, T highInclusive, int length) + where T : struct, IUnsignedNumber, IComparisonOperators + where TNegator : struct, INegator + { + if (CanUsePackedIndexOf(lowInclusive) && CanUsePackedIndexOf(highInclusive) && highInclusive >= lowInclusive) + { + ref char charSearchSpace = ref Unsafe.As(ref searchSpace); + char charLowInclusive = *(char*)&lowInclusive; + char charRange = (char)(*(char*)&highInclusive - charLowInclusive); + + return typeof(TNegator) == typeof(DontNegate) + ? PackedIndexOfAnyInRange(ref charSearchSpace, charLowInclusive, charRange, length) + : PackedIndexOfAnyExceptInRange(ref charSearchSpace, charLowInclusive, charRange, length); + } + + return NonPackedIndexOfAnyInRangeUnsignedNumber(ref searchSpace, lowInclusive, highInclusive, length); + } + + internal static int NonPackedIndexOfAnyInRangeUnsignedNumber(ref T searchSpace, T lowInclusive, T highInclusive, int length) where T : struct, IUnsignedNumber, IComparisonOperators where TNegator : struct, INegator { diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs index df95295a116fa..c84b07a3c9f75 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs @@ -78,7 +78,9 @@ private int IndexOfCharOrdinalIgnoreCase(char value) { char valueUc = (char)(value | 0x20); char valueLc = (char)(value & ~0x20); - return SpanHelpers.IndexOfAnyChar(ref _firstChar, valueLc, valueUc, Length); + return SpanHelpers.PackedIndexOfIsSupported + ? SpanHelpers.PackedIndexOfAny(ref _firstChar, valueLc, valueUc, Length) + : SpanHelpers.IndexOfAnyChar(ref _firstChar, valueLc, valueUc, Length); } return SpanHelpers.IndexOfChar(ref _firstChar, value, Length); From 178b1f8801cffbdaccad5d14cec6a9abc6edb1c0 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Fri, 16 Dec 2022 23:39:46 +0100 Subject: [PATCH 2/4] Add Contains and IndexOfValue(3 chars) --- .../System.Private.CoreLib.Shared.projitems | 7 +- .../src/System/Globalization/Ordinal.cs | 4 +- .../IndexOfAnyValues/IndexOfAny1CharValue.cs | 4 +- .../IndexOfAnyValues/IndexOfAny2CharValues.cs | 6 +- ...Any3Values.cs => IndexOfAny3ByteValues.cs} | 20 +- .../IndexOfAnyValues/IndexOfAny3CharValues.cs | 53 +++ .../IndexOfAnyCharValuesInRange.cs | 4 +- .../IndexOfAnyValues/IndexOfAnyValues.cs | 15 +- ...s.Char.Packed.cs => SpanHelpers.Packed.cs} | 353 ++++++++++++++++-- .../src/System/SpanHelpers.T.cs | 48 ++- .../src/System/String.Searching.cs | 4 +- 11 files changed, 453 insertions(+), 65 deletions(-) rename src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/{IndexOfAny3Values.cs => IndexOfAny3ByteValues.cs} (63%) create mode 100644 src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs rename src/libraries/System.Private.CoreLib/src/System/{SpanHelpers.Char.Packed.cs => SpanHelpers.Packed.cs} (61%) diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index d4056302ff0ce..58fe300c6ba3c 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -425,20 +425,21 @@ - + + + - @@ -1031,8 +1032,8 @@ - + diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index 0ed1bcc8c5bdb..f5157b0cbba8a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -339,8 +339,8 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan source, ReadOnly { // Do a quick search for the first element of "value". int relativeIndex = isLetter ? - SpanHelpers.PackedIndexOfIsSupported - ? SpanHelpers.PackedIndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) + PackedSpanHelpers.PackedIndexOfIsSupported + ? PackedSpanHelpers.PackedIndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength); if (relativeIndex < 0) diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs index 7579de199e77b..8461c8b2b50c5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs @@ -23,7 +23,7 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length) + ? PackedSpanHelpers.PackedIndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length) : SpanHelpers.NonPackedIndexOfValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), @@ -32,7 +32,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAnyExcept(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length) + ? PackedSpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length) : SpanHelpers.NonPackedIndexOfValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs index 9ca43deb7f522..8560999b3d4bf 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs @@ -14,7 +14,7 @@ internal sealed class IndexOfAny2CharValue : IndexOfAnyValues< public IndexOfAny2CharValue(char value0, char value1) => (_e0, _e1) = (value0, value1); - internal override char[] GetValues() => new[] { _e0 }; + internal override char[] GetValues() => new[] { _e0, _e1 }; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override bool ContainsCore(char value) => @@ -23,7 +23,7 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) + ? PackedSpanHelpers.PackedIndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) : SpanHelpers.NonPackedIndexOfAnyValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), @@ -33,7 +33,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAnyExcept(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) + ? PackedSpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) : SpanHelpers.NonPackedIndexOfAnyValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3Values.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3ByteValues.cs similarity index 63% rename from src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3Values.cs rename to src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3ByteValues.cs index f94509b19f6d6..ad569e18589ac 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3Values.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3ByteValues.cs @@ -2,42 +2,40 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; namespace System.Buffers { - internal sealed class IndexOfAny3Values : IndexOfAnyValues - where T : struct, INumber + internal sealed class IndexOfAny3ByteValues : IndexOfAnyValues { - private readonly T _e0, _e1, _e2; + private readonly byte _e0, _e1, _e2; - public IndexOfAny3Values(ReadOnlySpan values) + public IndexOfAny3ByteValues(ReadOnlySpan values) { Debug.Assert(values.Length == 3); (_e0, _e1, _e2) = (values[0], values[1], values[2]); } - internal override T[] GetValues() => new[] { _e0, _e1, _e2 }; + internal override byte[] GetValues() => new[] { _e0, _e1, _e2 }; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(T value) => + internal override bool ContainsCore(byte value) => value == _e0 || value == _e1 || value == _e2; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => + internal override int IndexOfAny(ReadOnlySpan span) => span.IndexOfAny(_e0, _e1, _e2); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => + internal override int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(_e0, _e1, _e2); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => + internal override int LastIndexOfAny(ReadOnlySpan span) => span.LastIndexOfAny(_e0, _e1, _e2); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => span.LastIndexOfAnyExcept(_e0, _e1, _e2); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs new file mode 100644 index 0000000000000..519fe27245def --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + internal sealed class IndexOfAny3CharValue : IndexOfAnyValues + where TShouldUsePacked : struct, IndexOfAnyValues.IRuntimeConst + { + private char _e0, _e1, _e2; + + public IndexOfAny3CharValue(char value0, char value1, char value2) => + (_e0, _e1, _e2) = (value0, value1, value2); + + internal override char[] GetValues() => new[] { _e0, _e1, _e2 }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0 || value == _e1 || value == _e2; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.PackedIndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) + : SpanHelpers.NonPackedIndexOfAnyValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + Unsafe.As(ref _e1), + Unsafe.As(ref _e2), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) + : SpanHelpers.NonPackedIndexOfAnyValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + Unsafe.As(ref _e1), + Unsafe.As(ref _e2), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAny(_e0, _e1, _e2); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0, _e1, _e2); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs index 11fe35bd9d865..961de773e2579 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs @@ -39,7 +39,7 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOfAnyInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) + ? PackedSpanHelpers.PackedIndexOfAnyInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) : SpanHelpers.NonPackedIndexOfAnyInRangeUnsignedNumber>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _lowInclusive), @@ -49,7 +49,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAnyExcept(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOfAnyExceptInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) + ? PackedSpanHelpers.PackedIndexOfAnyExceptInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) : SpanHelpers.NonPackedIndexOfAnyInRangeUnsignedNumber>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _lowInclusive), diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs index bccdd87d21428..24c62bd92b969 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs @@ -48,7 +48,7 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) return values.Length switch { 2 => new IndexOfAny2ByteValues(values), - 3 => new IndexOfAny3Values(values), + 3 => new IndexOfAny3ByteValues(values), 4 => new IndexOfAny4Values(values), _ => new IndexOfAny5Values(values), }; @@ -80,7 +80,7 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) if (values.Length == 1) { char value = values[0]; - return SpanHelpers.CanUsePackedIndexOf(value) + return PackedSpanHelpers.CanUsePackedIndexOf(value) ? new IndexOfAny1CharValue(value) : new IndexOfAny1CharValue(value); } @@ -95,14 +95,19 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) { char value0 = values[0]; char value1 = values[1]; - return SpanHelpers.CanUsePackedIndexOf(value0) && SpanHelpers.CanUsePackedIndexOf(value1) + return PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) ? new IndexOfAny2CharValue(value0, value1) : new IndexOfAny2CharValue(value0, value1); } if (values.Length == 3) { - return new IndexOfAny3Values(values); + char value0 = values[0]; + char value1 = values[1]; + char value2 = values[2]; + return PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) && PackedSpanHelpers.CanUsePackedIndexOf(value2) + ? new IndexOfAny3CharValue(value0, value1, value2) + : new IndexOfAny3CharValue(value0, value1, value2); } // IndexOfAnyAsciiSearcher for chars is slower than IndexOfAny3Values, but faster than IndexOfAny4Values @@ -180,7 +185,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), } Debug.Assert(typeof(T) == typeof(char)); - return (IndexOfAnyValues)(object)(SpanHelpers.CanUsePackedIndexOf(min) && SpanHelpers.CanUsePackedIndexOf(max) + return (IndexOfAnyValues)(object)(PackedSpanHelpers.CanUsePackedIndexOf(min) && PackedSpanHelpers.CanUsePackedIndexOf(max) ? new IndexOfAnyCharValuesInRange(*(char*)&min, *(char*)&max) : new IndexOfAnyCharValuesInRange(*(char*)&min, *(char*)&max)); } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs similarity index 61% rename from src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.Packed.cs rename to src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs index 02d76e66077cb..8fce1de5e3216 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.Packed.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -9,19 +9,23 @@ using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 + #pragma warning disable 8500 // sizeof of managed types namespace System { - internal static partial class SpanHelpers // .Char.Packed + // This is a separate class instead of 'partial SpanHelpers' to hide the private helpers + // included in this file which are specific to the packed implementation. + internal static partial class PackedSpanHelpers { - internal static bool PackedIndexOfIsSupported => Sse2.IsSupported || AdvSimd.IsSupported; + public static bool PackedIndexOfIsSupported => Sse2.IsSupported || AdvSimd.IsSupported; // Not all values can benefit from packing the searchSpace. See comments in PackSources below. // On X86, the values must be in the [1, 254] range. // On ARM, the values must be in the [0, 254] range. [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe bool CanUsePackedIndexOf(T value) => + public static unsafe bool CanUsePackedIndexOf(T value) => PackedIndexOfIsSupported && RuntimeHelpers.IsBitwiseEquatable() && sizeof(T) == sizeof(ushort) && @@ -30,31 +34,181 @@ internal static unsafe bool CanUsePackedIndexOf(T value) => : *(ushort*)&value < 255u); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOf(ref char searchSpace, char value, int length) => - PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + public static int PackedIndexOf(ref char searchSpace, char value, int length) => + PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOfAnyExcept(ref char searchSpace, char value, int length) => - PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + public static int PackedIndexOfAnyExcept(ref char searchSpace, char value, int length) => + PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOfAny(ref char searchSpace, char value0, char value1, int length) => - PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + public static int PackedIndexOfAny(ref char searchSpace, char value0, char value1, int length) => + PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOfAnyExcept(ref char searchSpace, char value0, char value1, int length) => - PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + public static int PackedIndexOfAnyExcept(ref char searchSpace, char value0, char value1, int length) => + PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOfAnyInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => - PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + public static int PackedIndexOfAny(ref char searchSpace, char value0, char value1, char value2, int length) => + PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOfAnyExceptInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => - PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + public static int PackedIndexOfAnyExcept(ref char searchSpace, char value0, char value1, char value2, int length) => + PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int PackedIndexOfAnyInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => + PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int PackedIndexOfAnyExceptInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => + PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + + public static bool PackedContains(ref short searchSpace, short value, int length) + { + Debug.Assert(CanUsePackedIndexOf(value)); + + if (length < Vector128.Count) + { + nuint offset = 0; + + if (length >= 4) + { + length -= 4; + + if (searchSpace == value || + Unsafe.Add(ref searchSpace, 1) == value || + Unsafe.Add(ref searchSpace, 2) == value || + Unsafe.Add(ref searchSpace, 3) == value) + { + return true; + } + + offset = 4; + } + + while (length > 0) + { + length -= 1; + + if (Unsafe.Add(ref searchSpace, offset) == value) + { + return true; + } + + offset += 1; + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 packedValue = Vector256.Create((byte)value); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue, packedSource); + + if (result != Vector256.Zero) + { + return true; + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we're only interested in whether any value matched. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue, packedSource); + + if (result != Vector256.Zero) + { + return true; + } + } + } + else + { + Vector128 packedValue = Vector128.Create((byte)value); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue, packedSource); + + if (result != Vector128.Zero) + { + return true; + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we're only interested in whether any value matched. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue, packedSource); + + if (result != Vector128.Zero) + { + return true; + } + } + } + } + + return false; + } private static int PackedIndexOf(ref short searchSpace, short value, int length) - where TNegator : struct, INegator + where TNegator : struct, SpanHelpers.INegator { Debug.Assert(CanUsePackedIndexOf(value)); @@ -195,7 +349,7 @@ private static int PackedIndexOf(ref short searchSpace, short value, i } private static int PackedIndexOfAny(ref short searchSpace, short value0, short value1, int length) - where TNegator : struct, INegator + where TNegator : struct, SpanHelpers.INegator { Debug.Assert(CanUsePackedIndexOf(value0)); Debug.Assert(CanUsePackedIndexOf(value1)); @@ -344,8 +498,161 @@ private static int PackedIndexOfAny(ref short searchSpace, short value return -1; } + private static int PackedIndexOfAny(ref short searchSpace, short value0, short value1, short value2, int length) + where TNegator : struct, SpanHelpers.INegator + { + Debug.Assert(CanUsePackedIndexOf(value0)); + Debug.Assert(CanUsePackedIndexOf(value1)); + Debug.Assert(CanUsePackedIndexOf(value2)); + + if (length < Vector128.Count) + { + nuint offset = 0; + short lookUp; + + if (length >= 4) + { + length -= 4; + + lookUp = searchSpace; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 0; + lookUp = Unsafe.Add(ref searchSpace, 1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 1; + lookUp = Unsafe.Add(ref searchSpace, 2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 2; + lookUp = Unsafe.Add(ref searchSpace, 3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 3; + + offset = 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset; + + offset += 1; + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 packedValue0 = Vector256.Create((byte)value0); + Vector256 packedValue1 = Vector256.Create((byte)value1); + Vector256 packedValue2 = Vector256.Create((byte)value2); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource) | Vector256.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource) | Vector256.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + else + { + Vector128 packedValue0 = Vector128.Create((byte)value0); + Vector128 packedValue1 = Vector128.Create((byte)value1); + Vector128 packedValue2 = Vector128.Create((byte)value2); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource) | Vector128.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource) | Vector128.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + } + + return -1; + } + private static int PackedIndexOfAnyInRange(ref short searchSpace, short lowInclusive, short rangeInclusive, int length) - where TNegator : struct, INegator + where TNegator : struct, SpanHelpers.INegator { Debug.Assert(CanUsePackedIndexOf(lowInclusive)); Debug.Assert(CanUsePackedIndexOf((short)(lowInclusive + rangeInclusive))); @@ -504,13 +811,13 @@ private static Vector128 PackSources(Vector128 source0, Vector128 NegateIfNeeded(Vector128 result) - where TNegator : struct, INegator => - typeof(TNegator) == typeof(DontNegate) ? result : ~result; + where TNegator : struct, SpanHelpers.INegator => + typeof(TNegator) == typeof(SpanHelpers.DontNegate) ? result : ~result; [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector256 NegateIfNeeded(Vector256 result) - where TNegator : struct, INegator => - typeof(TNegator) == typeof(DontNegate) ? result : ~result; + where TNegator : struct, SpanHelpers.INegator => + typeof(TNegator) == typeof(SpanHelpers.DontNegate) ? result : ~result; [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeFirstIndex(ref short searchSpace, ref short current, Vector128 equals) @@ -528,7 +835,6 @@ private static int ComputeFirstIndex(ref short searchSpace, ref short current, V return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / sizeof(short)); } -#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeFirstIndexOverlapped(ref short searchSpace, ref short current0, ref short current1, Vector128 equals) { @@ -556,7 +862,6 @@ private static int ComputeFirstIndexOverlapped(ref short searchSpace, ref short } return offsetInVector + (int)(Unsafe.ByteOffset(ref searchSpace, ref current0) / sizeof(short)); } -#pragma warning restore IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint FixUpPackedVector256Mask(uint mask) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 7f222ec0f238a..c223af0e751ab 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -1303,8 +1303,19 @@ public static int SequenceCompareTo(ref T first, int firstLength, ref T secon return firstLength.CompareTo(secondLength); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe bool ContainsValueType(ref T searchSpace, T value, int length) where T : struct, INumber + { + if (PackedSpanHelpers.CanUsePackedIndexOf(value)) + { + return PackedSpanHelpers.PackedContains(ref Unsafe.As(ref searchSpace), *(short*)&value, length); + } + + return NonPackedContainsValueType(ref searchSpace, value, length); + } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] - internal static bool ContainsValueType(ref T searchSpace, T value, int length) where T : struct, INumber + internal static bool NonPackedContainsValueType(ref T searchSpace, T value, int length) where T : struct, INumber { Debug.Assert(length >= 0, "Expected non-negative length"); Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types"); @@ -1437,11 +1448,11 @@ private static unsafe int IndexOfValueType(ref TValue searchSp where TValue : struct, INumber where TNegator : struct, INegator { - if (CanUsePackedIndexOf(value)) + if (PackedSpanHelpers.CanUsePackedIndexOf(value)) { return typeof(TNegator) == typeof(DontNegate) - ? PackedIndexOf(ref Unsafe.As(ref searchSpace), *(char*)&value, length) - : PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value, length); + ? PackedSpanHelpers.PackedIndexOf(ref Unsafe.As(ref searchSpace), *(char*)&value, length) + : PackedSpanHelpers.PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value, length); } return NonPackedIndexOfValueType(ref searchSpace, value, length); @@ -1594,11 +1605,11 @@ private static unsafe int IndexOfAnyValueType(ref TValue searc where TValue : struct, INumber where TNegator : struct, INegator { - if (CanUsePackedIndexOf(value0) && CanUsePackedIndexOf(value1)) + if (PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1)) { return typeof(TNegator) == typeof(DontNegate) - ? PackedIndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length) - : PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length); + ? PackedSpanHelpers.PackedIndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length) + : PackedSpanHelpers.PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length); } return NonPackedIndexOfAnyValueType(ref searchSpace, value0, value1, length); @@ -1766,8 +1777,23 @@ internal static int IndexOfAnyValueType(ref T searchSpace, T value0, T value1 internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, INumber => IndexOfAnyValueType>(ref searchSpace, value0, value1, value2, length); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + if (PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) && PackedSpanHelpers.CanUsePackedIndexOf(value2)) + { + return typeof(TNegator) == typeof(DontNegate) + ? PackedSpanHelpers.PackedIndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, *(char*)&value2, length) + : PackedSpanHelpers.PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, *(char*)&value2, length); + } + + return NonPackedIndexOfAnyValueType(ref searchSpace, value0, value1, value2, length); + } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length) + internal static int NonPackedIndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length) where TValue : struct, INumber where TNegator : struct, INegator { @@ -3059,15 +3085,15 @@ private static unsafe int IndexOfAnyInRangeUnsignedNumber(ref T sea where T : struct, IUnsignedNumber, IComparisonOperators where TNegator : struct, INegator { - if (CanUsePackedIndexOf(lowInclusive) && CanUsePackedIndexOf(highInclusive) && highInclusive >= lowInclusive) + if (PackedSpanHelpers.CanUsePackedIndexOf(lowInclusive) && PackedSpanHelpers.CanUsePackedIndexOf(highInclusive) && highInclusive >= lowInclusive) { ref char charSearchSpace = ref Unsafe.As(ref searchSpace); char charLowInclusive = *(char*)&lowInclusive; char charRange = (char)(*(char*)&highInclusive - charLowInclusive); return typeof(TNegator) == typeof(DontNegate) - ? PackedIndexOfAnyInRange(ref charSearchSpace, charLowInclusive, charRange, length) - : PackedIndexOfAnyExceptInRange(ref charSearchSpace, charLowInclusive, charRange, length); + ? PackedSpanHelpers.PackedIndexOfAnyInRange(ref charSearchSpace, charLowInclusive, charRange, length) + : PackedSpanHelpers.PackedIndexOfAnyExceptInRange(ref charSearchSpace, charLowInclusive, charRange, length); } return NonPackedIndexOfAnyInRangeUnsignedNumber(ref searchSpace, lowInclusive, highInclusive, length); diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs index c84b07a3c9f75..feda6cc648894 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs @@ -78,8 +78,8 @@ private int IndexOfCharOrdinalIgnoreCase(char value) { char valueUc = (char)(value | 0x20); char valueLc = (char)(value & ~0x20); - return SpanHelpers.PackedIndexOfIsSupported - ? SpanHelpers.PackedIndexOfAny(ref _firstChar, valueLc, valueUc, Length) + return PackedSpanHelpers.PackedIndexOfIsSupported + ? PackedSpanHelpers.PackedIndexOfAny(ref _firstChar, valueLc, valueUc, Length) : SpanHelpers.IndexOfAnyChar(ref _firstChar, valueLc, valueUc, Length); } From a59cff5f07f291891884a1bd76d5cca63b612fbb Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Thu, 29 Dec 2022 19:34:41 +0100 Subject: [PATCH 3/4] Stop using PackedIndexOf on ARM --- .../src/System/Globalization/Ordinal.cs | 2 +- .../IndexOfAnyValues/IndexOfAny1CharValue.cs | 4 +- .../IndexOfAnyValues/IndexOfAny2CharValues.cs | 4 +- .../IndexOfAnyValues/IndexOfAny3CharValues.cs | 4 +- .../IndexOfAnyCharValuesInRange.cs | 4 +- .../src/System/SpanHelpers.Packed.cs | 60 +++++++++---------- .../src/System/SpanHelpers.T.cs | 18 +++--- .../src/System/String.Searching.cs | 2 +- 8 files changed, 46 insertions(+), 52 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index f5157b0cbba8a..1dd36f6730661 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -340,7 +340,7 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan source, ReadOnly // Do a quick search for the first element of "value". int relativeIndex = isLetter ? PackedSpanHelpers.PackedIndexOfIsSupported - ? PackedSpanHelpers.PackedIndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) + ? PackedSpanHelpers.IndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength); if (relativeIndex < 0) diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs index 8461c8b2b50c5..762f1872cb64c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs @@ -23,7 +23,7 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => TShouldUsePacked.Value - ? PackedSpanHelpers.PackedIndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length) + ? PackedSpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length) : SpanHelpers.NonPackedIndexOfValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), @@ -32,7 +32,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAnyExcept(ReadOnlySpan span) => TShouldUsePacked.Value - ? PackedSpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length) + ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length) : SpanHelpers.NonPackedIndexOfValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs index 8560999b3d4bf..f649ffc752657 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs @@ -23,7 +23,7 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => TShouldUsePacked.Value - ? PackedSpanHelpers.PackedIndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) + ? PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) : SpanHelpers.NonPackedIndexOfAnyValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), @@ -33,7 +33,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAnyExcept(ReadOnlySpan span) => TShouldUsePacked.Value - ? PackedSpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) + ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) : SpanHelpers.NonPackedIndexOfAnyValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs index 519fe27245def..5ae583b0f174b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs @@ -23,7 +23,7 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => TShouldUsePacked.Value - ? PackedSpanHelpers.PackedIndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) + ? PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) : SpanHelpers.NonPackedIndexOfAnyValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), @@ -34,7 +34,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAnyExcept(ReadOnlySpan span) => TShouldUsePacked.Value - ? PackedSpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) + ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) : SpanHelpers.NonPackedIndexOfAnyValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs index 961de773e2579..018ef273b4839 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs @@ -39,7 +39,7 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => TShouldUsePacked.Value - ? PackedSpanHelpers.PackedIndexOfAnyInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) + ? PackedSpanHelpers.IndexOfAnyInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) : SpanHelpers.NonPackedIndexOfAnyInRangeUnsignedNumber>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _lowInclusive), @@ -49,7 +49,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAnyExcept(ReadOnlySpan span) => TShouldUsePacked.Value - ? PackedSpanHelpers.PackedIndexOfAnyExceptInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) + ? PackedSpanHelpers.IndexOfAnyExceptInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) : SpanHelpers.NonPackedIndexOfAnyInRangeUnsignedNumber>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _lowInclusive), diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs index 8fce1de5e3216..91d516a7f5d57 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -6,7 +6,6 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; #pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 @@ -19,53 +18,51 @@ namespace System // included in this file which are specific to the packed implementation. internal static partial class PackedSpanHelpers { - public static bool PackedIndexOfIsSupported => Sse2.IsSupported || AdvSimd.IsSupported; + // We only do this optimization on X86 as the packing is noticeably more expensive on ARM in comparison. + // While the impact on worst-case (match at the start) is minimal on X86, it's prohibitively large on ARM. + public static bool PackedIndexOfIsSupported => Sse2.IsSupported; // Not all values can benefit from packing the searchSpace. See comments in PackSources below. - // On X86, the values must be in the [1, 254] range. - // On ARM, the values must be in the [0, 254] range. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe bool CanUsePackedIndexOf(T value) => PackedIndexOfIsSupported && RuntimeHelpers.IsBitwiseEquatable() && sizeof(T) == sizeof(ushort) && - (Sse2.IsSupported - ? *(ushort*)&value - 1u < 254u - : *(ushort*)&value < 255u); + *(ushort*)&value - 1u < 254u; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int PackedIndexOf(ref char searchSpace, char value, int length) => - PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + public static int IndexOf(ref char searchSpace, char value, int length) => + IndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int PackedIndexOfAnyExcept(ref char searchSpace, char value, int length) => - PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + public static int IndexOfAnyExcept(ref char searchSpace, char value, int length) => + IndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int PackedIndexOfAny(ref char searchSpace, char value0, char value1, int length) => - PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + public static int IndexOfAny(ref char searchSpace, char value0, char value1, int length) => + IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int PackedIndexOfAnyExcept(ref char searchSpace, char value0, char value1, int length) => - PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value1, int length) => + IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int PackedIndexOfAny(ref char searchSpace, char value0, char value1, char value2, int length) => - PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + public static int IndexOfAny(ref char searchSpace, char value0, char value1, char value2, int length) => + IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int PackedIndexOfAnyExcept(ref char searchSpace, char value0, char value1, char value2, int length) => - PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value1, char value2, int length) => + IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int PackedIndexOfAnyInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => - PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + public static int IndexOfAnyInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => + IndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int PackedIndexOfAnyExceptInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => - PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + public static int IndexOfAnyExceptInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => + IndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); - public static bool PackedContains(ref short searchSpace, short value, int length) + public static bool Contains(ref short searchSpace, short value, int length) { Debug.Assert(CanUsePackedIndexOf(value)); @@ -207,7 +204,7 @@ public static bool PackedContains(ref short searchSpace, short value, int length return false; } - private static int PackedIndexOf(ref short searchSpace, short value, int length) + private static int IndexOf(ref short searchSpace, short value, int length) where TNegator : struct, SpanHelpers.INegator { Debug.Assert(CanUsePackedIndexOf(value)); @@ -348,7 +345,7 @@ private static int PackedIndexOf(ref short searchSpace, short value, i return -1; } - private static int PackedIndexOfAny(ref short searchSpace, short value0, short value1, int length) + private static int IndexOfAny(ref short searchSpace, short value0, short value1, int length) where TNegator : struct, SpanHelpers.INegator { Debug.Assert(CanUsePackedIndexOf(value0)); @@ -498,7 +495,7 @@ private static int PackedIndexOfAny(ref short searchSpace, short value return -1; } - private static int PackedIndexOfAny(ref short searchSpace, short value0, short value1, short value2, int length) + private static int IndexOfAny(ref short searchSpace, short value0, short value1, short value2, int length) where TNegator : struct, SpanHelpers.INegator { Debug.Assert(CanUsePackedIndexOf(value0)); @@ -651,7 +648,7 @@ private static int PackedIndexOfAny(ref short searchSpace, short value return -1; } - private static int PackedIndexOfAnyInRange(ref short searchSpace, short lowInclusive, short rangeInclusive, int length) + private static int IndexOfAnyInRange(ref short searchSpace, short lowInclusive, short rangeInclusive, int length) where TNegator : struct, SpanHelpers.INegator { Debug.Assert(CanUsePackedIndexOf(lowInclusive)); @@ -798,15 +795,12 @@ private static Vector256 PackSources(Vector256 source0, Vector256 PackSources(Vector128 source0, Vector128 source1) { + Debug.Assert(Sse2.IsSupported); // Pack two vectors of characters into bytes. While the type is Vector128, these are really UInt16 characters. // X86: Downcast every character using saturation. // - Values <= 32767 result in min(value, 255). // - Values > 32767 result in 0. Because of this we can't accept needles that contain 0. - // ARM64: Do narrowing saturation over unsigned values. - // - All values result in min(value, 255) - return Sse2.IsSupported - ? Sse2.PackUnsignedSaturate(source0, source1).AsByte() - : AdvSimd.ExtractNarrowingSaturateUpper(AdvSimd.ExtractNarrowingSaturateLower(source0.AsUInt16()), source1.AsUInt16()); + return Sse2.PackUnsignedSaturate(source0, source1).AsByte(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index c223af0e751ab..d3d643f13ee64 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -1308,7 +1308,7 @@ internal static unsafe bool ContainsValueType(ref T searchSpace, T value, int { if (PackedSpanHelpers.CanUsePackedIndexOf(value)) { - return PackedSpanHelpers.PackedContains(ref Unsafe.As(ref searchSpace), *(short*)&value, length); + return PackedSpanHelpers.Contains(ref Unsafe.As(ref searchSpace), *(short*)&value, length); } return NonPackedContainsValueType(ref searchSpace, value, length); @@ -1451,8 +1451,8 @@ private static unsafe int IndexOfValueType(ref TValue searchSp if (PackedSpanHelpers.CanUsePackedIndexOf(value)) { return typeof(TNegator) == typeof(DontNegate) - ? PackedSpanHelpers.PackedIndexOf(ref Unsafe.As(ref searchSpace), *(char*)&value, length) - : PackedSpanHelpers.PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value, length); + ? PackedSpanHelpers.IndexOf(ref Unsafe.As(ref searchSpace), *(char*)&value, length) + : PackedSpanHelpers.IndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value, length); } return NonPackedIndexOfValueType(ref searchSpace, value, length); @@ -1608,8 +1608,8 @@ private static unsafe int IndexOfAnyValueType(ref TValue searc if (PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1)) { return typeof(TNegator) == typeof(DontNegate) - ? PackedSpanHelpers.PackedIndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length) - : PackedSpanHelpers.PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length); + ? PackedSpanHelpers.IndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length) + : PackedSpanHelpers.IndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length); } return NonPackedIndexOfAnyValueType(ref searchSpace, value0, value1, length); @@ -1785,8 +1785,8 @@ private static unsafe int IndexOfAnyValueType(ref TValue searc if (PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) && PackedSpanHelpers.CanUsePackedIndexOf(value2)) { return typeof(TNegator) == typeof(DontNegate) - ? PackedSpanHelpers.PackedIndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, *(char*)&value2, length) - : PackedSpanHelpers.PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, *(char*)&value2, length); + ? PackedSpanHelpers.IndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, *(char*)&value2, length) + : PackedSpanHelpers.IndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, *(char*)&value2, length); } return NonPackedIndexOfAnyValueType(ref searchSpace, value0, value1, value2, length); @@ -3092,8 +3092,8 @@ private static unsafe int IndexOfAnyInRangeUnsignedNumber(ref T sea char charRange = (char)(*(char*)&highInclusive - charLowInclusive); return typeof(TNegator) == typeof(DontNegate) - ? PackedSpanHelpers.PackedIndexOfAnyInRange(ref charSearchSpace, charLowInclusive, charRange, length) - : PackedSpanHelpers.PackedIndexOfAnyExceptInRange(ref charSearchSpace, charLowInclusive, charRange, length); + ? PackedSpanHelpers.IndexOfAnyInRange(ref charSearchSpace, charLowInclusive, charRange, length) + : PackedSpanHelpers.IndexOfAnyExceptInRange(ref charSearchSpace, charLowInclusive, charRange, length); } return NonPackedIndexOfAnyInRangeUnsignedNumber(ref searchSpace, lowInclusive, highInclusive, length); diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs index feda6cc648894..331fd3392e988 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs @@ -79,7 +79,7 @@ private int IndexOfCharOrdinalIgnoreCase(char value) char valueUc = (char)(value | 0x20); char valueLc = (char)(value & ~0x20); return PackedSpanHelpers.PackedIndexOfIsSupported - ? PackedSpanHelpers.PackedIndexOfAny(ref _firstChar, valueLc, valueUc, Length) + ? PackedSpanHelpers.IndexOfAny(ref _firstChar, valueLc, valueUc, Length) : SpanHelpers.IndexOfAnyChar(ref _firstChar, valueLc, valueUc, Length); } From 72ca07904ef6946a1c0b93197795f480cdaa909d Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 1 Jan 2023 07:36:09 -0800 Subject: [PATCH 4/4] Improve code comment --- .../System.Private.CoreLib/src/System/SpanHelpers.Packed.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs index 91d516a7f5d57..3347348ee187a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -18,8 +18,8 @@ namespace System // included in this file which are specific to the packed implementation. internal static partial class PackedSpanHelpers { - // We only do this optimization on X86 as the packing is noticeably more expensive on ARM in comparison. - // While the impact on worst-case (match at the start) is minimal on X86, it's prohibitively large on ARM. + // We only do this optimization if we have support for X86 intrinsics (Sse2) as the packing is noticeably cheaper compared to ARM (AdvSimd). + // While the impact on the worst-case (match at the start) is minimal on X86, it's prohibitively large on ARM. public static bool PackedIndexOfIsSupported => Sse2.IsSupported; // Not all values can benefit from packing the searchSpace. See comments in PackSources below.