diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index efb7f5d50d795..8c6a7f1420d43 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -6,6 +6,8 @@ using System.Globalization; using System.Numerics; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; using System.Text; using Internal.Runtime.CompilerServices; @@ -1491,78 +1493,137 @@ private string[] SplitWithPostProcessing(ReadOnlySpan sepList, ReadOnlySpan /// to store indexes private void MakeSeparatorList(ReadOnlySpan separators, ref ValueListBuilder sepListBuilder) { - char sep0, sep1, sep2; - - switch (separators.Length) + // Special-case no separators to mean any whitespace is a separator. + if (separators.Length == 0) { - // Special-case no separators to mean any whitespace is a separator. - case 0: - for (int i = 0; i < Length; i++) + for (int i = 0; i < Length; i++) + { + if (char.IsWhiteSpace(this[i])) { - if (char.IsWhiteSpace(this[i])) - { - sepListBuilder.Append(i); - } + sepListBuilder.Append(i); } - break; + } + } - // Special-case the common cases of 1, 2, and 3 separators, with manual comparisons against each separator. - case 1: - sep0 = separators[0]; - for (int i = 0; i < Length; i++) + // Special-case the common cases of 1, 2, and 3 separators, with manual comparisons against each separator. + else if (separators.Length <= 3) + { + char sep0, sep1, sep2; + sep0 = separators[0]; + sep1 = separators.Length > 1 ? separators[1] : sep0; + sep2 = separators.Length > 2 ? separators[2] : sep1; + + if (Length >= 16 && Sse41.IsSupported) + { + MakeSeparatorListVectorized(ref sepListBuilder, sep0, sep1, sep2); + return; + } + + for (int i = 0; i < Length; i++) + { + char c = this[i]; + if (c == sep0 || c == sep1 || c == sep2) { - if (this[i] == sep0) - { - sepListBuilder.Append(i); - } + sepListBuilder.Append(i); } - break; - case 2: - sep0 = separators[0]; - sep1 = separators[1]; + } + } + + // Handle > 3 separators with a probabilistic map, ala IndexOfAny. + // This optimizes for chars being unlikely to match a separator. + else + { + unsafe + { + ProbabilisticMap map = default; + uint* charMap = (uint*)↦ + InitializeProbabilisticMap(charMap, separators); + for (int i = 0; i < Length; i++) { char c = this[i]; - if (c == sep0 || c == sep1) + if (IsCharBitSet(charMap, (byte)c) && IsCharBitSet(charMap, (byte)(c >> 8)) && + separators.Contains(c)) { sepListBuilder.Append(i); } } - break; - case 3: - sep0 = separators[0]; - sep1 = separators[1]; - sep2 = separators[2]; - for (int i = 0; i < Length; i++) + } + } + } + + private void MakeSeparatorListVectorized(ref ValueListBuilder sepListBuilder, char c, char c2, char c3) + { + // Redundant test so we won't prejit remainder of this method + // on platforms without SSE. + if (!Sse41.IsSupported) + { + throw new PlatformNotSupportedException(); + } + + // Constant that allows for the truncation of 16-bit (FFFF/0000) values within a register to 4-bit (F/0) + Vector128 shuffleConstant = Vector128.Create(0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + + Vector128 v1 = Vector128.Create(c); + Vector128 v2 = Vector128.Create(c2); + Vector128 v3 = Vector128.Create(c3); + + ref char c0 = ref MemoryMarshal.GetReference(this.AsSpan()); + int cond = Length & -Vector128.Count; + int i = 0; + + for (; i < cond; i += Vector128.Count) + { + Vector128 charVector = ReadVector(ref c0, i); + Vector128 cmp = Sse2.CompareEqual(charVector, v1); + + cmp = Sse2.Or(Sse2.CompareEqual(charVector, v2), cmp); + cmp = Sse2.Or(Sse2.CompareEqual(charVector, v3), cmp); + + if (Sse41.TestZ(cmp, cmp)) { continue; } + + Vector128 mask = Sse2.ShiftRightLogical(cmp.AsUInt64(), 4).AsByte(); + mask = Ssse3.Shuffle(mask, shuffleConstant); + + uint lowBits = Sse2.ConvertToUInt32(mask.AsUInt32()); + mask = Sse2.ShiftRightLogical(mask.AsUInt64(), 32).AsByte(); + uint highBits = Sse2.ConvertToUInt32(mask.AsUInt32()); + + for (int idx = i; lowBits != 0; idx++) + { + if ((lowBits & 0xF) != 0) { - char c = this[i]; - if (c == sep0 || c == sep1 || c == sep2) - { - sepListBuilder.Append(i); - } + sepListBuilder.Append(idx); } - break; - // Handle > 3 separators with a probabilistic map, ala IndexOfAny. - // This optimizes for chars being unlikely to match a separator. - default: - unsafe - { - ProbabilisticMap map = default; - uint* charMap = (uint*)↦ - InitializeProbabilisticMap(charMap, separators); + lowBits >>= 8; + } - for (int i = 0; i < Length; i++) - { - char c = this[i]; - if (IsCharBitSet(charMap, (byte)c) && IsCharBitSet(charMap, (byte)(c >> 8)) && - separators.Contains(c)) - { - sepListBuilder.Append(i); - } - } + for (int idx = i + 4; highBits != 0; idx++) + { + if ((highBits & 0xF) != 0) + { + sepListBuilder.Append(idx); } - break; + + highBits >>= 8; + } + } + + for (; i < Length; i++) + { + char curr = Unsafe.Add(ref c0, (IntPtr)(uint)i); + if (curr == c || curr == c2 || curr == c3) + { + sepListBuilder.Append(i); + } + } + + static Vector128 ReadVector(ref char c0, int offset) + { + ref char ci = ref Unsafe.Add(ref c0, (IntPtr)(uint)offset); + ref byte b = ref Unsafe.As(ref ci); + return Unsafe.ReadUnaligned>(ref b); } }