From fe94d1ba4ac6cb21c636757d4132fd87f34a3b57 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 13 Jul 2023 08:48:09 -0700 Subject: [PATCH 1/5] Ensure that INumberBase implements IUtf8SpanFormattable --- .../src/System/Numerics/INumberBase.cs | 39 ++++++++++++++++++- .../System.Runtime/ref/System.Runtime.cs | 3 +- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs index 4d943b1933888..77e0761b39fad 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs @@ -27,6 +27,7 @@ public interface INumberBase ISubtractionOperators, IUnaryPlusOperators, IUnaryNegationOperators, + IUtf8SpanFormattable, IUtf8SpanParsable where TSelf : INumberBase? { @@ -297,7 +298,7 @@ static virtual TSelf Parse(ReadOnlySpan utf8Text, NumberStyles style, IFor if (textMaxCharCount < 256) { utf16TextArray = null; - utf16Text = stackalloc char[512]; + utf16Text = stackalloc char[256]; } else { @@ -425,7 +426,7 @@ static virtual bool TryParse(ReadOnlySpan utf8Text, NumberStyles style, IF if (textMaxCharCount < 256) { utf16TextArray = null; - utf16Text = stackalloc char[512]; + utf16Text = stackalloc char[256]; } else { @@ -456,6 +457,40 @@ static virtual bool TryParse(ReadOnlySpan utf8Text, NumberStyles style, IF return succeeded; } + bool IUtf8SpanFormattable.TryFormat(Span utf8Destination, out int bytesWritten, ReadOnlySpan format, IFormatProvider? provider) + { + char[]? utf16DestinationArray; + scoped Span utf16Destination; + int destinationMaxCharCount = Encoding.UTF8.GetMaxCharCount(utf8Destination.Length); + + if (destinationMaxCharCount < 256) + { + utf16DestinationArray = null; + utf16Destination = stackalloc char[256]; + } + else + { + utf16DestinationArray = ArrayPool.Shared.Rent(destinationMaxCharCount); + utf16Destination = utf16DestinationArray.AsSpan(0, destinationMaxCharCount); + } + + if (!TryFormat(utf16Destination, out int charsWritten, format, provider)) + { + bytesWritten = 0; + return false; + } + + OperationStatus utf8DestinationStatus = Utf8.FromUtf16(utf16Destination, utf8Destination, out _, out bytesWritten, replaceInvalidSequences: false); + + if (utf8DestinationStatus != OperationStatus.Done) + { + bytesWritten = 0; + return false; + } + + return true; + } + static TSelf IUtf8SpanParsable.Parse(ReadOnlySpan utf8Text, IFormatProvider? provider) { // Convert text using stackalloc for <= 256 characters and ArrayPool otherwise diff --git a/src/libraries/System.Runtime/ref/System.Runtime.cs b/src/libraries/System.Runtime/ref/System.Runtime.cs index 08e6907f7e80e..58438a47c8c8f 100644 --- a/src/libraries/System.Runtime/ref/System.Runtime.cs +++ b/src/libraries/System.Runtime/ref/System.Runtime.cs @@ -10793,7 +10793,7 @@ public partial interface IMultiplyOperators where TSelf static virtual TResult operator checked *(TSelf left, TOther right) { throw null; } static abstract TResult operator *(TSelf left, TOther right); } - public partial interface INumberBase : System.IEquatable, System.IFormattable, System.IParsable, System.ISpanFormattable, System.ISpanParsable, System.Numerics.IAdditionOperators, System.Numerics.IAdditiveIdentity, System.Numerics.IDecrementOperators, System.Numerics.IDivisionOperators, System.Numerics.IEqualityOperators, System.Numerics.IIncrementOperators, System.Numerics.IMultiplicativeIdentity, System.Numerics.IMultiplyOperators, System.Numerics.ISubtractionOperators, System.Numerics.IUnaryNegationOperators, System.Numerics.IUnaryPlusOperators, System.IUtf8SpanParsable where TSelf : System.Numerics.INumberBase? + public partial interface INumberBase : System.IEquatable, System.IFormattable, System.IParsable, System.ISpanFormattable, System.ISpanParsable, System.Numerics.IAdditionOperators, System.Numerics.IAdditiveIdentity, System.Numerics.IDecrementOperators, System.Numerics.IDivisionOperators, System.Numerics.IEqualityOperators, System.Numerics.IIncrementOperators, System.Numerics.IMultiplicativeIdentity, System.Numerics.IMultiplyOperators, System.Numerics.ISubtractionOperators, System.Numerics.IUnaryNegationOperators, System.Numerics.IUnaryPlusOperators, System.IUtf8SpanFormattable, System.IUtf8SpanParsable where TSelf : System.Numerics.INumberBase? { static abstract TSelf One { get; } static abstract int Radix { get; } @@ -10835,6 +10835,7 @@ static virtual TSelf CreateTruncating(TOther value) static virtual TSelf Parse(System.ReadOnlySpan utf8Text, System.Globalization.NumberStyles style, System.IFormatProvider? provider) { throw null; } static abstract TSelf Parse(System.ReadOnlySpan s, System.Globalization.NumberStyles style, System.IFormatProvider? provider); static abstract TSelf Parse(string s, System.Globalization.NumberStyles style, System.IFormatProvider? provider); + bool System.IUtf8SpanFormattable.TryFormat(System.Span utf8Destination, out int bytesWritten, System.ReadOnlySpan format, System.IFormatProvider? provider) { throw null; } static TSelf System.IUtf8SpanParsable.Parse(System.ReadOnlySpan utf8Text, System.IFormatProvider? provider) { throw null; } static bool System.IUtf8SpanParsable.TryParse(System.ReadOnlySpan utf8Text, System.IFormatProvider? provider, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out TSelf result) { throw null; } protected static abstract bool TryConvertFromChecked(TOther value, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out TSelf result) From 183c7b2a78b3f72d17d583bbde1fdd5a8801b12d Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 13 Jul 2023 08:57:08 -0700 Subject: [PATCH 2/5] Ensure we return the rented buffers in the IUtf8SpanFormattable.TryFormat DIM --- .../src/System/Numerics/INumberBase.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs index 77e0761b39fad..5a7a762d58cfd 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs @@ -476,12 +476,24 @@ bool IUtf8SpanFormattable.TryFormat(Span utf8Destination, out int bytesWri if (!TryFormat(utf16Destination, out int charsWritten, format, provider)) { + if (utf16DestinationArray != null) + { + // Return rented buffers if necessary + ArrayPool.Shared.Return(utf16DestinationArray); + } + bytesWritten = 0; return false; } OperationStatus utf8DestinationStatus = Utf8.FromUtf16(utf16Destination, utf8Destination, out _, out bytesWritten, replaceInvalidSequences: false); + if (utf16DestinationArray != null) + { + // Return rented buffers if necessary + ArrayPool.Shared.Return(utf16DestinationArray); + } + if (utf8DestinationStatus != OperationStatus.Done) { bytesWritten = 0; From fc54f3013f4ae6c9763b397d273be07a564b83ac Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 13 Jul 2023 16:57:16 -0700 Subject: [PATCH 3/5] Remember to slice the utf16Destination buffer and ensure we throw if we couldn't transcode back to valid UTF-8 in the DIM --- .../src/Resources/Strings.resx | 3 +++ .../src/System/Numerics/INumberBase.cs | 16 ++++++++++++---- .../src/System/ThrowHelper.cs | 6 ++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx index 60ef1fbf28cad..0881f2a0438c2 100644 --- a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx +++ b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx @@ -2608,6 +2608,9 @@ Handle is not pinned. + + Formatted string contains characters not representable as valid UTF8. + Hashtable insert failed. Load factor too high. The most common cause is multiple threads writing to the Hashtable simultaneously. diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs index 5a7a762d58cfd..c7ac6a2c79c84 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs @@ -486,6 +486,9 @@ bool IUtf8SpanFormattable.TryFormat(Span utf8Destination, out int bytesWri return false; } + // Make sure we slice the buffer to just the characters written + utf16Destination = utf16Destination.Slice(charsWritten); + OperationStatus utf8DestinationStatus = Utf8.FromUtf16(utf16Destination, utf8Destination, out _, out bytesWritten, replaceInvalidSequences: false); if (utf16DestinationArray != null) @@ -494,13 +497,18 @@ bool IUtf8SpanFormattable.TryFormat(Span utf8Destination, out int bytesWri ArrayPool.Shared.Return(utf16DestinationArray); } - if (utf8DestinationStatus != OperationStatus.Done) + if (utf8DestinationStatus == OperationStatus.Done) { - bytesWritten = 0; - return false; + return true; + } + + if (utf8DestinationStatus != OperationStatus.DestinationTooSmall) + { + ThrowHelper.ThrowInvalidOperationException_InvalidUtf8(); } - return true; + bytesWritten = 0; + return false; } static TSelf IUtf8SpanParsable.Parse(ReadOnlySpan utf8Text, IFormatProvider? provider) diff --git a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs index 71e2904b8dfbb..2d4700576196c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs @@ -529,6 +529,12 @@ internal static void ThrowArraySegmentCtorValidationFailedExceptions(Array? arra throw GetArraySegmentCtorValidationFailedException(array, offset, count); } + [DoesNotReturn] + internal static void ThrowInvalidOperationException_InvalidUtf8() + { + throw new InvalidOperationException(SR.InvalidOperation_InvalidUtf8); + } + [DoesNotReturn] internal static void ThrowFormatException_BadFormatSpecifier() { From 0c8d6b3bbcbe8ed173ec27895ab74ce5b348e2a8 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 13 Jul 2023 17:13:35 -0700 Subject: [PATCH 4/5] Update src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs Co-authored-by: Miha Zupan --- .../System.Private.CoreLib/src/System/Numerics/INumberBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs index c7ac6a2c79c84..4368605183d2e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs @@ -487,7 +487,7 @@ bool IUtf8SpanFormattable.TryFormat(Span utf8Destination, out int bytesWri } // Make sure we slice the buffer to just the characters written - utf16Destination = utf16Destination.Slice(charsWritten); + utf16Destination = utf16Destination.Slice(0, charsWritten); OperationStatus utf8DestinationStatus = Utf8.FromUtf16(utf16Destination, utf8Destination, out _, out bytesWritten, replaceInvalidSequences: false); From 9ab160404032b5220900e94c044ff7323e7c31a9 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 13 Jul 2023 20:04:38 -0700 Subject: [PATCH 5/5] Update src/libraries/System.Private.CoreLib/src/Resources/Strings.resx Co-authored-by: Dan Moseley --- src/libraries/System.Private.CoreLib/src/Resources/Strings.resx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx index 0881f2a0438c2..79f75cc270672 100644 --- a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx +++ b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx @@ -2609,7 +2609,7 @@ Handle is not pinned. - Formatted string contains characters not representable as valid UTF8. + Formatted string contains characters not representable as valid UTF-8. Hashtable insert failed. Load factor too high. The most common cause is multiple threads writing to the Hashtable simultaneously.