From aaa38036e3d2e4f73c8c327ca19688ecaaf9a3b8 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Mon, 1 Jul 2024 18:20:23 +0100 Subject: [PATCH 01/10] Attempt at improving JsonNode.DeepEquals numeric equality. --- .../src/System/Text/Json/Document/JsonElement.cs | 14 ++++++++++++-- .../JsonNode/JsonValueTests.cs | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs index f5d2ff099e723..d1663ad3eece2 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers.Text; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; @@ -1241,8 +1242,17 @@ internal static bool DeepEquals(JsonElement left, JsonElement right) return true; case JsonValueKind.Number: - // JSON numbers are equal if their raw representations are equal. - return left.GetRawValue().Span.SequenceEqual(right.GetRawValue().Span); + ReadOnlySpan leftRawValue = left.GetRawValue().Span; + ReadOnlySpan rightRawValue = right.GetRawValue().Span; + + int bytesConsumed; + if (Utf8Parser.TryParse(leftRawValue, out decimal leftNumber, out bytesConsumed) && bytesConsumed == leftRawValue.Length && + Utf8Parser.TryParse(rightRawValue, out decimal rightNumber, out bytesConsumed) && bytesConsumed == rightRawValue.Length) + { + return leftNumber == rightNumber; + } + + return leftRawValue.SequenceEqual(rightRawValue); case JsonValueKind.String: if (right.ValueIsEscaped) diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs index 94505ff194149..12be1ef42f018 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs @@ -380,6 +380,20 @@ public static void DeepEqualsPrimitiveType() JsonNodeTests.AssertNotDeepEqual(JsonValue.Create(10), JsonValue.Create("10")); } + [Theory] + [InlineData("1", "1.0", true)] + [InlineData("-0.0", "0", true)] + [InlineData("-1.1e3", "-1100", true)] + [InlineData("79228162514264337593543950336", "792281625142643375935439503360e-1", false)] // Not equal since it exceeds decimal.MaxValue + [InlineData("1.75e+300", "1.75E+300", false)] // Not equal due to case difference in exponent + public static void DeepEqualsNumericType(string leftStr, string rightStr, bool areEqual) + { + JsonNode left = JsonNode.Parse(leftStr); + JsonNode right = JsonNode.Parse(rightStr); + + Assert.Equal(areEqual, JsonNode.DeepEquals(left, right)); + } + [Fact] public static void DeepEqualsJsonElement() { From 73224d5ca999b962d99aeb9af32f24f5e071b17b Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 2 Jul 2024 03:13:13 +0100 Subject: [PATCH 02/10] Implement arbitrary-precision decimal equality comparison. --- .../System/Text/Json/Document/JsonElement.cs | 12 +- .../src/System/Text/Json/JsonHelpers.cs | 235 ++++++++++++++++++ .../JsonNode/JsonValueTests.cs | 85 ++++++- 3 files changed, 314 insertions(+), 18 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs index d1663ad3eece2..0a08875c7c562 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs @@ -1242,17 +1242,7 @@ internal static bool DeepEquals(JsonElement left, JsonElement right) return true; case JsonValueKind.Number: - ReadOnlySpan leftRawValue = left.GetRawValue().Span; - ReadOnlySpan rightRawValue = right.GetRawValue().Span; - - int bytesConsumed; - if (Utf8Parser.TryParse(leftRawValue, out decimal leftNumber, out bytesConsumed) && bytesConsumed == leftRawValue.Length && - Utf8Parser.TryParse(rightRawValue, out decimal rightNumber, out bytesConsumed) && bytesConsumed == rightRawValue.Length) - { - return leftNumber == rightNumber; - } - - return leftRawValue.SequenceEqual(rightRawValue); + return JsonHelpers.AreEqualJsonNumbers(left.GetRawValue().Span, right.GetRawValue().Span); case JsonValueKind.String: if (right.ValueIsEscaped) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs index 63e2e177f49eb..3536d283247d4 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Buffers; +using System.Buffers.Text; using System.Collections; using System.Collections.Generic; using System.Diagnostics; @@ -260,5 +261,239 @@ public static bool HasAllSet(this BitArray bitArray) #else private static Regex CreateIntegerRegex() => new(IntegerRegexPattern, RegexOptions.Compiled, TimeSpan.FromMilliseconds(IntegerRegexTimeoutMs)); #endif + + /// + /// Compares two valid UTF-8 encoded JSON numbers for decimal equality. + /// + public static bool AreEqualJsonNumbers(ReadOnlySpan left, ReadOnlySpan right) + { + Debug.Assert(left.Length > 0 && right.Length > 0); + + ParseNumber(left, + out bool leftIsNegative, + out ReadOnlySpan leftIntegral, + out ReadOnlySpan leftFractional, + out int leftExponent); + + ParseNumber(right, + out bool rightIsNegative, + out ReadOnlySpan rightIntegral, + out ReadOnlySpan rightFractional, + out int rightExponent); + + int nDigits; + if (leftIsNegative != rightIsNegative || + leftExponent != rightExponent || + (nDigits = (leftIntegral.Length + leftFractional.Length)) != + rightIntegral.Length + rightFractional.Length) + { + return false; + } + + if (leftIntegral.Length == rightIntegral.Length) + { + return leftIntegral.SequenceEqual(rightIntegral) && + leftFractional.SequenceEqual(rightFractional); + } + + // There is differentiation in the integral and fractional lengths, + // concatenate both into singular buffers and compare them. + scoped Span leftDigits; + scoped Span rightDigits; + byte[]? rentedLeftBuffer; + byte[]? rentedRightBuffer; + + if (nDigits <= JsonConstants.StackallocByteThreshold) + { + leftDigits = stackalloc byte[JsonConstants.StackallocByteThreshold]; + rightDigits = stackalloc byte[JsonConstants.StackallocByteThreshold]; + rentedLeftBuffer = rentedRightBuffer = null; + } + else + { + leftDigits = (rentedLeftBuffer = ArrayPool.Shared.Rent(nDigits)); + rightDigits = (rentedRightBuffer = ArrayPool.Shared.Rent(nDigits)); + } + + leftIntegral.CopyTo(leftDigits); + leftFractional.CopyTo(leftDigits.Slice(leftIntegral.Length)); + rightIntegral.CopyTo(rightDigits); + rightFractional.CopyTo(rightDigits.Slice(rightIntegral.Length)); + + bool result = leftDigits.Slice(0, nDigits).SequenceEqual(rightDigits.Slice(0, nDigits)); + + if (rentedLeftBuffer != null) + { + Debug.Assert(rentedRightBuffer != null); + rentedLeftBuffer.AsSpan(0, nDigits).Clear(); + rentedRightBuffer.AsSpan(0, nDigits).Clear(); + ArrayPool.Shared.Return(rentedLeftBuffer); + ArrayPool.Shared.Return(rentedRightBuffer); + } + + return result; + + static void ParseNumber( + ReadOnlySpan span, + out bool isNegative, + out ReadOnlySpan integral, + out ReadOnlySpan fractional, + out int exponent) + { + // Parses a JSON number into its integral, fractional, and exponent parts. + // The returned components use a normal-form representation wherein two numbers + // are equal if and only if the sign and exponent are equal and additionally the + // concatenation of the integral and fractional parts are sequence equal. + // Under this scheme the number 0 is represented by a pair of empty spans. + + Debug.Assert(span.Length > 0); + + if (span[0] == '-') + { + isNegative = true; + span = span.Slice(1); + } + else + { + Debug.Assert(char.IsDigit((char)span[0]), "leading plus not allowed in valid JSON numbers."); + isNegative = false; + } + +#if NET + int i = span.IndexOfAny(s_decimalPointOrExponent); +#else + int i = span.IndexOfAny((byte)'.', (byte)'e', (byte)'E'); +#endif + if (i < 0) + { + integral = span; + fractional = default; + exponent = 0; + goto Normalize; + } + + integral = span.Slice(0, i); + + if (span[i] == '.') + { + span = span.Slice(i + 1); +#if NET + i = span.IndexOfAny(s_exponent); +#else + i = span.IndexOfAny((byte)'e', (byte)'E'); +#endif + + if (i < 0) + { + fractional = span; + exponent = 0; + goto Normalize; + } + + fractional = span.Slice(0, i); + } + else + { + fractional = default; + } + + Debug.Assert(span[i] is (byte)'e' or (byte)'E'); + bool success = Utf8Parser.TryParse(span.Slice(i + 1), out exponent, out _); + Debug.Assert(success); + + Normalize: + if (integral[0] == '0') + { + // Normalize "0" to the empty span. + Debug.Assert(integral.Length == 1, "Leading zeros not permitted in JSON numbers."); + integral = default; + } + + if (IndexOfFirstTrailingZero(fractional) is >= 0 and int iz) + { + // Trim all trailing zeros from the fractional part. + fractional = fractional.Slice(0, iz); + } + + if (fractional.IsEmpty && IndexOfFirstTrailingZero(integral) is >= 0 and int fz) + { + // There is no fractional part, trim all trailing zeros from + // the integral part and increase the exponent accordingly. + exponent += integral.Length - fz; + integral = integral.Slice(0, fz); + } + + // Normalize the exponent by subtracting the length of the fractional part. + exponent -= fractional.Length; + + if (integral.IsEmpty) + { + // Handle representations that only have a fractional component. + + if (IndexOfLastLeadingZero(fractional) is >= 0 and int lz) + { + // Trim all leading zeros from the fractional segment as + // they have already been accounted for in the exponent. + fractional = fractional.Slice(lz + 1); + } + + if (fractional.IsEmpty) + { + // Normalize zero representations. + isNegative = false; + exponent = 0; + } + } + + static int IndexOfLastLeadingZero(ReadOnlySpan span) + { + if (span.IsEmpty) + { + return -1; + } +#if NET + int firstNonZero = span.IndexOfAnyExcept((byte)'0'); + return firstNonZero == -1 ? span.Length - 1 : firstNonZero - 1; +#else + for (int i = 0; i < span.Length; i++) + { + if (span[i] != '0') + { + return i - 1; + } + } + + return span.Length - 1; +#endif + } + + static int IndexOfFirstTrailingZero(ReadOnlySpan span) + { + if (span.IsEmpty) + { + return -1; + } +#if NET + int lastNonZero = span.LastIndexOfAnyExcept((byte)'0'); + return lastNonZero == span.Length - 1 ? -1 : lastNonZero + 1; +#else + for (int i = span.Length - 1; i >= 0; i--) + { + if (span[i] != '0') + { + return i == span.Length - 1 ? -1 : i + 1; + } + } + + return 0; +#endif + } + } + } + +#if NET + private static readonly SearchValues s_decimalPointOrExponent = SearchValues.Create([(byte)'.', (byte)'e', (byte)'E']); + private static readonly SearchValues s_exponent = SearchValues.Create([(byte)'e', (byte)'E']); +#endif } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs index 12be1ef42f018..c21f07d6fa3c2 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs @@ -381,17 +381,88 @@ public static void DeepEqualsPrimitiveType() } [Theory] - [InlineData("1", "1.0", true)] - [InlineData("-0.0", "0", true)] - [InlineData("-1.1e3", "-1100", true)] - [InlineData("79228162514264337593543950336", "792281625142643375935439503360e-1", false)] // Not equal since it exceeds decimal.MaxValue - [InlineData("1.75e+300", "1.75E+300", false)] // Not equal due to case difference in exponent - public static void DeepEqualsNumericType(string leftStr, string rightStr, bool areEqual) + [InlineData("-0.0", "0")] + [InlineData("0", "0.0000e4")] + [InlineData("0", "0.0000e-4")] + [InlineData("1", "1.0")] + [InlineData("1", "1e0")] + [InlineData("1", "1.0000")] + [InlineData("1", "1.0000e0")] + [InlineData("1", "0.10000e1")] + [InlineData("1", "10.0000e-1")] + [InlineData("10001", "1.0001e4")] + [InlineData("10001e-3", "1.0001e1")] + [InlineData("1", "0.1e1")] + [InlineData("0.1", "1e-1")] + [InlineData("0.001", "1e-3")] + [InlineData("1e9", "1000000000")] + [InlineData("11", "1.100000000e1")] + [InlineData("3.141592653589793", "3141592653589793E-15")] + [InlineData("0.000000000000000000000000000000000000000001", "1e-42")] + [InlineData("1000000000000000000000000000000000000000000", "1e42")] + [InlineData("-1.1e3", "-1100")] + [InlineData("79228162514264337593543950336", "792281625142643375935439503360e-1")] // decimal.MaxValue + 1 + [InlineData("79228162514.264337593543950336", "792281625142643375935439503360e-19")] + [InlineData("1.75e+300", "1.75E+300")] // Variations in exponent casing + [InlineData( // > 256 digits + "1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001" , + + "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001" + "E-512")] + public static void DeepEqualsNumericType(string leftStr, string rightStr) { JsonNode left = JsonNode.Parse(leftStr); JsonNode right = JsonNode.Parse(rightStr); - Assert.Equal(areEqual, JsonNode.DeepEquals(left, right)); + JsonNodeTests.AssertDeepEqual(left, right); + } + + [Theory] + [InlineData("0", "1")] + [InlineData("1", "-1")] + [InlineData("1.1", "-1.1")] + [InlineData("1.1e5", "-1.1e5")] + [InlineData("0", "1e-1024")] + [InlineData("1", "1.1")] + [InlineData("1", "1e1")] + [InlineData("1", "1.00001")] + [InlineData("1", "1.0000e1")] + [InlineData("1", "0.1000e-1")] + [InlineData("1", "10.0000e-2")] + [InlineData("10001", "1.0001e3")] + [InlineData("10001e-3", "1.0001e2")] + [InlineData("1", "0.1e2")] + [InlineData("0.1", "1e-2")] + [InlineData("0.001", "1e-4")] + [InlineData("1e9", "1000000001")] + [InlineData("11", "1.100000001e1")] + [InlineData("0.000000000000000000000000000000000000000001", "1e-43")] + [InlineData("1000000000000000000000000000000000000000000", "1e43")] + [InlineData("-1.1e3", "-1100.1")] + [InlineData("79228162514264337593543950336", "7922816251426433759354395033600e-1")] // decimal.MaxValue + 1 + [InlineData("79228162514.264337593543950336", "7922816251426433759354395033601e-19")] + [InlineData("1.75e+300", "1.75E+301")] // Variations in exponent casing + [InlineData( // > 256 digits + "1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", + + "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003" + "E-512")] + public static void NotDeepEqualsNumericType(string leftStr, string rightStr) + { + JsonNode left = JsonNode.Parse(leftStr); + JsonNode right = JsonNode.Parse(rightStr); + + JsonNodeTests.AssertNotDeepEqual(left, right); } [Fact] From de5b311f4e8c02fc541770a385190e268332f198 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 2 Jul 2024 14:24:46 +0100 Subject: [PATCH 03/10] Address feedback --- .../src/System/Text/Json/JsonHelpers.cs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs index 3536d283247d4..695a340343c9d 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs @@ -359,11 +359,7 @@ static void ParseNumber( isNegative = false; } -#if NET - int i = span.IndexOfAny(s_decimalPointOrExponent); -#else int i = span.IndexOfAny((byte)'.', (byte)'e', (byte)'E'); -#endif if (i < 0) { integral = span; @@ -377,12 +373,7 @@ static void ParseNumber( if (span[i] == '.') { span = span.Slice(i + 1); -#if NET - i = span.IndexOfAny(s_exponent); -#else i = span.IndexOfAny((byte)'e', (byte)'E'); -#endif - if (i < 0) { fractional = span; @@ -490,10 +481,5 @@ static int IndexOfFirstTrailingZero(ReadOnlySpan span) } } } - -#if NET - private static readonly SearchValues s_decimalPointOrExponent = SearchValues.Create([(byte)'.', (byte)'e', (byte)'E']); - private static readonly SearchValues s_exponent = SearchValues.Create([(byte)'e', (byte)'E']); -#endif } } From 8d030baaf329f8d07d8ea660a78f5a8d72e82f06 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 2 Jul 2024 14:46:56 +0100 Subject: [PATCH 04/10] Add more comments. --- .../src/System/Text/Json/JsonHelpers.cs | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs index 695a340343c9d..1e1ac41eeac40 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs @@ -395,45 +395,46 @@ static void ParseNumber( Normalize: if (integral[0] == '0') { - // Normalize "0" to the empty span. Debug.Assert(integral.Length == 1, "Leading zeros not permitted in JSON numbers."); + + // Normalize "0" to the empty span. integral = default; + + if (IndexOfLastLeadingZero(fractional) is >= 0 and int lz) + { + // Trim leading zeros from the fractional part + // and update the exponent accordingly. + // e.g. 0.000123 -> 0.123e-3 + fractional = fractional.Slice(lz + 1); + exponent -= lz + 1; + } } if (IndexOfFirstTrailingZero(fractional) is >= 0 and int iz) { - // Trim all trailing zeros from the fractional part. + // Trim trailing zeros from the fractional part. + // e.g. 3.1400 -> 3.14 fractional = fractional.Slice(0, iz); } if (fractional.IsEmpty && IndexOfFirstTrailingZero(integral) is >= 0 and int fz) { - // There is no fractional part, trim all trailing zeros from + // There is no fractional part, trim trailing zeros from // the integral part and increase the exponent accordingly. + // e.g. 1000 -> 1e3 exponent += integral.Length - fz; integral = integral.Slice(0, fz); } // Normalize the exponent by subtracting the length of the fractional part. + // e.g. 3.14 -> 314e-2 exponent -= fractional.Length; - if (integral.IsEmpty) + if (integral.IsEmpty && fractional.IsEmpty) { - // Handle representations that only have a fractional component. - - if (IndexOfLastLeadingZero(fractional) is >= 0 and int lz) - { - // Trim all leading zeros from the fractional segment as - // they have already been accounted for in the exponent. - fractional = fractional.Slice(lz + 1); - } - - if (fractional.IsEmpty) - { - // Normalize zero representations. - isNegative = false; - exponent = 0; - } + // Normalize zero representations. + isNegative = false; + exponent = 0; } static int IndexOfLastLeadingZero(ReadOnlySpan span) From 732882a86b1aece29b096c2fc5b1c09e6f5847ed Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 2 Jul 2024 14:59:11 +0100 Subject: [PATCH 05/10] Update src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs Co-authored-by: Stephen Toub --- .../System.Text.Json/src/System/Text/Json/JsonHelpers.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs index 1e1ac41eeac40..6c39b84de6ed8 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs @@ -445,7 +445,7 @@ static int IndexOfLastLeadingZero(ReadOnlySpan span) } #if NET int firstNonZero = span.IndexOfAnyExcept((byte)'0'); - return firstNonZero == -1 ? span.Length - 1 : firstNonZero - 1; + return firstNonZero < 0 ? span.Length - 1 : firstNonZero - 1; #else for (int i = 0; i < span.Length; i++) { From b2d9b08eb2d5364d0a3ed4f9eaef5bae17b32fd4 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 2 Jul 2024 15:18:09 +0100 Subject: [PATCH 06/10] Address feedback --- .../src/System/Text/Json/JsonHelpers.cs | 76 ++++++++++--------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs index 6c39b84de6ed8..40b827c1b7b87 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs @@ -346,29 +346,34 @@ static void ParseNumber( // concatenation of the integral and fractional parts are sequence equal. // Under this scheme the number 0 is represented by a pair of empty spans. + bool neg; + ReadOnlySpan intg; + ReadOnlySpan frac; + int exp; + Debug.Assert(span.Length > 0); if (span[0] == '-') { - isNegative = true; + neg = true; span = span.Slice(1); } else { Debug.Assert(char.IsDigit((char)span[0]), "leading plus not allowed in valid JSON numbers."); - isNegative = false; + neg = false; } int i = span.IndexOfAny((byte)'.', (byte)'e', (byte)'E'); if (i < 0) { - integral = span; - fractional = default; - exponent = 0; + intg = span; + frac = default; + exp = 0; goto Normalize; } - integral = span.Slice(0, i); + intg = span.Slice(0, i); if (span[i] == '.') { @@ -376,73 +381,75 @@ static void ParseNumber( i = span.IndexOfAny((byte)'e', (byte)'E'); if (i < 0) { - fractional = span; - exponent = 0; + frac = span; + exp = 0; goto Normalize; } - fractional = span.Slice(0, i); + frac = span.Slice(0, i); } else { - fractional = default; + frac = default; } Debug.Assert(span[i] is (byte)'e' or (byte)'E'); - bool success = Utf8Parser.TryParse(span.Slice(i + 1), out exponent, out _); + bool success = Utf8Parser.TryParse(span.Slice(i + 1), out exp, out _); Debug.Assert(success); Normalize: - if (integral[0] == '0') + if (intg[0] == '0') { - Debug.Assert(integral.Length == 1, "Leading zeros not permitted in JSON numbers."); + Debug.Assert(intg.Length == 1, "Leading zeros not permitted in JSON numbers."); // Normalize "0" to the empty span. - integral = default; + intg = default; - if (IndexOfLastLeadingZero(fractional) is >= 0 and int lz) + if (IndexOfLastLeadingZero(frac) is >= 0 and int lz) { // Trim leading zeros from the fractional part // and update the exponent accordingly. // e.g. 0.000123 -> 0.123e-3 - fractional = fractional.Slice(lz + 1); - exponent -= lz + 1; + frac = frac.Slice(lz + 1); + exp -= lz + 1; } } - if (IndexOfFirstTrailingZero(fractional) is >= 0 and int iz) + if (IndexOfFirstTrailingZero(frac) is >= 0 and int iz) { // Trim trailing zeros from the fractional part. // e.g. 3.1400 -> 3.14 - fractional = fractional.Slice(0, iz); + frac = frac.Slice(0, iz); } - if (fractional.IsEmpty && IndexOfFirstTrailingZero(integral) is >= 0 and int fz) + if (frac.IsEmpty && IndexOfFirstTrailingZero(intg) is >= 0 and int fz) { // There is no fractional part, trim trailing zeros from // the integral part and increase the exponent accordingly. // e.g. 1000 -> 1e3 - exponent += integral.Length - fz; - integral = integral.Slice(0, fz); + exp += intg.Length - fz; + intg = intg.Slice(0, fz); } // Normalize the exponent by subtracting the length of the fractional part. // e.g. 3.14 -> 314e-2 - exponent -= fractional.Length; + exp -= frac.Length; - if (integral.IsEmpty && fractional.IsEmpty) + if (intg.IsEmpty && frac.IsEmpty) { // Normalize zero representations. - isNegative = false; - exponent = 0; + neg = false; + exp = 0; } + // Copy to out parameters. + isNegative = neg; + integral = intg; + fractional = frac; + exponent = exp; + static int IndexOfLastLeadingZero(ReadOnlySpan span) { - if (span.IsEmpty) - { - return -1; - } #if NET int firstNonZero = span.IndexOfAnyExcept((byte)'0'); return firstNonZero < 0 ? span.Length - 1 : firstNonZero - 1; @@ -461,14 +468,15 @@ static int IndexOfLastLeadingZero(ReadOnlySpan span) static int IndexOfFirstTrailingZero(ReadOnlySpan span) { - if (span.IsEmpty) - { - return -1; - } #if NET int lastNonZero = span.LastIndexOfAnyExcept((byte)'0'); return lastNonZero == span.Length - 1 ? -1 : lastNonZero + 1; #else + if (span.IsEmpty) + { + return -1; + } + for (int i = span.Length - 1; i >= 0; i--) { if (span[i] != '0') From e3efed1f9a987fb98cd9c02101138e52e1f370ad Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 2 Jul 2024 15:44:24 +0100 Subject: [PATCH 07/10] Improve comments --- .../src/System/Text/Json/JsonHelpers.cs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs index 40b827c1b7b87..ea358f4510924 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs @@ -341,10 +341,14 @@ static void ParseNumber( out int exponent) { // Parses a JSON number into its integral, fractional, and exponent parts. - // The returned components use a normal-form representation wherein two numbers - // are equal if and only if the sign and exponent are equal and additionally the - // concatenation of the integral and fractional parts are sequence equal. - // Under this scheme the number 0 is represented by a pair of empty spans. + // The returned components use a normal-form decimal representation: + // + // Number := sign * * 10^exponent + // + // where integral and fractional are sequences of digits whose concatenation + // represents the signifand of the number without leading or trailing zeros. + // Two such normal-form numbers are treated as equal if and only if they have + // equal signs, significands, and exponents. bool neg; ReadOnlySpan intg; @@ -402,9 +406,6 @@ static void ParseNumber( { Debug.Assert(intg.Length == 1, "Leading zeros not permitted in JSON numbers."); - // Normalize "0" to the empty span. - intg = default; - if (IndexOfLastLeadingZero(frac) is >= 0 and int lz) { // Trim leading zeros from the fractional part @@ -413,6 +414,9 @@ static void ParseNumber( frac = frac.Slice(lz + 1); exp -= lz + 1; } + + // Normalize "0" to the empty span. + intg = default; } if (IndexOfFirstTrailingZero(frac) is >= 0 and int iz) From a6bc0f45b28f881f53b76d727d2497a68c7aa7ba Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 2 Jul 2024 19:53:41 +0100 Subject: [PATCH 08/10] Update src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs --- .../System.Text.Json/src/System/Text/Json/JsonHelpers.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs index ea358f4510924..de7d4e7bc614d 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs @@ -346,7 +346,7 @@ static void ParseNumber( // Number := sign * * 10^exponent // // where integral and fractional are sequences of digits whose concatenation - // represents the signifand of the number without leading or trailing zeros. + // represents the significand of the number without leading or trailing zeros. // Two such normal-form numbers are treated as equal if and only if they have // equal signs, significands, and exponents. From 65f338398742c8dbb6962d70ba38ba2c3f69380c Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 2 Jul 2024 20:06:25 +0100 Subject: [PATCH 09/10] Trim frac trailing zeros before trimming leading zeros. --- .../src/System/Text/Json/JsonHelpers.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs index de7d4e7bc614d..e01844be8a2e5 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs @@ -401,7 +401,15 @@ static void ParseNumber( bool success = Utf8Parser.TryParse(span.Slice(i + 1), out exp, out _); Debug.Assert(success); - Normalize: + Normalize: // Calculates the normal form of the number. + + if (IndexOfFirstTrailingZero(frac) is >= 0 and int iz) + { + // Trim trailing zeros from the fractional part. + // e.g. 3.1400 -> 3.14 + frac = frac.Slice(0, iz); + } + if (intg[0] == '0') { Debug.Assert(intg.Length == 1, "Leading zeros not permitted in JSON numbers."); @@ -419,13 +427,6 @@ static void ParseNumber( intg = default; } - if (IndexOfFirstTrailingZero(frac) is >= 0 and int iz) - { - // Trim trailing zeros from the fractional part. - // e.g. 3.1400 -> 3.14 - frac = frac.Slice(0, iz); - } - if (frac.IsEmpty && IndexOfFirstTrailingZero(intg) is >= 0 and int fz) { // There is no fractional part, trim trailing zeros from From 007dd064ad50e23115d62c6cd503a9137d05ccfa Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Thu, 4 Jul 2024 22:33:54 +0100 Subject: [PATCH 10/10] Add handling for exponent values > Int32 --- .../System.Text.Json/src/Resources/Strings.resx | 3 +++ .../src/System/Text/Json/JsonHelpers.cs | 7 +++++-- .../src/System/Text/Json/ThrowHelper.cs | 6 ++++++ .../JsonNode/JsonValueTests.cs | 13 +++++++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Text.Json/src/Resources/Strings.resx b/src/libraries/System.Text.Json/src/Resources/Strings.resx index b8791ea8fb171..dc9458c6d705b 100644 --- a/src/libraries/System.Text.Json/src/Resources/Strings.resx +++ b/src/libraries/System.Text.Json/src/Resources/Strings.resx @@ -246,6 +246,9 @@ The requested operation requires an element of type '{0}', but the target element has type '{1}'. + + The exponent value in the specified JSON number is too large. + Cannot add callbacks to the 'Modifiers' property after the resolver has been used for the first time. diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs index e01844be8a2e5..3933b9ee623dc 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs @@ -398,8 +398,11 @@ static void ParseNumber( } Debug.Assert(span[i] is (byte)'e' or (byte)'E'); - bool success = Utf8Parser.TryParse(span.Slice(i + 1), out exp, out _); - Debug.Assert(success); + if (!Utf8Parser.TryParse(span.Slice(i + 1), out exp, out _)) + { + Debug.Assert(span.Length >= 10); + ThrowHelper.ThrowArgumentOutOfRangeException_JsonNumberExponentTooLarge(nameof(exponent)); + } Normalize: // Calculates the normal form of the number. diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs index 6976d42b967bc..fb1786be551da 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs @@ -36,6 +36,12 @@ public static void ThrowArgumentOutOfRangeException_MaxDepthMustBePositive(strin throw GetArgumentOutOfRangeException(parameterName, SR.MaxDepthMustBePositive); } + [DoesNotReturn] + public static void ThrowArgumentOutOfRangeException_JsonNumberExponentTooLarge(string parameterName) + { + throw GetArgumentOutOfRangeException(parameterName, SR.JsonNumberExponentTooLarge); + } + private static ArgumentOutOfRangeException GetArgumentOutOfRangeException(string parameterName, string message) { return new ArgumentOutOfRangeException(parameterName, message); diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs index c21f07d6fa3c2..29ae9662eedd4 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs @@ -428,6 +428,7 @@ public static void DeepEqualsNumericType(string leftStr, string rightStr) [InlineData("1.1", "-1.1")] [InlineData("1.1e5", "-1.1e5")] [InlineData("0", "1e-1024")] + [InlineData("1", "0.1")] [InlineData("1", "1.1")] [InlineData("1", "1e1")] [InlineData("1", "1.00001")] @@ -447,6 +448,7 @@ public static void DeepEqualsNumericType(string leftStr, string rightStr) [InlineData("79228162514264337593543950336", "7922816251426433759354395033600e-1")] // decimal.MaxValue + 1 [InlineData("79228162514.264337593543950336", "7922816251426433759354395033601e-19")] [InlineData("1.75e+300", "1.75E+301")] // Variations in exponent casing + [InlineData("1e2147483647", "1e-2147483648")] // int.MaxValue, int.MinValue exponents [InlineData( // > 256 digits "1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + @@ -465,6 +467,17 @@ public static void NotDeepEqualsNumericType(string leftStr, string rightStr) JsonNodeTests.AssertNotDeepEqual(left, right); } + [Theory] + [InlineData(int.MinValue - 1L)] + [InlineData(int.MaxValue + 1L)] + [InlineData(long.MinValue)] + [InlineData(long.MaxValue)] + public static void DeepEquals_ExponentExceedsInt32_ThrowsArgumentOutOfRangeException(long exponent) + { + JsonNode node = JsonNode.Parse($"1e{exponent}"); + Assert.Throws(() => JsonNode.DeepEquals(node, node)); + } + [Fact] public static void DeepEqualsJsonElement() {