Skip to content

Commit

Permalink
Improve JsonNode.DeepEquals numeric equality. (#104255)
Browse files Browse the repository at this point in the history 
* Attempt at improving JsonNode.DeepEquals numeric equality.

* Implement arbitrary-precision decimal equality comparison.

* Address feedback

* Add more comments.

* Update src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs

Co-authored-by: Stephen Toub <stoub@microsoft.com>

* Address feedback

* Improve comments

* Update src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs

* Trim frac trailing zeros before trimming leading zeros.

* Add handling for exponent values > Int32

---------

Co-authored-by: Stephen Toub <stoub@microsoft.com>
  • Loading branch information
@eiriktsarpalis @stephentoub
eiriktsarpalis and stephentoub authored Jul 8, 2024
1 parent 670d11f commit 90d4c7d
Show file tree
Hide file tree
Showing 5 changed files with 347 additions and 2 deletions.
3 changes: 3 additions & 0 deletions src/libraries/System.Text.Json/src/Resources/Strings.resx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,9 @@
<data name="JsonElementHasWrongType" xml:space="preserve">
<value>The requested operation requires an element of type '{0}', but the target element has type '{1}'.</value>
</data>
<data name="JsonNumberExponentTooLarge" xml:space="preserve">
<value>The exponent value in the specified JSON number is too large.</value>
</data>
<data name="DefaultTypeInfoResolverImmutable" xml:space="preserve">
<value>Cannot add callbacks to the 'Modifiers' property after the resolver has been used for the first time.</value>
</data>
Expand Down
4 changes: 2 additions & 2 deletions src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Buffers.Text;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
Expand Down Expand Up @@ -1241,8 +1242,7 @@ internal static bool DeepEquals(JsonElement left, JsonElement right)
return true;

case JsonValueKind.Number:
// JSON numbers are equal if their raw representations are equal.
return left.GetRawValue().Span.SequenceEqual(right.GetRawValue().Span);
return JsonHelpers.AreEqualJsonNumbers(left.GetRawValue().Span, right.GetRawValue().Span);

case JsonValueKind.String:
if (right.ValueIsEscaped)
Expand Down
238 changes: 238 additions & 0 deletions src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.

using System.Buffers;
using System.Buffers.Text;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
Expand Down Expand Up @@ -289,5 +290,242 @@ public static bool HasAllSet(this BitArray bitArray)
#else
private static Regex CreateIntegerRegex() => new(IntegerRegexPattern, RegexOptions.Compiled, TimeSpan.FromMilliseconds(IntegerRegexTimeoutMs));
#endif

/// <summary>
/// Compares two valid UTF-8 encoded JSON numbers for decimal equality.
/// </summary>
public static bool AreEqualJsonNumbers(ReadOnlySpan<byte> left, ReadOnlySpan<byte> right)
{
Debug.Assert(left.Length > 0 && right.Length > 0);

ParseNumber(left,
out bool leftIsNegative,
out ReadOnlySpan<byte> leftIntegral,
out ReadOnlySpan<byte> leftFractional,
out int leftExponent);

ParseNumber(right,
out bool rightIsNegative,
out ReadOnlySpan<byte> rightIntegral,
out ReadOnlySpan<byte> rightFractional,
out int rightExponent);

int nDigits;
if (leftIsNegative != rightIsNegative ||
leftExponent != rightExponent ||
(nDigits = (leftIntegral.Length + leftFractional.Length)) !=
rightIntegral.Length + rightFractional.Length)
{
return false;
}

if (leftIntegral.Length == rightIntegral.Length)
{
return leftIntegral.SequenceEqual(rightIntegral) &&
leftFractional.SequenceEqual(rightFractional);
}

// There is differentiation in the integral and fractional lengths,
// concatenate both into singular buffers and compare them.
scoped Span<byte> leftDigits;
scoped Span<byte> rightDigits;
byte[]? rentedLeftBuffer;
byte[]? rentedRightBuffer;

if (nDigits <= JsonConstants.StackallocByteThreshold)
{
leftDigits = stackalloc byte[JsonConstants.StackallocByteThreshold];
rightDigits = stackalloc byte[JsonConstants.StackallocByteThreshold];
rentedLeftBuffer = rentedRightBuffer = null;
}
else
{
leftDigits = (rentedLeftBuffer = ArrayPool<byte>.Shared.Rent(nDigits));
rightDigits = (rentedRightBuffer = ArrayPool<byte>.Shared.Rent(nDigits));
}

leftIntegral.CopyTo(leftDigits);
leftFractional.CopyTo(leftDigits.Slice(leftIntegral.Length));
rightIntegral.CopyTo(rightDigits);
rightFractional.CopyTo(rightDigits.Slice(rightIntegral.Length));

bool result = leftDigits.Slice(0, nDigits).SequenceEqual(rightDigits.Slice(0, nDigits));

if (rentedLeftBuffer != null)
{
Debug.Assert(rentedRightBuffer != null);
rentedLeftBuffer.AsSpan(0, nDigits).Clear();
rentedRightBuffer.AsSpan(0, nDigits).Clear();
ArrayPool<byte>.Shared.Return(rentedLeftBuffer);
ArrayPool<byte>.Shared.Return(rentedRightBuffer);
}

return result;

static void ParseNumber(
ReadOnlySpan<byte> span,
out bool isNegative,
out ReadOnlySpan<byte> integral,
out ReadOnlySpan<byte> fractional,
out int exponent)
{
// Parses a JSON number into its integral, fractional, and exponent parts.
// The returned components use a normal-form decimal representation:
//
// Number := sign * <integral + fractional> * 10^exponent
//
// where integral and fractional are sequences of digits whose concatenation
// represents the significand of the number without leading or trailing zeros.
// Two such normal-form numbers are treated as equal if and only if they have
// equal signs, significands, and exponents.

bool neg;
ReadOnlySpan<byte> intg;
ReadOnlySpan<byte> frac;
int exp;

Debug.Assert(span.Length > 0);

if (span[0] == '-')
{
neg = true;
span = span.Slice(1);
}
else
{
Debug.Assert(char.IsDigit((char)span[0]), "leading plus not allowed in valid JSON numbers.");
neg = false;
}

int i = span.IndexOfAny((byte)'.', (byte)'e', (byte)'E');
if (i < 0)
{
intg = span;
frac = default;
exp = 0;
goto Normalize;
}

intg = span.Slice(0, i);

if (span[i] == '.')
{
span = span.Slice(i + 1);
i = span.IndexOfAny((byte)'e', (byte)'E');
if (i < 0)
{
frac = span;
exp = 0;
goto Normalize;
}

frac = span.Slice(0, i);
}
else
{
frac = default;
}

Debug.Assert(span[i] is (byte)'e' or (byte)'E');
if (!Utf8Parser.TryParse(span.Slice(i + 1), out exp, out _))
{
Debug.Assert(span.Length >= 10);
ThrowHelper.ThrowArgumentOutOfRangeException_JsonNumberExponentTooLarge(nameof(exponent));
}

Normalize: // Calculates the normal form of the number.

if (IndexOfFirstTrailingZero(frac) is >= 0 and int iz)
{
// Trim trailing zeros from the fractional part.
// e.g. 3.1400 -> 3.14
frac = frac.Slice(0, iz);
}

if (intg[0] == '0')
{
Debug.Assert(intg.Length == 1, "Leading zeros not permitted in JSON numbers.");

if (IndexOfLastLeadingZero(frac) is >= 0 and int lz)
{
// Trim leading zeros from the fractional part
// and update the exponent accordingly.
// e.g. 0.000123 -> 0.123e-3
frac = frac.Slice(lz + 1);
exp -= lz + 1;
}

// Normalize "0" to the empty span.
intg = default;
}

if (frac.IsEmpty && IndexOfFirstTrailingZero(intg) is >= 0 and int fz)
{
// There is no fractional part, trim trailing zeros from
// the integral part and increase the exponent accordingly.
// e.g. 1000 -> 1e3
exp += intg.Length - fz;
intg = intg.Slice(0, fz);
}

// Normalize the exponent by subtracting the length of the fractional part.
// e.g. 3.14 -> 314e-2
exp -= frac.Length;

if (intg.IsEmpty && frac.IsEmpty)
{
// Normalize zero representations.
neg = false;
exp = 0;
}

// Copy to out parameters.
isNegative = neg;
integral = intg;
fractional = frac;
exponent = exp;

static int IndexOfLastLeadingZero(ReadOnlySpan<byte> span)
{
#if NET
int firstNonZero = span.IndexOfAnyExcept((byte)'0');
return firstNonZero < 0 ? span.Length - 1 : firstNonZero - 1;
#else
for (int i = 0; i < span.Length; i++)
{
if (span[i] != '0')
{
return i - 1;
}
}

return span.Length - 1;
#endif
}

static int IndexOfFirstTrailingZero(ReadOnlySpan<byte> span)
{
#if NET
int lastNonZero = span.LastIndexOfAnyExcept((byte)'0');
return lastNonZero == span.Length - 1 ? -1 : lastNonZero + 1;
#else
if (span.IsEmpty)
{
return -1;
}

for (int i = span.Length - 1; i >= 0; i--)
{
if (span[i] != '0')
{
return i == span.Length - 1 ? -1 : i + 1;
}
}

return 0;
#endif
}
}
}
}
}
6 changes: 6 additions & 0 deletions src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ public static void ThrowArgumentOutOfRangeException_MaxDepthMustBePositive(strin
throw GetArgumentOutOfRangeException(parameterName, SR.MaxDepthMustBePositive);
}

[DoesNotReturn]
public static void ThrowArgumentOutOfRangeException_JsonNumberExponentTooLarge(string parameterName)
{
throw GetArgumentOutOfRangeException(parameterName, SR.JsonNumberExponentTooLarge);
}

private static ArgumentOutOfRangeException GetArgumentOutOfRangeException(string parameterName, string message)
{
return new ArgumentOutOfRangeException(parameterName, message);
Expand Down
98 changes: 98 additions & 0 deletions src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonNode/JsonValueTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,104 @@ public static void DeepEqualsPrimitiveType()
JsonNodeTests.AssertNotDeepEqual(JsonValue.Create(10), JsonValue.Create("10"));
}

[Theory]
[InlineData("-0.0", "0")]
[InlineData("0", "0.0000e4")]
[InlineData("0", "0.0000e-4")]
[InlineData("1", "1.0")]
[InlineData("1", "1e0")]
[InlineData("1", "1.0000")]
[InlineData("1", "1.0000e0")]
[InlineData("1", "0.10000e1")]
[InlineData("1", "10.0000e-1")]
[InlineData("10001", "1.0001e4")]
[InlineData("10001e-3", "1.0001e1")]
[InlineData("1", "0.1e1")]
[InlineData("0.1", "1e-1")]
[InlineData("0.001", "1e-3")]
[InlineData("1e9", "1000000000")]
[InlineData("11", "1.100000000e1")]
[InlineData("3.141592653589793", "3141592653589793E-15")]
[InlineData("0.000000000000000000000000000000000000000001", "1e-42")]
[InlineData("1000000000000000000000000000000000000000000", "1e42")]
[InlineData("-1.1e3", "-1100")]
[InlineData("79228162514264337593543950336", "792281625142643375935439503360e-1")] // decimal.MaxValue + 1
[InlineData("79228162514.264337593543950336", "792281625142643375935439503360e-19")]
[InlineData("1.75e+300", "1.75E+300")] // Variations in exponent casing
[InlineData( // > 256 digits
"1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001" ,

"100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001" + "E-512")]
public static void DeepEqualsNumericType(string leftStr, string rightStr)
{
JsonNode left = JsonNode.Parse(leftStr);
JsonNode right = JsonNode.Parse(rightStr);

JsonNodeTests.AssertDeepEqual(left, right);
}

[Theory]
[InlineData("0", "1")]
[InlineData("1", "-1")]
[InlineData("1.1", "-1.1")]
[InlineData("1.1e5", "-1.1e5")]
[InlineData("0", "1e-1024")]
[InlineData("1", "0.1")]
[InlineData("1", "1.1")]
[InlineData("1", "1e1")]
[InlineData("1", "1.00001")]
[InlineData("1", "1.0000e1")]
[InlineData("1", "0.1000e-1")]
[InlineData("1", "10.0000e-2")]
[InlineData("10001", "1.0001e3")]
[InlineData("10001e-3", "1.0001e2")]
[InlineData("1", "0.1e2")]
[InlineData("0.1", "1e-2")]
[InlineData("0.001", "1e-4")]
[InlineData("1e9", "1000000001")]
[InlineData("11", "1.100000001e1")]
[InlineData("0.000000000000000000000000000000000000000001", "1e-43")]
[InlineData("1000000000000000000000000000000000000000000", "1e43")]
[InlineData("-1.1e3", "-1100.1")]
[InlineData("79228162514264337593543950336", "7922816251426433759354395033600e-1")] // decimal.MaxValue + 1
[InlineData("79228162514.264337593543950336", "7922816251426433759354395033601e-19")]
[InlineData("1.75e+300", "1.75E+301")] // Variations in exponent casing
[InlineData("1e2147483647", "1e-2147483648")] // int.MaxValue, int.MinValue exponents
[InlineData( // > 256 digits
"1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",

"100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +
"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003" + "E-512")]
public static void NotDeepEqualsNumericType(string leftStr, string rightStr)
{
JsonNode left = JsonNode.Parse(leftStr);
JsonNode right = JsonNode.Parse(rightStr);

JsonNodeTests.AssertNotDeepEqual(left, right);
}

[Theory]
[InlineData(int.MinValue - 1L)]
[InlineData(int.MaxValue + 1L)]
[InlineData(long.MinValue)]
[InlineData(long.MaxValue)]
public static void DeepEquals_ExponentExceedsInt32_ThrowsArgumentOutOfRangeException(long exponent)
{
JsonNode node = JsonNode.Parse($"1e{exponent}");
Assert.Throws<ArgumentOutOfRangeException>(() => JsonNode.DeepEquals(node, node));
}

[Fact]
public static void DeepEqualsJsonElement()
{
Expand Down

0 comments on commit 90d4c7d

Please sign in to comment.