From 2b9a5ec03562dfd2b91d0d1b53e5d8cb867627fb Mon Sep 17 00:00:00 2001 From: Layomi Akinrinade Date: Tue, 8 Jun 2021 19:42:22 -0700 Subject: [PATCH 1/6] Add write-raw APIs to Utf8JsonWriter --- .../System.Text.Json/ref/System.Text.Json.cs | 3 + .../src/System.Text.Json.csproj | 1 + .../src/System/Text/Json/JsonConstants.cs | 3 + .../JsonSerializer.Read.String.cs | 4 +- .../Writer/Utf8JsonWriter.WriteValues.Raw.cs | 107 ++++++++ .../System.Text.Json.Tests.csproj | 1 + .../Utf8JsonWriterTests.WriteRaw.cs | 259 ++++++++++++++++++ .../Utf8JsonWriterTests.cs | 13 +- 8 files changed, 383 insertions(+), 8 deletions(-) create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs create mode 100644 src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs index ed8880c117fea..fa415f8df784f 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs @@ -461,6 +461,9 @@ public void WritePropertyName(System.ReadOnlySpan utf8PropertyName) { } public void WritePropertyName(System.ReadOnlySpan propertyName) { } public void WritePropertyName(string propertyName) { } public void WritePropertyName(System.Text.Json.JsonEncodedText propertyName) { } + public void WriteRawValue(string json, bool skipInputValidation = false) { } + public void WriteRawValue(System.ReadOnlySpan utf8Json, bool skipInputValidation = false) { } + public void WriteRawValue(System.ReadOnlySpan json, bool skipInputValidation = false) { } public void WriteStartArray() { } public void WriteStartArray(System.ReadOnlySpan utf8PropertyName) { } public void WriteStartArray(System.ReadOnlySpan propertyName) { } diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 8a6315b1fe7c4..8cb1375c6d08e 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -257,6 +257,7 @@ + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs index e82811377a682..bb27d8b1a563b 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs @@ -64,6 +64,9 @@ internal static class JsonConstants // All other UTF-16 characters can be represented by either 1 or 2 UTF-8 bytes. public const int MaxExpansionFactorWhileTranscoding = 3; + // When transcoding from UTF8 -> UTF16, the byte count threshold where we rent from the array pool before performing a normal alloc. + public const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024; + public const int MaxEscapedTokenSize = 1_000_000_000; // Max size for already escaped value. public const int MaxUnescapedTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping; // 166_666_666 bytes public const int MaxBase64ValueTokenSize = (MaxEscapedTokenSize >> 2) * 3 / MaxExpansionFactorWhileEscaping; // 125_000_000 bytes diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.String.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.String.cs index 1cd7f8cc6eb11..29e06ef3742f1 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.String.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.String.cs @@ -355,12 +355,10 @@ public static partial class JsonSerializer private static TValue? ReadUsingMetadata(ReadOnlySpan json, JsonTypeInfo jsonTypeInfo) { - const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024; - byte[]? tempArray = null; // For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold. - Span utf8 = json.Length <= (ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ? + Span utf8 = json.Length <= (JsonConstants.ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ? // Use a pooled alloc. tempArray = ArrayPool.Shared.Rent(json.Length * JsonConstants.MaxExpansionFactorWhileTranscoding) : // Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs new file mode 100644 index 0000000000000..3ea7e1f26c8aa --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs @@ -0,0 +1,107 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; + +namespace System.Text.Json +{ + public sealed partial class Utf8JsonWriter + { + /// + /// Writes the input as JSON content. + /// + /// The raw JSON content to write. + /// Whether to skip validation of the input JSON content. + public void WriteRawValue(string json, bool skipInputValidation = false) + { + if (json == null) + { + throw new ArgumentNullException(nameof(json)); + } + + WriteRawValue(json.AsSpan(), skipInputValidation); + } + + /// + /// Writes the input as JSON content. + /// + /// The raw JSON content to write. + /// Whether to skip validation of the input JSON content. + public void WriteRawValue(ReadOnlySpan json, bool skipInputValidation = false) + { + byte[]? tempArray = null; + + // For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold. + Span utf8Json = json.Length <= (JsonConstants.ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ? + // Use a pooled alloc. + tempArray = ArrayPool.Shared.Rent(json.Length * JsonConstants.MaxExpansionFactorWhileTranscoding) : + // Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation) + // and by using a normal alloc we can avoid the Clear(). + new byte[JsonReaderHelper.GetUtf8ByteCount(json)]; + + try + { + int actualByteCount = JsonReaderHelper.GetUtf8FromText(json, utf8Json); + utf8Json = utf8Json.Slice(0, actualByteCount); + WriteRawValue(utf8Json, skipInputValidation); + } + finally + { + if (tempArray != null) + { + utf8Json.Clear(); + ArrayPool.Shared.Return(tempArray); + } + } + } + + /// + /// Writes the input as JSON content. + /// + /// The raw JSON content to write. + /// Whether to skip validation of the input JSON content. + public void WriteRawValue(ReadOnlySpan utf8Json, bool skipInputValidation = false) + { + if (utf8Json.Length == 0) + { + ThrowHelper.ThrowArgumentException(SR.ExpectedJsonTokens); + } + + if (!skipInputValidation) + { + Utf8JsonReader reader = new Utf8JsonReader(utf8Json); + + try + { + while (reader.Read()); + } + catch (JsonReaderException ex) + { + ThrowHelper.ThrowArgumentException(ex.Message); + } + } + + int maxRequired = utf8Json.Length + 1; // Optionally, 1 list separator + + if (_memory.Length - BytesPending < maxRequired) + { + Grow(maxRequired); + } + + Span output = _memory.Span; + + if (_currentDepth < 0) + { + output[BytesPending++] = JsonConstants.ListSeparator; + } + + utf8Json.CopyTo(output.Slice(BytesPending)); + BytesPending += utf8Json.Length; + + SetFlagToAddListSeparatorBeforeNextItem(); + + // Treat all raw JSON value writes as string. + _tokenType = JsonTokenType.String; + } + } +} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj index ee45bdc5e97e9..d3ba50335a1d6 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj @@ -189,6 +189,7 @@ + diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs new file mode 100644 index 0000000000000..219d49393416f --- /dev/null +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs @@ -0,0 +1,259 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Xunit; + +namespace System.Text.Json.Tests +{ + public partial class Utf8JsonWriterTests + { + private const string TestGuidAsStr = "eb97fadd-3ebf-4781-8722-f4773989160e"; + private readonly static Guid s_guid = Guid.Parse(TestGuidAsStr); + private static byte[] s_guidAsJson = WrapInQuotes(TestGuidAsStr); + + private static byte[] s_oneAsJson = new byte[] { (byte)'1' }; + + [Theory] + [MemberData(nameof(GetRootLevelPrimitives))] + [MemberData(nameof(GetArrays))] + public static void WriteRawValidJson(byte[] rawJson, Action verifyWithDeserialize) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + + RunTests(skipInputValidation: true); + RunTests(skipInputValidation: false); + + void RunTests(bool skipInputValidation) + { + // ROS + writer.Reset(); + ms.SetLength(0); + writer.WriteRawValue(rawJson, skipInputValidation); + writer.Flush(); + verifyWithDeserialize(ms.ToArray()); + + // string + string rawJsonAsStr = Encoding.UTF8.GetString(rawJson); + writer.Reset(); + ms.SetLength(0); + writer.WriteRawValue(rawJsonAsStr, skipInputValidation); + writer.Flush(); + verifyWithDeserialize(ms.ToArray()); + + // ROS + writer.Reset(); + ms.SetLength(0); + writer.WriteRawValue(rawJsonAsStr.AsSpan(), skipInputValidation); + writer.Flush(); + verifyWithDeserialize(ms.ToArray()); + } + } + + public static IEnumerable GetRootLevelPrimitives() + { + Action validate; + + validate = (data) => Assert.Equal(123456789, JsonSerializer.Deserialize(data)); + yield return new object[] { Encoding.UTF8.GetBytes("123456789"), validate }; + + validate = (data) => Assert.Equal(1234.56789, JsonSerializer.Deserialize(data)); + yield return new object[] { Encoding.UTF8.GetBytes("1234.56789"), validate }; + + validate = (data) => Assert.Equal(@"Hello", JsonSerializer.Deserialize(data)); + yield return new object[] { Encoding.UTF8.GetBytes(@"""Hello"""), validate }; + + validate = (data) => Assert.Equal(s_guid, JsonSerializer.Deserialize(data)); + yield return new object[] { s_guidAsJson, validate }; + } + + public static IEnumerable GetArrays() + { + Action validate; + + byte[] json = JsonSerializer.SerializeToUtf8Bytes(Enumerable.Repeat(1234.56789, 4)); + validate = (data) => + { + foreach (double d in JsonSerializer.Deserialize(data)) + { + Assert.Equal(1234.56789, d); + } + }; + yield return new object[] { json, validate }; + + json = JsonSerializer.SerializeToUtf8Bytes(Enumerable.Repeat("Hello", 4)); + validate = (data) => + { + foreach (string str in JsonSerializer.Deserialize(data)) + { + Assert.Equal("Hello", str); + } + }; + yield return new object[] { json, validate }; + + json = JsonSerializer.SerializeToUtf8Bytes(Enumerable.Repeat("Hello", 4)); + validate = (data) => + { + foreach (string str in JsonSerializer.Deserialize(data)) + { + Assert.Equal("Hello", str); + } + }; + yield return new object[] { json, validate }; + } + + private static byte[] WrapInQuotes(string json) + { + byte[] buffer = new byte[json.Length + 2]; + buffer[0] = (byte)'"'; + Encoding.UTF8.GetBytes(json).CopyTo(buffer, 1); + buffer[json.Length + 1] = (byte)'"'; + return buffer; + } + + [Theory] + [InlineData(true, 0, "[]")] + [InlineData(false, 0, "[]")] + [InlineData(true, 1, "[1]")] + [InlineData(false, 1, "[1]")] + [InlineData(true, 5, "[1,1,1,1,1]")] + [InlineData(false, 5, "[1,1,1,1,1]")] + public static void WriteRawArrayElements(bool skipInputValidation, int numElements, string expectedJson) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + writer.WriteStartArray(); + + for (int i = 0; i < numElements; i++) + { + writer.WriteRawValue(s_oneAsJson, skipInputValidation); + } + + writer.WriteEndArray(); + + writer.Flush(); + Assert.Equal(expectedJson, Encoding.UTF8.GetString(ms.ToArray())); + } + + [Theory] + [InlineData(true, 0, "{}")] + [InlineData(false, 0, "{}")] + [InlineData(true, 1, @"{""int"":1}")] + [InlineData(false, 1, @"{""int"":1}")] + [InlineData(true, 3, @"{""int"":1,""int"":1,""int"":1}")] + [InlineData(false, 3, @"{""int"":1,""int"":1,""int"":1}")] + public static void WriteRawObjectProperty(bool skipInputValidation, int numElements, string expectedJson) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + writer.WriteStartObject(); + + for (int i = 0; i < numElements; i++) + { + writer.WritePropertyName("int"); + writer.WriteRawValue(s_oneAsJson, skipInputValidation); + } + + writer.WriteEndObject(); + + writer.Flush(); + Assert.Equal(expectedJson, Encoding.UTF8.GetString(ms.ToArray())); + } + + [Theory] + [InlineData("[")] + [InlineData("}")] + [InlineData("[}")] + [InlineData("xxx")] + [InlineData("{hello:")] + public static void WriteRawInvalidJson(string json) + { + RunTest(true); + RunTest(false); + + void RunTest(bool skipValidation) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + + if (!skipValidation) + { + Assert.Throws(() => writer.WriteRawValue(json)); + } + else + { + writer.WriteRawValue(json, true); + } + } + } + + /// + /// This test is constrained to run on Windows and MacOSX because it causes + /// problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + /// succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + /// time the memory is accessed which triggers the full memory allocation. + /// Also see + /// + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] + [ConditionalFact(nameof(IsX64))] + [OuterLoop] + public void WriteLargeRawJsonToStreamWithoutFlushing() + { + var largeArray = new char[150_000_000]; + largeArray.AsSpan().Fill('a'); + + // Text size chosen so that after several doublings of the underlying buffer we reach ~2 GB (but don't go over) + JsonEncodedText text1 = JsonEncodedText.Encode(largeArray.AsSpan(0, 7_500)); + JsonEncodedText text2 = JsonEncodedText.Encode(largeArray.AsSpan(0, 5_000)); + JsonEncodedText text3 = JsonEncodedText.Encode(largeArray.AsSpan(0, 150_000_000)); + + using (var output = new MemoryStream()) + using (var writer = new Utf8JsonWriter(output)) + { + writer.WriteStartArray(); + writer.WriteRawValue(text1.EncodedUtf8Bytes); + Assert.Equal(7_503, writer.BytesPending); + + for (int i = 0; i < 30_000; i++) + { + writer.WriteRawValue(text2.EncodedUtf8Bytes); + } + Assert.Equal(150_097_503, writer.BytesPending); + + for (int i = 0; i < 13; i++) + { + writer.WriteRawValue(text3.EncodedUtf8Bytes); + } + Assert.Equal(2_100_097_542, writer.BytesPending); + + // Next write forces a grow beyond max array length + + Assert.Throws(() => writer.WriteRawValue(text3.EncodedUtf8Bytes)); + + Assert.Equal(2_100_097_542, writer.BytesPending); + + var text4 = JsonEncodedText.Encode(largeArray.AsSpan(0, 1)); + for (int i = 0; i < 10_000_000; i++) + { + writer.WriteRawValue(text4.EncodedUtf8Bytes); + } + + Assert.Equal(2_100_097_542 + (4 * 10_000_000), writer.BytesPending); + } + } + + [Fact] + public static void WriteRawNullOrEmptyTokenInvalid() + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + Assert.Throws(() => writer.WriteRawValue(json: default(string))); + Assert.Throws(() => writer.WriteRawValue(json: "")); + Assert.Throws(() => writer.WriteRawValue(json: default(ReadOnlySpan))); + Assert.Throws(() => writer.WriteRawValue(utf8Json: default)); + } + } +} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs index f4796dcf60e40..ce68f8495e112 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs @@ -17,7 +17,7 @@ namespace System.Text.Json.Tests { - public class Utf8JsonWriterTests + public partial class Utf8JsonWriterTests { private const int MaxExpansionFactorWhileEscaping = 6; private const int MaxEscapedTokenSize = 1_000_000_000; // Max size for already escaped value. @@ -734,10 +734,13 @@ private static string GetExpectedLargeArrayOfStrings(int length) return stringBuilder.ToString(); } - // NOTE: WritingTooLargeProperty test is constrained to run on Windows and MacOSX because it causes - // problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can - // succeed even if there is not enough memory but then the test may get killed by the OOM killer at the - // time the memory is accessed which triggers the full memory allocation. + /// + /// This test is constrained to run on Windows and MacOSX because it causes + /// problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + /// succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + /// time the memory is accessed which triggers the full memory allocation. + /// Also see + /// [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalFact(nameof(IsX64))] [OuterLoop] From 3ac5592c190795ee9b9b2dae0916272cdd83d02d Mon Sep 17 00:00:00 2001 From: Layomi Akinrinade Date: Tue, 29 Jun 2021 11:20:02 -0700 Subject: [PATCH 2/6] Address review feedback --- .../src/System/Text/Json/JsonConstants.cs | 2 + .../Writer/Utf8JsonWriter.WriteValues.Raw.cs | 124 +++++++++++--- .../System.Text.Json.Tests.csproj | 2 +- .../Utf8JsonWriterTests.WriteRaw.cs | 154 +++++++++++++++--- .../Utf8JsonWriterTests.cs | 2 +- 5 files changed, 234 insertions(+), 50 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs index bb27d8b1a563b..c1568c8f2233f 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs @@ -67,6 +67,8 @@ internal static class JsonConstants // When transcoding from UTF8 -> UTF16, the byte count threshold where we rent from the array pool before performing a normal alloc. public const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024; + public const int MaxRawValueLength = int.MaxValue / MaxExpansionFactorWhileTranscoding; + public const int MaxEscapedTokenSize = 1_000_000_000; // Max size for already escaped value. public const int MaxUnescapedTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping; // 166_666_666 bytes public const int MaxBase64ValueTokenSize = (MaxEscapedTokenSize >> 2) * 3 / MaxExpansionFactorWhileEscaping; // 125_000_000 bytes diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs index 3ea7e1f26c8aa..051f9e2d16965 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs @@ -8,26 +8,93 @@ namespace System.Text.Json public sealed partial class Utf8JsonWriter { /// - /// Writes the input as JSON content. + /// Writes the input as JSON content. It is expected that the input content is a single complete JSON value. /// /// The raw JSON content to write. - /// Whether to skip validation of the input JSON content. + /// Whether to validate if the input is an RFC 8259-compliant JSON payload. + /// Thrown if is . + /// Thrown if the length of the input is zero or greater than 715,827,882. + /// Thrown if is , and the input is not RFC 8259-compliant. + /// + /// When writing untrused JSON values, do not set to as this can result in invalid JSON + /// being written, and/or the overall payload being written to the writer instance being invalid. + /// + /// When using this method, the input content will be written to the writer destination as-is, unless validation fails. + /// + /// The value for the writer instance is honored when using this method. + /// + /// The and values for the writer instance are not applied when using this method. + /// public void WriteRawValue(string json, bool skipInputValidation = false) { + if (!_options.SkipValidation) + { + ValidateWritingValue(); + } + if (json == null) { throw new ArgumentNullException(nameof(json)); } - WriteRawValue(json.AsSpan(), skipInputValidation); + TranscodeAndWriteRawValue(json.AsSpan(), skipInputValidation); } /// - /// Writes the input as JSON content. + /// Writes the input as JSON content. It is expected that the input content is a single complete JSON value. /// /// The raw JSON content to write. - /// Whether to skip validation of the input JSON content. + /// Whether to validate if the input is an RFC 8259-compliant JSON payload. + /// Thrown if the length of the input is zero or greater than 715,827,882. + /// Thrown if is , and the input is not RFC 8259-compliant. + /// + /// When writing untrused JSON values, do not set to as this can result in invalid JSON + /// being written, and/or the overall payload being written to the writer instance being invalid. + /// + /// When using this method, the input content will be written to the writer destination as-is, unless validation fails. + /// + /// The value for the writer instance is honored when using this method. + /// + /// The and values for the writer instance are not applied when using this method. + /// public void WriteRawValue(ReadOnlySpan json, bool skipInputValidation = false) + { + if (!_options.SkipValidation) + { + ValidateWritingValue(); + } + + TranscodeAndWriteRawValue(json, skipInputValidation); + } + + /// + /// Writes the input as JSON content. It is expected that the input content is a single complete JSON value. + /// + /// The raw JSON content to write. + /// Whether to validate if the input is an RFC 8259-compliant JSON payload. + /// Thrown if the length of the input is zero or greater than 715,827,882. + /// Thrown if is , and the input is not RFC 8259-compliant. + /// + /// When writing untrused JSON values, do not set to as this can result in invalid JSON + /// being written, and/or the overall payload being written to the writer instance being invalid. + /// + /// When using this method, the input content will be written to the writer destination as-is, unless validation fails. + /// + /// The value for the writer instance is honored when using this method. + /// + /// The and values for the writer instance are not applied when using this method. + /// + public void WriteRawValue(ReadOnlySpan utf8Json, bool skipInputValidation = false) + { + if (!_options.SkipValidation) + { + ValidateWritingValue(); + } + + WriteRawValueInternal(utf8Json, skipInputValidation); + } + + private void TranscodeAndWriteRawValue(ReadOnlySpan json, bool skipInputValidation) { byte[]? tempArray = null; @@ -55,33 +122,40 @@ public void WriteRawValue(ReadOnlySpan json, bool skipInputValidation = fa } } - /// - /// Writes the input as JSON content. - /// - /// The raw JSON content to write. - /// Whether to skip validation of the input JSON content. - public void WriteRawValue(ReadOnlySpan utf8Json, bool skipInputValidation = false) + private void WriteRawValueInternal(ReadOnlySpan utf8Json, bool skipInputValidation) { - if (utf8Json.Length == 0) + int len = utf8Json.Length; + + if (len == 0) { ThrowHelper.ThrowArgumentException(SR.ExpectedJsonTokens); } - - if (!skipInputValidation) + else if (len > JsonConstants.MaxRawValueLength) { - Utf8JsonReader reader = new Utf8JsonReader(utf8Json); + ThrowHelper.ThrowArgumentException_ValueTooLarge(len); + } - try - { - while (reader.Read()); - } - catch (JsonReaderException ex) + if (skipInputValidation) + { + // Treat all unvalidated raw JSON value writes as string. If the payload is valid, this approach does + // not affect structural validation since a string token is equivalent to a complete object, array, + // or other complete JSON tokens when considering structural validation on subsequent writer calls. + // If the payload is not valid, then we make no guarantees about the structural validation of the final payload. + _tokenType = JsonTokenType.String; + } + else + { + // Utilize reader validation. + Utf8JsonReader reader = new(utf8Json); + while (reader.Read()) { - ThrowHelper.ThrowArgumentException(ex.Message); + _tokenType = reader.TokenType; } } - int maxRequired = utf8Json.Length + 1; // Optionally, 1 list separator + // TODO (https://github.com/dotnet/runtime/issues/29293): + // investigate writing this in chunks, rather than requesting one potentially long, contiguous buffer. + int maxRequired = len + 1; // Optionally, 1 list separator if (_memory.Length - BytesPending < maxRequired) { @@ -96,12 +170,10 @@ public void WriteRawValue(ReadOnlySpan utf8Json, bool skipInputValidation } utf8Json.CopyTo(output.Slice(BytesPending)); - BytesPending += utf8Json.Length; + BytesPending += len; - SetFlagToAddListSeparatorBeforeNextItem(); - // Treat all raw JSON value writes as string. - _tokenType = JsonTokenType.String; + SetFlagToAddListSeparatorBeforeNextItem(); } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj index d3ba50335a1d6..964f2c6d33a79 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj @@ -186,10 +186,10 @@ + - diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs index 219d49393416f..27a342e43a5ec 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs @@ -12,7 +12,6 @@ public partial class Utf8JsonWriterTests { private const string TestGuidAsStr = "eb97fadd-3ebf-4781-8722-f4773989160e"; private readonly static Guid s_guid = Guid.Parse(TestGuidAsStr); - private static byte[] s_guidAsJson = WrapInQuotes(TestGuidAsStr); private static byte[] s_oneAsJson = new byte[] { (byte)'1' }; @@ -67,7 +66,8 @@ public static IEnumerable GetRootLevelPrimitives() yield return new object[] { Encoding.UTF8.GetBytes(@"""Hello"""), validate }; validate = (data) => Assert.Equal(s_guid, JsonSerializer.Deserialize(data)); - yield return new object[] { s_guidAsJson, validate }; + byte[] guidAsJson = WrapInQuotes(Encoding.UTF8.GetBytes(TestGuidAsStr)); + yield return new object[] { guidAsJson, validate }; } public static IEnumerable GetArrays() @@ -105,13 +105,13 @@ public static IEnumerable GetArrays() yield return new object[] { json, validate }; } - private static byte[] WrapInQuotes(string json) + private static byte[] WrapInQuotes(ReadOnlySpan buffer) { - byte[] buffer = new byte[json.Length + 2]; - buffer[0] = (byte)'"'; - Encoding.UTF8.GetBytes(json).CopyTo(buffer, 1); - buffer[json.Length + 1] = (byte)'"'; - return buffer; + byte[] quotedBuffer = new byte[buffer.Length + 2]; + quotedBuffer[0] = (byte)'"'; + buffer.CopyTo(quotedBuffer.AsSpan().Slice(1)); + quotedBuffer[buffer.Length + 1] = (byte)'"'; + return quotedBuffer; } [Theory] @@ -169,6 +169,11 @@ public static void WriteRawObjectProperty(bool skipInputValidation, int numEleme [InlineData("[}")] [InlineData("xxx")] [InlineData("{hello:")] + [InlineData("\\u007Bhello:")] + [InlineData(@"{""hello:""""")] + [InlineData(" ")] + [InlineData("// This is a single line comment")] + [InlineData("/* This is a multi-\nline comment*/")] public static void WriteRawInvalidJson(string json) { RunTest(true); @@ -181,11 +186,52 @@ void RunTest(bool skipValidation) if (!skipValidation) { - Assert.Throws(() => writer.WriteRawValue(json)); + Assert.ThrowsAny(() => writer.WriteRawValue(json)); } else { writer.WriteRawValue(json, true); + writer.Flush(); + Assert.True(Encoding.UTF8.GetBytes(json).SequenceEqual(ms.ToArray())); + } + } + } + + [Fact] + public static void WriteRawNullOrEmptyTokenInvalid() + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + Assert.Throws(() => writer.WriteRawValue(json: default(string))); + Assert.Throws(() => writer.WriteRawValue(json: "")); + Assert.Throws(() => writer.WriteRawValue(json: default(ReadOnlySpan))); + Assert.Throws(() => writer.WriteRawValue(utf8Json: default)); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public static void WriteRawHonorSkipValidation(bool skipValidation) + { + RunTest(true); + RunTest(false); + + void RunTest(bool skipInputValidation) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms, new JsonWriterOptions { SkipValidation = skipValidation }); + + writer.WriteStartObject(); + + if (skipValidation) + { + writer.WriteRawValue(@"{}", skipInputValidation); + writer.Flush(); + Assert.True(ms.ToArray().SequenceEqual(new byte[] { (byte)'{', (byte)'{', (byte)'}' })); + } + else + { + Assert.Throws(() => writer.WriteRawValue(@"{}", skipInputValidation)); } } } @@ -195,12 +241,12 @@ void RunTest(bool skipValidation) /// problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can /// succeed even if there is not enough memory but then the test may get killed by the OOM killer at the /// time the memory is accessed which triggers the full memory allocation. - /// Also see + /// Also see /// [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalFact(nameof(IsX64))] [OuterLoop] - public void WriteLargeRawJsonToStreamWithoutFlushing() + public void WriteRawLargeJsonToStreamWithoutFlushing() { var largeArray = new char[150_000_000]; largeArray.AsSpan().Fill('a'); @@ -214,46 +260,110 @@ public void WriteLargeRawJsonToStreamWithoutFlushing() using (var writer = new Utf8JsonWriter(output)) { writer.WriteStartArray(); - writer.WriteRawValue(text1.EncodedUtf8Bytes); + writer.WriteRawValue(WrapInQuotes(text1.EncodedUtf8Bytes)); Assert.Equal(7_503, writer.BytesPending); for (int i = 0; i < 30_000; i++) { - writer.WriteRawValue(text2.EncodedUtf8Bytes); + writer.WriteRawValue(WrapInQuotes(text2.EncodedUtf8Bytes)); } Assert.Equal(150_097_503, writer.BytesPending); for (int i = 0; i < 13; i++) { - writer.WriteRawValue(text3.EncodedUtf8Bytes); + writer.WriteRawValue(WrapInQuotes(text3.EncodedUtf8Bytes)); } Assert.Equal(2_100_097_542, writer.BytesPending); // Next write forces a grow beyond max array length - Assert.Throws(() => writer.WriteRawValue(text3.EncodedUtf8Bytes)); + Assert.Throws(() => writer.WriteRawValue(WrapInQuotes(text3.EncodedUtf8Bytes))); Assert.Equal(2_100_097_542, writer.BytesPending); var text4 = JsonEncodedText.Encode(largeArray.AsSpan(0, 1)); for (int i = 0; i < 10_000_000; i++) { - writer.WriteRawValue(text4.EncodedUtf8Bytes); + writer.WriteRawValue(WrapInQuotes(text4.EncodedUtf8Bytes)); } Assert.Equal(2_100_097_542 + (4 * 10_000_000), writer.BytesPending); } } - [Fact] - public static void WriteRawNullOrEmptyTokenInvalid() + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] + [ConditionalTheory(nameof(IsX64))] + [InlineData(JsonTokenType.String)] + [InlineData(JsonTokenType.StartArray)] + [InlineData(JsonTokenType.StartObject)] + public static void WriteRawMaxInputLength(JsonTokenType tokenType) { + // Max raw payload length supported by the writer. + int maxLength = int.MaxValue / 3; + + byte[] payload = new byte[maxLength]; + payload[0] = (byte)'"'; + payload[maxLength - 1] = (byte)'"'; + + for (int i = 1; i < maxLength - 1; i++) + { + payload[i] = (byte)'a'; + } + using MemoryStream ms = new(); using Utf8JsonWriter writer = new(ms); - Assert.Throws(() => writer.WriteRawValue(json: default(string))); - Assert.Throws(() => writer.WriteRawValue(json: "")); - Assert.Throws(() => writer.WriteRawValue(json: default(ReadOnlySpan))); - Assert.Throws(() => writer.WriteRawValue(utf8Json: default)); + + switch(tokenType) + { + case JsonTokenType.String: + writer.WriteRawValue(payload); + writer.Flush(); + Assert.Equal(payload.Length, writer.BytesCommitted); + break; + case JsonTokenType.StartArray: + writer.WriteStartArray(); + writer.WriteRawValue(payload); + writer.WriteRawValue(payload); + writer.WriteEndArray(); + writer.Flush(); + // Start/EndArray + comma, 2 array elements + Assert.Equal(3 + (payload.Length * 2), writer.BytesCommitted); + break; + case JsonTokenType.StartObject: + writer.WriteStartObject(); + writer.WritePropertyName("1"); + writer.WriteRawValue(payload); + writer.WritePropertyName("2"); + writer.WriteRawValue(payload); + writer.WriteEndObject(); + writer.Flush(); + // Start/EndToken + comma, 2 property names, 2 property values + Assert.Equal(3 + (4 * 2) + (payload.Length * 2), writer.BytesCommitted); + break; + default: + Assert.True(false, "Unexpected test configuration"); + break; + } + } + + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] + [ConditionalTheory(nameof(IsX64))] + [InlineData((int.MaxValue / 3) + 1)] + [InlineData(int.MaxValue / 3 + 2)] + public static void WriteRawLengthGreaterThanMax(int len) + { + byte[] payload = new byte[len]; + payload[0] = (byte)'"'; + payload[len - 1] = (byte)'"'; + + for (int i = 1; i < len - 1; i++) + { + payload[i] = (byte)'a'; + } + + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + Assert.Throws(() => writer.WriteRawValue(payload)); } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs index ce68f8495e112..f18f23a64f3c6 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs @@ -739,7 +739,7 @@ private static string GetExpectedLargeArrayOfStrings(int length) /// problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can /// succeed even if there is not enough memory but then the test may get killed by the OOM killer at the /// time the memory is accessed which triggers the full memory allocation. - /// Also see + /// Also see /// [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalFact(nameof(IsX64))] From e47ee370094752454c1d9ebfa47271d78dd6ef1f Mon Sep 17 00:00:00 2001 From: Layomi Akinrinade Date: Thu, 1 Jul 2021 14:34:34 -0700 Subject: [PATCH 3/6] Address review feedback --- .../src/System/Text/Json/JsonConstants.cs | 4 +- .../Writer/Utf8JsonWriter.WriteValues.Raw.cs | 59 +++-- .../Utf8JsonWriterTests.WriteRaw.cs | 215 ++++++++++++++---- 3 files changed, 220 insertions(+), 58 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs index c1568c8f2233f..a2e91cd3b08fc 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs @@ -67,7 +67,9 @@ internal static class JsonConstants // When transcoding from UTF8 -> UTF16, the byte count threshold where we rent from the array pool before performing a normal alloc. public const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024; - public const int MaxRawValueLength = int.MaxValue / MaxExpansionFactorWhileTranscoding; + // The maximum number of characters allowed when writing raw UTF-16 JSON. This is the maximum length that we can guarantee can + // be safely transcoded to UTF-8 and fit within an integer-length span, given the max expansion factor of a single character (3). + public const int MaxUtf16RawValueLength = int.MaxValue / MaxExpansionFactorWhileTranscoding; public const int MaxEscapedTokenSize = 1_000_000_000; // Max size for already escaped value. public const int MaxUnescapedTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping; // 166_666_666 bytes diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs index 051f9e2d16965..48983c62ae777 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Buffers; +using System.Diagnostics; namespace System.Text.Json { @@ -14,12 +15,16 @@ public sealed partial class Utf8JsonWriter /// Whether to validate if the input is an RFC 8259-compliant JSON payload. /// Thrown if is . /// Thrown if the length of the input is zero or greater than 715,827,882. - /// Thrown if is , and the input is not RFC 8259-compliant. + /// + /// Thrown if is , and the input + /// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259) + /// or the input JSON exceeds the default recursive depth of 64. + /// /// /// When writing untrused JSON values, do not set to as this can result in invalid JSON /// being written, and/or the overall payload being written to the writer instance being invalid. /// - /// When using this method, the input content will be written to the writer destination as-is, unless validation fails. + /// When using this method, the input content will be written to the writer destination as-is, unless validation fails (when it is enabled). /// /// The value for the writer instance is honored when using this method. /// @@ -46,12 +51,16 @@ public void WriteRawValue(string json, bool skipInputValidation = false) /// The raw JSON content to write. /// Whether to validate if the input is an RFC 8259-compliant JSON payload. /// Thrown if the length of the input is zero or greater than 715,827,882. - /// Thrown if is , and the input is not RFC 8259-compliant. + /// + /// Thrown if is , and the input + /// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259) + /// or the input JSON exceeds the default recursive depth of 64. + /// /// /// When writing untrused JSON values, do not set to as this can result in invalid JSON /// being written, and/or the overall payload being written to the writer instance being invalid. /// - /// When using this method, the input content will be written to the writer destination as-is, unless validation fails. + /// When using this method, the input content will be written to the writer destination as-is, unless validation fails (when it is enabled). /// /// The value for the writer instance is honored when using this method. /// @@ -72,13 +81,17 @@ public void WriteRawValue(ReadOnlySpan json, bool skipInputValidation = fa /// /// The raw JSON content to write. /// Whether to validate if the input is an RFC 8259-compliant JSON payload. - /// Thrown if the length of the input is zero or greater than 715,827,882. - /// Thrown if is , and the input is not RFC 8259-compliant. + /// Thrown if the length of the input is zero or equal to . + /// + /// Thrown if is , and the input + /// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259) + /// or the input JSON exceeds the default recursive depth of 64. + /// /// /// When writing untrused JSON values, do not set to as this can result in invalid JSON /// being written, and/or the overall payload being written to the writer instance being invalid. /// - /// When using this method, the input content will be written to the writer destination as-is, unless validation fails. + /// When using this method, the input content will be written to the writer destination as-is, unless validation fails (when it is enabled). /// /// The value for the writer instance is honored when using this method. /// @@ -91,11 +104,21 @@ public void WriteRawValue(ReadOnlySpan utf8Json, bool skipInputValidation ValidateWritingValue(); } - WriteRawValueInternal(utf8Json, skipInputValidation); + if (utf8Json.Length == int.MaxValue) + { + ThrowHelper.ThrowArgumentException_ValueTooLarge(int.MaxValue); + } + + WriteRawValueCore(utf8Json, skipInputValidation); } private void TranscodeAndWriteRawValue(ReadOnlySpan json, bool skipInputValidation) { + if (json.Length > JsonConstants.MaxUtf16RawValueLength) + { + ThrowHelper.ThrowArgumentException_ValueTooLarge(json.Length); + } + byte[]? tempArray = null; // For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold. @@ -122,7 +145,7 @@ private void TranscodeAndWriteRawValue(ReadOnlySpan json, bool skipInputVa } } - private void WriteRawValueInternal(ReadOnlySpan utf8Json, bool skipInputValidation) + private void WriteRawValueCore(ReadOnlySpan utf8Json, bool skipInputValidation) { int len = utf8Json.Length; @@ -130,10 +153,12 @@ private void WriteRawValueInternal(ReadOnlySpan utf8Json, bool skipInputVa { ThrowHelper.ThrowArgumentException(SR.ExpectedJsonTokens); } - else if (len > JsonConstants.MaxRawValueLength) - { - ThrowHelper.ThrowArgumentException_ValueTooLarge(len); - } + + // In the UTF-16-based entry point methods above, we validate that the payload length <= int.MaxValue /3. + // The result of this division will be rounded down, so even if every input character needs to be transcoded + // (with expansion factor of 3), the resulting payload would be less than int.MaxValue, + // as (int.MaxValue/3) * 3 is less than int.MaxValue. + Debug.Assert(len < int.MaxValue); if (skipInputValidation) { @@ -147,15 +172,13 @@ private void WriteRawValueInternal(ReadOnlySpan utf8Json, bool skipInputVa { // Utilize reader validation. Utf8JsonReader reader = new(utf8Json); - while (reader.Read()) - { - _tokenType = reader.TokenType; - } + while (reader.Read()); + _tokenType = reader.TokenType; } // TODO (https://github.com/dotnet/runtime/issues/29293): // investigate writing this in chunks, rather than requesting one potentially long, contiguous buffer. - int maxRequired = len + 1; // Optionally, 1 list separator + int maxRequired = len + 1; // Optionally, 1 list separator. We've guarded against integer overflow earlier in the call stack. if (_memory.Length - BytesPending < maxRequired) { diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs index 27a342e43a5ec..84c171ea6ad6f 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs @@ -109,7 +109,7 @@ private static byte[] WrapInQuotes(ReadOnlySpan buffer) { byte[] quotedBuffer = new byte[buffer.Length + 2]; quotedBuffer[0] = (byte)'"'; - buffer.CopyTo(quotedBuffer.AsSpan().Slice(1)); + buffer.CopyTo(quotedBuffer.AsSpan(1)); quotedBuffer[buffer.Length + 1] = (byte)'"'; return quotedBuffer; } @@ -236,6 +236,56 @@ void RunTest(bool skipInputValidation) } } + [Fact] + public static void WriteRawDepthExceedsMaxOf64Fail() + { + + RunTest(GenerateJsonUsingDepth(1), false); + RunTest(GenerateJsonUsingDepth(64), false); + RunTest(GenerateJsonUsingDepth(65), true); + RunTest(GenerateJsonUsingDepth(65), false, true); + + void RunTest(string json, bool expectFail, bool skipInputValidation = false) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + + if (expectFail) + { + Assert.ThrowsAny(() => writer.WriteRawValue(json, skipInputValidation)); + } + else + { + writer.WriteRawValue(json, skipInputValidation); + writer.Flush(); + + Assert.Equal(json, Encoding.UTF8.GetString(ms.ToArray())); + } + } + } + + private static string GenerateJsonUsingDepth(int depth) + { + Assert.True(depth > 0 && depth <= 65, "Test depth out of range"); + + StringBuilder sb = new(); + sb.Append("{"); + + for (int i = 0; i < depth - 1; i++) + { + sb.Append(@"""prop"":{"); + } + + for (int i = 0; i < depth - 1; i++) + { + sb.Append("}"); + } + + sb.Append("}"); + + return sb.ToString(); + } + /// /// This test is constrained to run on Windows and MacOSX because it causes /// problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can @@ -296,52 +346,86 @@ public void WriteRawLargeJsonToStreamWithoutFlushing() [InlineData(JsonTokenType.String)] [InlineData(JsonTokenType.StartArray)] [InlineData(JsonTokenType.StartObject)] - public static void WriteRawMaxInputLength(JsonTokenType tokenType) + public static void WriteRawMaxUtf16InputLength(JsonTokenType tokenType) { // Max raw payload length supported by the writer. int maxLength = int.MaxValue / 3; - byte[] payload = new byte[maxLength]; - payload[0] = (byte)'"'; - payload[maxLength - 1] = (byte)'"'; + StringBuilder sb = new(); + sb.Append('"'); for (int i = 1; i < maxLength - 1; i++) { - payload[i] = (byte)'a'; + sb.Append('a'); } - using MemoryStream ms = new(); - using Utf8JsonWriter writer = new(ms); + sb.Append('"'); + + string payload = sb.ToString(); - switch(tokenType) + RunTest(OverloadParamType.ROSChar); + RunTest(OverloadParamType.String); + RunTest(OverloadParamType.ByteArray); + + void RunTest(OverloadParamType paramType) { - case JsonTokenType.String: - writer.WriteRawValue(payload); - writer.Flush(); - Assert.Equal(payload.Length, writer.BytesCommitted); - break; - case JsonTokenType.StartArray: - writer.WriteStartArray(); - writer.WriteRawValue(payload); - writer.WriteRawValue(payload); - writer.WriteEndArray(); - writer.Flush(); - // Start/EndArray + comma, 2 array elements - Assert.Equal(3 + (payload.Length * 2), writer.BytesCommitted); + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + + switch (tokenType) + { + case JsonTokenType.String: + WriteRawValueWithSetting(writer, payload, paramType); + writer.Flush(); + Assert.Equal(payload.Length, writer.BytesCommitted); + break; + case JsonTokenType.StartArray: + writer.WriteStartArray(); + WriteRawValueWithSetting(writer, payload, paramType); + WriteRawValueWithSetting(writer, payload, paramType); + writer.WriteEndArray(); + writer.Flush(); + // Start/EndArray + comma, 2 array elements + Assert.Equal(3 + (payload.Length * 2), writer.BytesCommitted); + break; + case JsonTokenType.StartObject: + writer.WriteStartObject(); + writer.WritePropertyName("1"); + WriteRawValueWithSetting(writer, payload, paramType); + writer.WritePropertyName("2"); + WriteRawValueWithSetting(writer, payload, paramType); + writer.WriteEndObject(); + writer.Flush(); + // Start/EndToken + comma, 2 property names, 2 property values + Assert.Equal(3 + (4 * 2) + (payload.Length * 2), writer.BytesCommitted); + break; + default: + Assert.True(false, "Unexpected test configuration"); + break; + } + } + } + + private enum OverloadParamType + { + ROSChar, + String, + ByteArray + } + + private static void WriteRawValueWithSetting(Utf8JsonWriter writer, string payload, OverloadParamType param) + { + switch (param) + { + case OverloadParamType.ROSChar: + writer.WriteRawValue(payload.AsSpan()); break; - case JsonTokenType.StartObject: - writer.WriteStartObject(); - writer.WritePropertyName("1"); - writer.WriteRawValue(payload); - writer.WritePropertyName("2"); + case OverloadParamType.String: writer.WriteRawValue(payload); - writer.WriteEndObject(); - writer.Flush(); - // Start/EndToken + comma, 2 property names, 2 property values - Assert.Equal(3 + (4 * 2) + (payload.Length * 2), writer.BytesCommitted); break; - default: - Assert.True(false, "Unexpected test configuration"); + case OverloadParamType.ByteArray: + byte[] payloadAsBytes = Encoding.UTF8.GetBytes(payload); + writer.WriteRawValue(payloadAsBytes); break; } } @@ -350,20 +434,73 @@ public static void WriteRawMaxInputLength(JsonTokenType tokenType) [ConditionalTheory(nameof(IsX64))] [InlineData((int.MaxValue / 3) + 1)] [InlineData(int.MaxValue / 3 + 2)] - public static void WriteRawLengthGreaterThanMax(int len) + public static void WriteRawUtf16LengthGreaterThanMax(int len) { - byte[] payload = new byte[len]; - payload[0] = (byte)'"'; - payload[len - 1] = (byte)'"'; + StringBuilder sb = new(); + sb.Append('"'); for (int i = 1; i < len - 1; i++) { - payload[i] = (byte)'a'; + sb.Append('a'); } + sb.Append('"'); + + string payload = sb.ToString(); + using MemoryStream ms = new(); using Utf8JsonWriter writer = new(ms); - Assert.Throws(() => writer.WriteRawValue(payload)); + + // UTF-16 overloads not compatible with this length. + Assert.Throws(() => WriteRawValueWithSetting(writer, payload, OverloadParamType.ROSChar)); + Assert.Throws(() => WriteRawValueWithSetting(writer, payload, OverloadParamType.String)); + + // UTF-8 overload is okay. + WriteRawValueWithSetting(writer, payload, OverloadParamType.ByteArray); + writer.Flush(); + + Assert.Equal(payload.Length, Encoding.UTF8.GetString(ms.ToArray()).Length); + } + + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] + [ConditionalFact(nameof(IsX64))] + public static void WriteRawTranscodeFromUtf16ToUtf8TooLong() + { + // Max raw payload length supported by the writer. + int maxLength = int.MaxValue / 3; + + StringBuilder sb = new(); + sb.Append('"'); + + for (int i = 1; i < maxLength - 1; i++) + { + sb.Append('的'); // Non-UTF-8 character than will expand during transcoding + } + + sb.Append('"'); + + string payload = sb.ToString(); + + RunTest(OverloadParamType.ROSChar); + RunTest(OverloadParamType.String); + RunTest(OverloadParamType.ByteArray); + + void RunTest(OverloadParamType paramType) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + + try + { + WriteRawValueWithSetting(writer, payload, paramType); + writer.Flush(); + + // All characters in the payload will be expanded during transcoding, except for the quotes. + int expectedLength = ((payload.Length - 2) * 3) + 2; + Assert.Equal(expectedLength, writer.BytesCommitted); + } + catch (OutOfMemoryException) { } // OutOfMemoryException is okay since the transcoding output is probably too large. + } } } } From eea5f1b0510bccf3a360bad94cfc0913c80c57d6 Mon Sep 17 00:00:00 2001 From: Layomi Akinrinade Date: Fri, 2 Jul 2021 09:46:05 -0700 Subject: [PATCH 4/6] Address review feedback --- .../Writer/Utf8JsonWriter.WriteValues.Raw.cs | 11 +++--- .../Utf8JsonWriterTests.WriteRaw.cs | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs index 48983c62ae777..2b8070c920a08 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs @@ -14,11 +14,11 @@ public sealed partial class Utf8JsonWriter /// The raw JSON content to write. /// Whether to validate if the input is an RFC 8259-compliant JSON payload. /// Thrown if is . - /// Thrown if the length of the input is zero or greater than 715,827,882. + /// Thrown if the length of the input is zero or greater than 715,827,882 ( / 3). /// /// Thrown if is , and the input /// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259) - /// or the input JSON exceeds the default recursive depth of 64. + /// or the input JSON exceeds a recursive depth of 64. /// /// /// When writing untrused JSON values, do not set to as this can result in invalid JSON @@ -50,11 +50,11 @@ public void WriteRawValue(string json, bool skipInputValidation = false) /// /// The raw JSON content to write. /// Whether to validate if the input is an RFC 8259-compliant JSON payload. - /// Thrown if the length of the input is zero or greater than 715,827,882. + /// Thrown if the length of the input is zero or greater than 715,827,882 ( / 3). /// /// Thrown if is , and the input /// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259) - /// or the input JSON exceeds the default recursive depth of 64. + /// or the input JSON exceeds a recursive depth of 64. /// /// /// When writing untrused JSON values, do not set to as this can result in invalid JSON @@ -85,7 +85,7 @@ public void WriteRawValue(ReadOnlySpan json, bool skipInputValidation = fa /// /// Thrown if is , and the input /// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259) - /// or the input JSON exceeds the default recursive depth of 64. + /// or the input JSON exceeds a recursive depth of 64. /// /// /// When writing untrused JSON values, do not set to as this can result in invalid JSON @@ -195,7 +195,6 @@ private void WriteRawValueCore(ReadOnlySpan utf8Json, bool skipInputValida utf8Json.CopyTo(output.Slice(BytesPending)); BytesPending += len; - SetFlagToAddListSeparatorBeforeNextItem(); } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs index 84c171ea6ad6f..4a17ccfdbe7d6 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs @@ -62,9 +62,15 @@ public static IEnumerable GetRootLevelPrimitives() validate = (data) => Assert.Equal(1234.56789, JsonSerializer.Deserialize(data)); yield return new object[] { Encoding.UTF8.GetBytes("1234.56789"), validate }; + validate = (data) => Assert.Equal(1234.56789, JsonSerializer.Deserialize(data)); + yield return new object[] { Encoding.UTF8.GetBytes(" 1234.56789 "), validate }; + validate = (data) => Assert.Equal(@"Hello", JsonSerializer.Deserialize(data)); yield return new object[] { Encoding.UTF8.GetBytes(@"""Hello"""), validate }; + validate = (data) => Assert.Equal(@"Hello", JsonSerializer.Deserialize(data)); + yield return new object[] { Encoding.UTF8.GetBytes(@" ""Hello"" "), validate }; + validate = (data) => Assert.Equal(s_guid, JsonSerializer.Deserialize(data)); byte[] guidAsJson = WrapInQuotes(Encoding.UTF8.GetBytes(TestGuidAsStr)); yield return new object[] { guidAsJson, validate }; @@ -103,6 +109,39 @@ public static IEnumerable GetArrays() } }; yield return new object[] { json, validate }; + + json = Encoding.UTF8.GetBytes("[ 1, 1,1,1,1 ] "); + validate = (data) => + { + foreach (int val in JsonSerializer.Deserialize(data)) + { + Assert.Equal(1, val); + } + }; + yield return new object[] { json, validate }; + } + + public static IEnumerable GetObjects() + { + Action validate; + + byte[] json = Encoding.UTF8.GetBytes(@"{""Hello"":""World""}"); ; + validate = (data) => + { + KeyValuePair kvp = JsonSerializer.Deserialize>(data).Single(); + Assert.Equal("Hello", kvp.Key); + Assert.Equal("World", kvp.Value); + }; + yield return new object[] { json, validate }; + + json = Encoding.UTF8.GetBytes(@" { ""Hello"" :""World"" } "); ; + validate = (data) => + { + KeyValuePair kvp = JsonSerializer.Deserialize>(data).Single(); + Assert.Equal("Hello", kvp.Key); + Assert.Equal("World", kvp.Value); + }; + yield return new object[] { json, validate }; } private static byte[] WrapInQuotes(ReadOnlySpan buffer) From e15805770a647fbc29f3d26dc7af333a12f95a5e Mon Sep 17 00:00:00 2001 From: Layomi Akinrinade Date: Fri, 2 Jul 2021 12:30:50 -0700 Subject: [PATCH 5/6] Make long-running tests outerloop --- .../System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs index 4a17ccfdbe7d6..eb59af3510cb9 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs @@ -382,6 +382,7 @@ public void WriteRawLargeJsonToStreamWithoutFlushing() [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalTheory(nameof(IsX64))] + [OuterLoop] [InlineData(JsonTokenType.String)] [InlineData(JsonTokenType.StartArray)] [InlineData(JsonTokenType.StartObject)] @@ -473,6 +474,7 @@ private static void WriteRawValueWithSetting(Utf8JsonWriter writer, string paylo [ConditionalTheory(nameof(IsX64))] [InlineData((int.MaxValue / 3) + 1)] [InlineData(int.MaxValue / 3 + 2)] + [OuterLoop] public static void WriteRawUtf16LengthGreaterThanMax(int len) { StringBuilder sb = new(); @@ -503,6 +505,7 @@ public static void WriteRawUtf16LengthGreaterThanMax(int len) [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalFact(nameof(IsX64))] + [OuterLoop] public static void WriteRawTranscodeFromUtf16ToUtf8TooLong() { // Max raw payload length supported by the writer. From 86584f09be0716ca148d0d81c02492e34b740ecc Mon Sep 17 00:00:00 2001 From: Layomi Akinrinade Date: Sat, 3 Jul 2021 04:02:31 -0700 Subject: [PATCH 6/6] Fix call to core logic --- .../System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs index 2b8070c920a08..5ac9c8cebe315 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs @@ -133,7 +133,7 @@ private void TranscodeAndWriteRawValue(ReadOnlySpan json, bool skipInputVa { int actualByteCount = JsonReaderHelper.GetUtf8FromText(json, utf8Json); utf8Json = utf8Json.Slice(0, actualByteCount); - WriteRawValue(utf8Json, skipInputValidation); + WriteRawValueCore(utf8Json, skipInputValidation); } finally {