From cbe96dd3cd22d672cc0c59e612c27ce7dd6a25a8 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Wed, 4 Sep 2024 00:39:53 +0530 Subject: [PATCH 1/2] Add streaming support for JSON (#2801) This commit adds support for streaming JSON data to and from Sql Server --- .../Microsoft/Data/SqlClient/SqlDataReader.cs | 2 +- .../src/Microsoft/Data/SqlClient/TdsParser.cs | 5 +- .../Microsoft/Data/SqlClient/SqlDataReader.cs | 2 +- .../src/Microsoft/Data/SqlClient/TdsParser.cs | 5 +- .../src/Microsoft/Data/SqlClient/SqlEnums.cs | 1 + .../ManualTests/DataCommon/DataTestUtility.cs | 11 + ....Data.SqlClient.ManualTesting.Tests.csproj | 1 + .../SQL/JsonTest/JsonStreamTest.cs | 197 ++++++++++++++++++ 8 files changed, 218 insertions(+), 6 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs diff --git a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs index bc6d66ecae..53a628fc53 100644 --- a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs +++ b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs @@ -1994,7 +1994,7 @@ override public TextReader GetTextReader(int i) { encoding = _metaData[i].encoding; } - + encoding = mt.SqlDbType == SqlDbTypeExtensions.Json ? System.Text.Encoding.UTF8 : encoding; _currentTextReader = new SqlSequentialTextReader(this, i, encoding); _lastColumnWithDataChunkRead = i; return _currentTextReader; diff --git a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs index 09cf542fdf..fcad6a4b11 100644 --- a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -11913,7 +11913,7 @@ private async Task WriteTextFeed(TextDataFeed feed, Encoding encoding, bool need encoding = encoding ?? TextDataFeed.DefaultEncoding; - using (ConstrainedTextWriter writer = new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size)) + using (ConstrainedTextWriter writer = encoding == Encoding.UTF8 ? new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null)), size) : new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size)) { if (needBom) { @@ -12177,7 +12177,8 @@ private Task WriteUnterminatedValue(object value, MetaType type, byte scale, int } else { - return NullIfCompletedWriteTask(WriteTextFeed(tdf, null, IsBOMNeeded(type, value), stateObj, paramSize)); + Encoding encoding = type.NullableType == TdsEnums.SQLJSON ? Encoding.UTF8 : null; + return NullIfCompletedWriteTask(WriteTextFeed(tdf, encoding, IsBOMNeeded(type, value), stateObj, paramSize)); } } else diff --git a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs index 617e1a3b57..743fd7092a 100644 --- a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs +++ b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs @@ -2282,7 +2282,7 @@ override public TextReader GetTextReader(int i) { encoding = _metaData[i].encoding; } - + encoding = mt.SqlDbType == SqlDbTypeExtensions.Json ? System.Text.Encoding.UTF8 : encoding; _currentTextReader = new SqlSequentialTextReader(this, i, encoding); _lastColumnWithDataChunkRead = i; return _currentTextReader; diff --git a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs index d540b73dc6..19dcfa4d28 100644 --- a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -12910,7 +12910,7 @@ private async Task WriteTextFeed(TextDataFeed feed, Encoding encoding, bool need char[] inBuff = new char[constTextBufferSize]; encoding = encoding ?? new UnicodeEncoding(false, false); - ConstrainedTextWriter writer = new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size); + ConstrainedTextWriter writer = encoding == Encoding.UTF8 ? new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null)), size) : new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size); if (needBom) { @@ -13169,7 +13169,8 @@ private Task WriteUnterminatedValue(object value, MetaType type, byte scale, int } else { - return NullIfCompletedWriteTask(WriteTextFeed(tdf, null, IsBOMNeeded(type, value), stateObj, paramSize)); + Encoding encoding = type.NullableType == TdsEnums.SQLJSON ? Encoding.UTF8 : null; + return NullIfCompletedWriteTask(WriteTextFeed(tdf, encoding, IsBOMNeeded(type, value), stateObj, paramSize)); } } else diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs index 0f8da0c874..318df52a0d 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs @@ -123,6 +123,7 @@ private static bool _IsCharType(SqlDbType type) => type == SqlDbType.Char || type == SqlDbType.VarChar || type == SqlDbType.Text || + type == SqlDbTypeExtensions.Json || type == SqlDbType.Xml; private static bool _IsNCharType(SqlDbType type) => diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs index 9da517b8b6..0076005aad 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs @@ -590,6 +590,17 @@ public static string GetUniqueNameForSqlServer(string prefix, bool withBracket = return name; } + public static void CreateTable(SqlConnection sqlConnection, string tableName, string createBody) + { + DropTable(sqlConnection, tableName); + string tableCreate = "CREATE TABLE " + tableName + createBody; + using (SqlCommand command = sqlConnection.CreateCommand()) + { + command.CommandText = tableCreate; + command.ExecuteNonQuery(); + } + } + public static void DropTable(SqlConnection sqlConnection, string tableName) { ResurrectConnection(sqlConnection); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj index a5040219df..8bcd2dfd16 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj @@ -293,6 +293,7 @@ + diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs new file mode 100644 index 0000000000..e6c35c9b6e --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Data; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Xunit; +using Xunit.Abstractions; +using Newtonsoft.Json.Linq; + + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests +{ + public class JsonRecord + { + public int Id { get; set; } + public string Name { get; set; } + } + + public class JsonStreamTest + { + private readonly ITestOutputHelper _output; + private static readonly string _jsonFile = "randomRecords.json"; + private static readonly string _outputFile = "serverRecords.json"; + + public JsonStreamTest(ITestOutputHelper output) + { + _output = output; + } + + private void GenerateJsonFile(int noOfRecords, string filename) + { + DeleteFile(filename); + var random = new Random(); + var records = new List(); + int recordCount = noOfRecords; + + for (int i = 0; i < recordCount; i++) + { + records.Add(new JsonRecord + { + Id = i + 1, + Name = "𩸽json" + random.Next(1, noOfRecords), + }); + } + + string json = JsonConvert.SerializeObject(records, Formatting.Indented); + File.WriteAllText(filename, json); + Assert.True(File.Exists(filename)); + _output.WriteLine("Generated JSON file "+filename); + } + + private void CompareJsonFiles() + { + using (var stream1 = File.OpenText(_jsonFile)) + using (var stream2 = File.OpenText(_outputFile)) + using (var reader1 = new JsonTextReader(stream1)) + using (var reader2 = new JsonTextReader(stream2)) + { + var jToken1 = JToken.ReadFrom(reader1); + var jToken2 = JToken.ReadFrom(reader2); + Assert.True(JToken.DeepEquals(jToken1, jToken2)); + } + } + + private void PrintJsonDataToFile(SqlConnection connection) + { + DeleteFile(_outputFile); + using (SqlCommand command = new SqlCommand("SELECT [data] FROM [jsonTab]", connection)) + { + using (SqlDataReader reader = command.ExecuteReader(CommandBehavior.SequentialAccess)) + { + using (StreamWriter sw = new StreamWriter(_outputFile)) + { + while (reader.Read()) + { + char[] buffer = new char[4096]; + int charsRead = 0; + + using (TextReader data = reader.GetTextReader(0)) + { + do + { + charsRead = data.Read(buffer, 0, buffer.Length); + sw.Write(buffer, 0, charsRead); + + } while (charsRead > 0); + } + _output.WriteLine("Output written to " + _outputFile); + } + } + } + } + } + + private async Task PrintJsonDataToFileAsync(SqlConnection connection) + { + DeleteFile(_outputFile); + using (SqlCommand command = new SqlCommand("SELECT [data] FROM [jsonTab]", connection)) + { + using (SqlDataReader reader = await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess)) + { + using (StreamWriter sw = new StreamWriter(_outputFile)) + { + while (await reader.ReadAsync()) + { + char[] buffer = new char[4096]; + int charsRead = 0; + + using (TextReader data = reader.GetTextReader(0)) + { + do + { + charsRead = await data.ReadAsync(buffer, 0, buffer.Length); + await sw.WriteAsync(buffer, 0, charsRead); + + } while (charsRead > 0); + } + _output.WriteLine("Output written to file " + _outputFile); + } + } + } + } + } + + private void StreamJsonFileToServer(SqlConnection connection) + { + using (SqlCommand cmd = new SqlCommand("INSERT INTO [jsonTab] (data) VALUES (@jsondata)", connection)) + { + using (StreamReader jsonFile = File.OpenText(_jsonFile)) + { + cmd.Parameters.Add("@jsondata", Microsoft.Data.SqlDbTypeExtensions.Json, -1).Value = jsonFile; + cmd.ExecuteNonQuery(); + } + } + } + + private async Task StreamJsonFileToServerAsync(SqlConnection connection) + { + using (SqlCommand cmd = new SqlCommand("INSERT INTO [jsonTab] (data) VALUES (@jsondata)", connection)) + { + using (StreamReader jsonFile = File.OpenText(_jsonFile)) + { + cmd.Parameters.Add("@jsondata", Microsoft.Data.SqlDbTypeExtensions.Json, -1).Value = jsonFile; + await cmd.ExecuteNonQueryAsync(); + } + } + } + + private void DeleteFile(string filename) + { + if (File.Exists(filename)) + { + File.Delete(filename); + } + + } + + + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.IsJsonSupported))] + public void TestJsonStreaming() + { + GenerateJsonFile(10000, _jsonFile); + + using (SqlConnection connection = new SqlConnection(DataTestUtility.TCPConnectionString)) + { + connection.Open(); + DataTestUtility.CreateTable(connection, "jsonTab", "(data json)"); + StreamJsonFileToServer(connection); + PrintJsonDataToFile(connection); + CompareJsonFiles(); + DeleteFile(_jsonFile); + DeleteFile(_outputFile); + } + } + + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.IsJsonSupported))] + public async Task TestJsonStreamingAsync() + { + GenerateJsonFile(10000, _jsonFile); + + using (SqlConnection connection = new SqlConnection(DataTestUtility.TCPConnectionString)) + { + await connection.OpenAsync(); + DataTestUtility.CreateTable(connection, "jsonTab", "(data json)"); + await StreamJsonFileToServerAsync(connection); + await PrintJsonDataToFileAsync(connection); + CompareJsonFiles(); + DeleteFile(_jsonFile); + DeleteFile(_outputFile); + } + } + } +} + From 64832a21195e63e931179856f38105dfc68d5a7f Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Thu, 26 Sep 2024 23:17:33 +0530 Subject: [PATCH 2/2] Address review comments --- .../netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs | 8 ++++++-- .../netcore/src/Microsoft/Data/SqlClient/TdsParser.cs | 4 ++-- .../netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs | 8 ++++++-- .../netfx/src/Microsoft/Data/SqlClient/TdsParser.cs | 4 ++-- .../tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs | 4 ---- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs index 53a628fc53..35a8bacfe2 100644 --- a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs +++ b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs @@ -14,6 +14,7 @@ using System.IO; using System.Reflection; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; using System.Xml; @@ -1985,7 +1986,11 @@ override public TextReader GetTextReader(int i) } System.Text.Encoding encoding; - if (mt.IsNCharType) + if (mt.SqlDbType == SqlDbTypeExtensions.Json) + { + encoding = new UTF8Encoding(); + } + else if (mt.IsNCharType) { // NChar types always use unicode encoding = SqlUnicodeEncoding.SqlUnicodeEncodingInstance; @@ -1994,7 +1999,6 @@ override public TextReader GetTextReader(int i) { encoding = _metaData[i].encoding; } - encoding = mt.SqlDbType == SqlDbTypeExtensions.Json ? System.Text.Encoding.UTF8 : encoding; _currentTextReader = new SqlSequentialTextReader(this, i, encoding); _lastColumnWithDataChunkRead = i; return _currentTextReader; diff --git a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs index fcad6a4b11..f0811f268c 100644 --- a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -11913,7 +11913,7 @@ private async Task WriteTextFeed(TextDataFeed feed, Encoding encoding, bool need encoding = encoding ?? TextDataFeed.DefaultEncoding; - using (ConstrainedTextWriter writer = encoding == Encoding.UTF8 ? new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null)), size) : new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size)) + using (ConstrainedTextWriter writer = new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size)) { if (needBom) { @@ -12177,7 +12177,7 @@ private Task WriteUnterminatedValue(object value, MetaType type, byte scale, int } else { - Encoding encoding = type.NullableType == TdsEnums.SQLJSON ? Encoding.UTF8 : null; + Encoding encoding = type.NullableType == TdsEnums.SQLJSON ? new UTF8Encoding() : null; return NullIfCompletedWriteTask(WriteTextFeed(tdf, encoding, IsBOMNeeded(type, value), stateObj, paramSize)); } } diff --git a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs index 743fd7092a..e1bae15b5c 100644 --- a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs +++ b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs @@ -13,6 +13,7 @@ using System.IO; using System.Reflection; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; using System.Xml; @@ -2273,7 +2274,11 @@ override public TextReader GetTextReader(int i) } System.Text.Encoding encoding; - if (mt.IsNCharType) + if (mt.SqlDbType == SqlDbTypeExtensions.Json) + { + encoding = new UTF8Encoding(); + } + else if (mt.IsNCharType) { // NChar types always use unicode encoding = SqlUnicodeEncoding.SqlUnicodeEncodingInstance; @@ -2282,7 +2287,6 @@ override public TextReader GetTextReader(int i) { encoding = _metaData[i].encoding; } - encoding = mt.SqlDbType == SqlDbTypeExtensions.Json ? System.Text.Encoding.UTF8 : encoding; _currentTextReader = new SqlSequentialTextReader(this, i, encoding); _lastColumnWithDataChunkRead = i; return _currentTextReader; diff --git a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs index 19dcfa4d28..c98eeb8db4 100644 --- a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -12910,7 +12910,7 @@ private async Task WriteTextFeed(TextDataFeed feed, Encoding encoding, bool need char[] inBuff = new char[constTextBufferSize]; encoding = encoding ?? new UnicodeEncoding(false, false); - ConstrainedTextWriter writer = encoding == Encoding.UTF8 ? new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null)), size) : new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size); + ConstrainedTextWriter writer = new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size); if (needBom) { @@ -13169,7 +13169,7 @@ private Task WriteUnterminatedValue(object value, MetaType type, byte scale, int } else { - Encoding encoding = type.NullableType == TdsEnums.SQLJSON ? Encoding.UTF8 : null; + Encoding encoding = type.NullableType == TdsEnums.SQLJSON ? new UTF8Encoding() : null; return NullIfCompletedWriteTask(WriteTextFeed(tdf, encoding, IsBOMNeeded(type, value), stateObj, paramSize)); } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs index e6c35c9b6e..a82fee1665 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs @@ -155,15 +155,12 @@ private void DeleteFile(string filename) { File.Delete(filename); } - } - [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.IsJsonSupported))] public void TestJsonStreaming() { GenerateJsonFile(10000, _jsonFile); - using (SqlConnection connection = new SqlConnection(DataTestUtility.TCPConnectionString)) { connection.Open(); @@ -180,7 +177,6 @@ public void TestJsonStreaming() public async Task TestJsonStreamingAsync() { GenerateJsonFile(10000, _jsonFile); - using (SqlConnection connection = new SqlConnection(DataTestUtility.TCPConnectionString)) { await connection.OpenAsync();