Improved Json Serialization (#6152)

* System.Text.Json.Serialization * System.Text.Deserialization * Fixes * lint * Fixes * Fix * Fixes * Fixes * Lint * Fix invalid Json * u8 * Fixes * Fixes * Fix * Don't go via exception for short reverts * Fix * Fix * Multi-doc batch * Lint * Merge conflict * Fixes * Lint * Fix tests * Fix test * Tidy up * Turn of sync IO * Async * Don't double Advance * Async wrapper stream use * Fix build break * Merge issues * Compile issues * File scoped namespaces * Formatting * Merge fixes * Test Merge fixes * lang ver * Purge Newtonsoft * Fix evm tests * Fix parentBeaconBlockRootString serialization * Change type for serialization * Ignore syncing * Capitalised bools are invalid json * Capitalised bools are invalid json * Fix merge conflict * Fix data serialization * Fix trace entry * Merge conflicts * Formatting * Missed merge * whitespace * Fix EthereumTests Json * Ethereum tests * Fix hex prefix * Fix difficultly test * Add serializer options * Fix abi tests * Merge conflict * Feedback * revert to previous * Remove Netwonsoft * cosmetics * Pipeline Json responses * Merge fixes * Whitespace * fix test * Send multiple messages * Optional stream closing * Case insensitive * Update Nethermind.Serialization.Json.csproj * Whitespace * Update JavaScriptObjectConverter.cs * Feedback * Undo AI * Put orders back * Feedback * Whitespace * Use static constructor * Remove irrelevant test parameter * Add BigIntegerConverterTests back * Use FrozenDictionary * Deserialize collection to ArrayPoolList using GetArrayLength for initial capacity. Dispose it later. Move WhiteSpace to utf8 * simplify * Throw more explicit exception * name cleanup * Merge conflicts * Merge conflict * Add comments * more comments * Consolidate Disposable * Use FrozenDictionary for RpcLookup * Case sensitive * Be consistent on parsing type * Use correct type in test * Optimizations * Optimizations * Optimize * Optimize * Vectorize FromUtf8 hex * Optimization * Optimize * Parallel deserialize * Revert "Parallel deserialize" This reverts commit 94b08e3. * Use using * Fix check * Parallel deserialize * Increase parallelThreshold * Use correct length * Cache Reflection Parameters * Fix tracing output * Match geth output * Turn AllowSynchronousIO back on * Revert object type arrays --------- Co-authored-by: lukasz.rozmej <lukasz.rozmej@gmail.com>
NethermindEth · Dec 11, 2023 · 82b816a · 82b816a
1 parent 3f2f149
commit 82b816a
Show file tree

Hide file tree

Showing 289 changed files with 8,510 additions and 6,291 deletions.
diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props
@@ -53,7 +53,6 @@
     <PackageVersion Include="Nethermind.DotNetty.Transport" Version="1.0.1" />
     <PackageVersion Include="Nethermind.Gmp" Version="1.0.1" />
     <PackageVersion Include="Nethermind.Numerics.Int256" Version="1.1.1" />
-    <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="Nito.Collections.Deque" Version="1.2.1" />
     <PackageVersion Include="NLog" Version="5.2.7" />
     <PackageVersion Include="NLog.Targets.Seq" Version="3.0.0" />
@@ -79,6 +78,7 @@
     <PackageVersion Include="System.Diagnostics.TextWriterTraceListener" Version="4.3.0" />
     <PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="7.0.3" />
     <PackageVersion Include="System.IO.Compression.ZipFile" Version="4.3.0" />
+    <PackageVersion Include="System.IO.Pipelines" Version="8.0.0" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Net.Http" Version="4.3.4" />
     <PackageVersion Include="System.Runtime.Caching" Version="8.0.0" />

diff --git a/src/Nethermind/Ethereum.Abi.Test/AbiTest.cs b/src/Nethermind/Ethereum.Abi.Test/AbiTest.cs
@@ -1,19 +1,19 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace Ethereum.Abi.Test
 {
     public class AbiTest
     {
-        [JsonProperty(PropertyName = "args")]
+        [JsonPropertyName("args")]
         public object[] Args { get; set; }
 
-        [JsonProperty(PropertyName = "result")]
+        [JsonPropertyName("result")]
         public string Result { get; set; }
 
-        [JsonProperty(PropertyName = "types")]
+        [JsonPropertyName("types")]
         public string[] Types { get; set; }
     }
 }
diff --git a/src/Nethermind/Ethereum.Abi.Test/Tests.cs b/src/Nethermind/Ethereum.Abi.Test/Tests.cs
@@ -5,11 +5,12 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
 using FluentAssertions;
 using Nethermind.Abi;
 using Nethermind.Core.Extensions;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
 using NUnit.Framework;
 
 namespace Ethereum.Abi.Test
@@ -56,7 +57,7 @@ public void Test_abi_encoding()
                 }
             }
 
-            Dictionary<string, AbiTest> tests = JsonConvert.DeserializeObject<Dictionary<string, AbiTest>>(text);
+            Dictionary<string, AbiTest> tests = JsonSerializer.Deserialize<Dictionary<string, AbiTest>>(text);
             foreach ((string testName, AbiTest abiTest) in tests)
             {
                 AbiSignature signature = new(
@@ -71,9 +72,9 @@ public void Test_abi_encoding()
 
         public object JsonToObject(object jsonObject)
         {
-            if (jsonObject is JArray array)
+            if (jsonObject is JsonArray array)
             {
-                return array.Select(t => t.Value<long>()).ToArray();
+                return array.Select(t => t.GetValue<long>()).ToArray();
             }
 
             return jsonObject;

diff --git a/src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs b/src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs
@@ -7,6 +7,8 @@
 using System.IO;
 using System.Linq;
 using System.Numerics;
+using System.Text.Json.Serialization;
+
 using Ethereum.Test.Base;
 using Nethermind.Core;
 using Nethermind.Core.Extensions;

diff --git a/...ermind/Ethereum.Blockchain.Block.Legacy.Test/Ethereum.Blockchain.Block.Legacy.Test.csproj b/...ermind/Ethereum.Blockchain.Block.Legacy.Test/Ethereum.Blockchain.Block.Legacy.Test.csproj
@@ -10,7 +10,6 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="NUnit" />
     <PackageReference Include="NUnit.Analyzers">
       <PrivateAssets>all</PrivateAssets>

diff --git a/src/Nethermind/Ethereum.Blockchain.Legacy.Test/Ethereum.Blockchain.Legacy.Test.csproj b/src/Nethermind/Ethereum.Blockchain.Legacy.Test/Ethereum.Blockchain.Legacy.Test.csproj
@@ -15,7 +15,6 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="NUnit3TestAdapter" />
   </ItemGroup>
 

diff --git a/src/Nethermind/Ethereum.Blockchain.Test/TransactionJsonTest.cs b/src/Nethermind/Ethereum.Blockchain.Test/TransactionJsonTest.cs
@@ -19,7 +19,7 @@ public class TransactionJsonTest : GeneralStateTestBase
         public void Can_load_access_lists()
         {
             const string lists =
-                "{\"accessLists\": [[{address: \"0x0001020304050607080900010203040506070809\", storageKeys: [\"0x00\", \"0x01\"]}]]}";
+                "{\"accessLists\": [[{\"address\": \"0x0001020304050607080900010203040506070809\", \"storageKeys\": [\"0x00\", \"0x01\"]}]]}";
 
             EthereumJsonSerializer serializer = new EthereumJsonSerializer();
             TransactionJson txJson = serializer.Deserialize<TransactionJson>(lists);

diff --git a/src/Nethermind/Ethereum.Difficulty.Test/DifficultyTestHexJson.cs b/src/Nethermind/Ethereum.Difficulty.Test/DifficultyTestHexJson.cs
@@ -1,6 +1,8 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Text.Json.Serialization;
+
 namespace Ethereum.Difficulty.Test
 {
     public class DifficultyTestHexJson

diff --git a/src/Nethermind/Ethereum.Difficulty.Test/DifficultyTestJson.cs b/src/Nethermind/Ethereum.Difficulty.Test/DifficultyTestJson.cs
@@ -1,6 +1,8 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Text.Json.Serialization;
+
 namespace Ethereum.Difficulty.Test
 {
     public class DifficultyTestJson

diff --git a/src/Nethermind/Ethereum.HexPrefix.Test/HexPrefixTests.cs b/src/Nethermind/Ethereum.HexPrefix.Test/HexPrefixTests.cs
@@ -3,8 +3,11 @@
 
 using System.Collections.Generic;
 using System.Linq;
+using System.Text.Json.Serialization;
+
 using Ethereum.Test.Base;
 using Nethermind.Core.Extensions;
+
 using NUnit.Framework;
 
 namespace Ethereum.HexPrefix.Test
@@ -18,7 +21,7 @@ public static IEnumerable<HexPrefixTest> LoadTests()
         {
             return TestLoader.LoadFromFile<Dictionary<string, HexPrefixTestJson>, HexPrefixTest>(
                 "hexencodetest.json",
-                c => c.Select(p => new HexPrefixTest(p.Key, p.Value.Seq, p.Value.Term, p.Value.Out)));
+                c => c.Select(p => new HexPrefixTest(p.Key, p.Value.Seq.Select(x => (byte)x).ToArray(), p.Value.Term, p.Value.Out)));
         }
 
         [TestCaseSource(nameof(LoadTests))]
@@ -36,7 +39,7 @@ public void Test(HexPrefixTest test)
 
         private class HexPrefixTestJson
         {
-            public byte[] Seq { get; set; }
+            public int[] Seq { get; set; }
             public bool Term { get; set; }
             public string Out { get; set; }
         }

diff --git a/src/Nethermind/Ethereum.KeyAddress.Test/KeyAddressTests.cs b/src/Nethermind/Ethereum.KeyAddress.Test/KeyAddressTests.cs
@@ -5,13 +5,14 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.Json.Serialization;
+
 using Ethereum.Test.Base;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Crypto;
 using Nethermind.Int256;
 using Nethermind.Logging;
-using Newtonsoft.Json;
 using NUnit.Framework;
 
 namespace Ethereum.KeyAddress.Test
@@ -96,7 +97,7 @@ private class SigOfEmptyString
 
         private class KeyAddressTestJson
         {
-            [JsonProperty("sig_of_emptystring")]
+            [JsonPropertyName("sig_of_emptystring")]
             public SigOfEmptyString Signature { get; set; }
 
             public string Seed { get; set; }

diff --git a/src/Nethermind/Ethereum.KeyStore.Test/KeyStoreJsonTests.cs b/src/Nethermind/Ethereum.KeyStore.Test/KeyStoreJsonTests.cs
@@ -4,6 +4,8 @@
 using System;
 using System.IO;
 using System.Security;
+using System.Text.Json.Serialization;
+
 using Nethermind.Core;
 using Nethermind.Core.Collections;
 using Nethermind.Crypto;
@@ -12,7 +14,7 @@
 using Nethermind.KeyStore.Config;
 using Nethermind.Logging;
 using Nethermind.Serialization.Json;
-using Newtonsoft.Json;
+
 using NUnit.Framework;
 
 namespace Ethereum.KeyStore.Test
@@ -129,7 +131,7 @@ private class KeyStoreTestsModel
 
         private class KeyStoreTestModel
         {
-            [JsonProperty(PropertyName = "Json")]
+            [JsonPropertyName("Json")]
             public KeyStoreItem KeyData { get; set; }
             public string Password { get; set; }
             public string Priv { get; set; }

diff --git a/src/Nethermind/Ethereum.PoW.Test/EthashTests.cs b/src/Nethermind/Ethereum.PoW.Test/EthashTests.cs
@@ -7,6 +7,8 @@
 using System.IO;
 using System.Linq;
 using System.Numerics;
+using System.Text.Json.Serialization;
+
 using Ethereum.Test.Base;
 using Nethermind.Consensus.Ethash;
 using Nethermind.Core;
@@ -15,7 +17,6 @@
 using Nethermind.Crypto;
 using Nethermind.Logging;
 using Nethermind.Serialization.Rlp;
-using Newtonsoft.Json;
 using NUnit.Framework;
 
 namespace Ethereum.PoW.Test
@@ -103,16 +104,16 @@ private class EthashTestJson
             public string Header { get; set; }
             public string Seed { get; set; }
 
-            [JsonProperty("cache_size")]
+            [JsonPropertyName("cache_size")]
             public int CacheSize { get; set; }
 
-            [JsonProperty("full_size")]
+            [JsonPropertyName("full_size")]
             public int FullSize { get; set; }
 
-            [JsonProperty("header_hash")]
+            [JsonPropertyName("header_hash")]
             public string HeaderHash { get; set; }
 
-            [JsonProperty("cache_hash")]
+            [JsonPropertyName("cache_hash")]
             public string CacheHash { get; set; }
 
             public string Result { get; set; }

diff --git a/src/Nethermind/Ethereum.Test.Base/AccessListJson.cs b/src/Nethermind/Ethereum.Test.Base/AccessListJson.cs
@@ -2,16 +2,13 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using Nethermind.Core;
-using Newtonsoft.Json;
 
 namespace Ethereum.Test.Base
 {
     public class AccessListItemJson
     {
-        [JsonProperty("address")]
         public Address Address { get; set; }
 
-        [JsonProperty("storageKeys")]
         public byte[][] StorageKeys { get; set; }
     }
 }
diff --git a/src/Nethermind/Ethereum.Test.Base/EthereumTestResult.cs b/src/Nethermind/Ethereum.Test.Base/EthereumTestResult.cs
@@ -1,8 +1,9 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Text.Json.Serialization;
+
 using Nethermind.Core.Crypto;
-using Newtonsoft.Json;
 
 namespace Ethereum.Test.Base
 {

diff --git a/src/Nethermind/Ethereum.Test.Base/GeneralStateTestJson.cs b/src/Nethermind/Ethereum.Test.Base/GeneralStateTestJson.cs
@@ -2,13 +2,13 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System.Collections.Generic;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace Ethereum.Test.Base
 {
     public class GeneralStateTestJson
     {
-        [JsonProperty("_info")]
+        [JsonPropertyName("_info")]
         public GeneralStateTestInfoJson? Info { get; set; }
         public GeneralStateTestEnvJson? Env { get; set; }
         public Dictionary<string, PostStateJson[]>? Post { get; set; }

diff --git a/src/Nethermind/Ethereum.Test.Base/TestBlockJson.cs b/src/Nethermind/Ethereum.Test.Base/TestBlockJson.cs
@@ -1,7 +1,8 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
-using Newtonsoft.Json;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 
 namespace Ethereum.Test.Base
 {
@@ -11,7 +12,7 @@ public class TestBlockJson
         public TestBlockHeaderJson[]? UncleHeaders { get; set; }
         public string? Rlp { get; set; }
         public LegacyTransactionJson[]? Transactions { get; set; }
-        [JsonProperty("expectException")]
+        [JsonPropertyName("expectException")]
         public string? ExpectedException { get; set; }
     }
 }
diff --git a/src/Nethermind/Ethereum.Test.Base/TestLoader.cs b/src/Nethermind/Ethereum.Test.Base/TestLoader.cs
@@ -3,12 +3,14 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Numerics;
 using System.Reflection;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
 using NUnit.Framework;
 
 namespace Ethereum.Test.Base
@@ -17,30 +19,33 @@ public static class TestLoader
     {
         public static object PrepareInput(object input)
         {
-            string s = input as string;
-            if (s != null && s.StartsWith("#"))
+            if (input is string s && s.StartsWith("#"))
             {
                 BigInteger bigInteger = BigInteger.Parse(s.Substring(1));
                 input = bigInteger;
             }
 
-            if (input is JArray)
-            {
-                input = ((JArray)input).Select(PrepareInput).ToArray();
-            }
+            JsonElement token = (JsonElement)input;
 
-            JToken token = input as JToken;
-            if (token != null)
+            if (token.ValueKind == JsonValueKind.Array)
             {
-                if (token.Type == JTokenType.String)
+                object[] array = new object[token.GetArrayLength()];
+                for (int i = 0; i < array.Length; i++)
                 {
-                    return token.Value<string>();
+                    array[i] = PrepareInput(token[i]);
                 }
 
-                if (token.Type == JTokenType.Integer)
-                {
-                    return token.Value<long>();
-                }
+                input = array;
+            }
+
+            if (token.ValueKind == JsonValueKind.String)
+            {
+                return token.GetString()!;
+            }
+
+            if (token.ValueKind == JsonValueKind.Number)
+            {
+                return token.GetInt64();
             }
 
             return input;
@@ -58,14 +63,21 @@ public static IEnumerable<TTest> LoadFromFile<TContainer, TTest>(
                 throw new ArgumentException($"Cannot find test resource: {testFileName}");
             }
 
+            var jsonOptions = new JsonSerializerOptions()
+            {
+                PropertyNameCaseInsensitive = true,
+                PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+                DictionaryKeyPolicy = JsonNamingPolicy.CamelCase,
+                NumberHandling = JsonNumberHandling.AllowReadingFromString
+            };
             using (Stream stream = assembly.GetManifestResourceStream(resourceName))
             {
                 Assert.NotNull(stream);
                 using (StreamReader reader = new(stream))
                 {
                     string testJson = reader.ReadToEnd();
                     TContainer testSpecs =
-                        JsonConvert.DeserializeObject<TContainer>(testJson);
+                        JsonSerializer.Deserialize<TContainer>(testJson, jsonOptions);
                     return testExtractor(testSpecs);
                 }
             }