From b0e8c2c406566448ef25125f0a503350d62addc5 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 16 Oct 2024 23:35:32 +0800 Subject: [PATCH] Perf/parallelize storage commit (#7605) Co-authored-by: Lukasz Rozmej --- .../Ethereum.Trie.Test/TrieTests.cs | 2 +- .../Store/PatriciaTreeBenchmarks.cs | 56 ++-- .../Builders/TestItem.Tree.cs | 12 +- .../Builders/TrieBuilder.cs | 2 +- .../Eip1186/ProofConverterTests.cs | 8 +- .../PatriciaTreeTests.cs | 14 +- .../Proofs/AccountProofCollectorTests.cs | 58 ++-- .../RecreateStateFromAccountRangesTests.cs | 2 +- .../Nethermind.State.Test/StateTreeTests.cs | 56 ++-- .../PersistentStorageProvider.cs | 24 +- .../Nethermind.State/StateProvider.cs | 4 +- src/Nethermind/Nethermind.State/WorldState.cs | 7 +- .../FastSync/StateSyncFeedHealingTests.cs | 4 +- .../FastSync/StateSyncFeedTests.cs | 26 +- .../FastSync/StateSyncFeedTestsBase.cs | 2 +- .../RangeQueryVisitorTests.cs | 8 +- .../RecreateStateFromAccountRangesTests.cs | 4 +- .../SnapSync/SnapServerTest.cs | 6 +- .../SyncServerTests.cs | 9 +- .../TrieScenarios.cs | 60 ++-- .../SnapSync/SnapProviderHelper.cs | 4 +- .../IFullTrieStoreExtensions.cs | 37 ++ .../Pruning/TreeStoreTests.cs | 261 +++++++------- .../Nethermind.Trie.Test/TrieNodeTests.cs | 12 +- .../Nethermind.Trie.Test/TrieTests.cs | 97 +++--- .../Nethermind.Trie.Test/VisitingTests.cs | 9 +- .../Nethermind.Trie/CachedTrieStore.cs | 4 +- .../Nethermind.Trie/PatriciaTree.cs | 19 +- .../Nethermind.Trie/PreCachedTrieStore.cs | 10 +- .../Pruning/BlockCommitPackage.cs | 20 ++ .../Pruning/IScopedTrieStore.cs | 5 +- .../Nethermind.Trie/Pruning/ITrieStore.cs | 41 ++- .../Nethermind.Trie/Pruning/NullCommitter.cs | 21 ++ .../Nethermind.Trie/Pruning/NullTrieStore.cs | 10 +- .../Pruning/ReadOnlyTrieStore.cs | 11 +- .../Pruning/ScopedTrieStore.cs | 4 +- .../Nethermind.Trie/Pruning/TrieStore.cs | 317 +++++++++--------- .../Nethermind.Trie/TrieStoreWithReadFlags.cs | 4 +- 38 files changed, 680 insertions(+), 570 deletions(-) create mode 100644 src/Nethermind/Nethermind.Trie.Test/IFullTrieStoreExtensions.cs create mode 100644 src/Nethermind/Nethermind.Trie/Pruning/NullCommitter.cs diff --git a/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs b/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs index ab7f94d6220..aedee4459aa 100644 --- a/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs +++ b/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs @@ -307,7 +307,7 @@ public void Delete_on_empty() { PatriciaTree patriciaTree = new PatriciaTree(_db, Keccak.EmptyTreeHash, false, true, NullLogManager.Instance); patriciaTree.Set(Keccak.Compute("1").Bytes, new byte[0]); - patriciaTree.Commit(0); + patriciaTree.Commit(); Assert.That(patriciaTree.RootHash, Is.EqualTo(PatriciaTree.EmptyTreeHash)); } diff --git a/src/Nethermind/Nethermind.Benchmark/Store/PatriciaTreeBenchmarks.cs b/src/Nethermind/Nethermind.Benchmark/Store/PatriciaTreeBenchmarks.cs index 5d568377914..07d700e9352 100644 --- a/src/Nethermind/Nethermind.Benchmark/Store/PatriciaTreeBenchmarks.cs +++ b/src/Nethermind/Nethermind.Benchmark/Store/PatriciaTreeBenchmarks.cs @@ -56,14 +56,14 @@ public class PatriciaTreeBenchmarks tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(1); + tree.Commit(); }), ("set_3_via_hash", tree => { tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); - tree.Commit(1); + tree.Commit(); }), ("set_3_delete_1", tree => { @@ -71,7 +71,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(1); + tree.Commit(); }), ("set_3_delete_2", tree => { @@ -80,7 +80,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(1); + tree.Commit(); }), ("set_3_delete_all", tree => { @@ -90,7 +90,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), null); - tree.Commit(1); + tree.Commit(); }), ("extension_read_full_match", tree => { @@ -99,7 +99,7 @@ public class PatriciaTreeBenchmarks Account account = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111")); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("extension_read_missing", tree => { @@ -108,7 +108,7 @@ public class PatriciaTreeBenchmarks Account account = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd")); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("extension_new_branch", tree => { @@ -117,7 +117,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"), _account2); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("extension_delete_missing", tree => { @@ -126,7 +126,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeddddddddddddddddddddddddd"), null); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("extenson_create_new_extension", tree => { @@ -136,7 +136,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeaaaaaaaaaaaaaaaab11111111"), _account3); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_new_value", tree => { @@ -144,7 +144,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account1); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_no_change", tree => { @@ -152,7 +152,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account0); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_delete", tree => { @@ -160,7 +160,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), null); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_delete_missing", tree => { @@ -168,7 +168,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("1111111111111111111111111111111ddddddddddddddddddddddddddddddddd"), null); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_update_extension", tree => { @@ -176,7 +176,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000000000000000000000000000"), _account1); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_read", tree => { @@ -184,7 +184,7 @@ public class PatriciaTreeBenchmarks Account account = tree.Get(new Hash256("1111111111111111111111111111111111111111111111111111111111111111")); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_update_missing", tree => { @@ -192,7 +192,7 @@ public class PatriciaTreeBenchmarks Account account = tree.Get(new Hash256("111111111111111111111111111111111111111111111111111111111ddddddd")); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("branch_update_missing", tree => { @@ -201,7 +201,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), _account2); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("branch_read_missing", tree => { @@ -210,7 +210,7 @@ public class PatriciaTreeBenchmarks Account account = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222")); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("branch_delete_missing", tree => { @@ -219,7 +219,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), null); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), }; @@ -247,7 +247,7 @@ public void Setup() { tempTree.Set(_entries[i].Item1, _entries[i].Item2); } - tempTree.Commit(0); + tempTree.Commit(); _rootHash = tempTree.RootHash; _fullTree = new StateTree(); @@ -255,7 +255,7 @@ public void Setup() { _fullTree.Set(_entries[i].Item1, _entries[i].Item2); } - _fullTree.Commit(0); + _fullTree.Commit(); _uncommittedFullTree = new StateTree(); for (int i = 0; i < _entryCount; i++) @@ -330,23 +330,23 @@ public void InsertAndCommit() { tempTree.Set(_entries[i].Item1, _entries[i].Item2); } - tempTree.Commit(0); + tempTree.Commit(); } [Benchmark] public void InsertAndCommitRepeatedlyTimes() { - StateTree tempTree = new StateTree( - new TrieStore(new MemDb(), + TrieStore trieStore = new TrieStore(new MemDb(), Prune.WhenCacheReaches(1.MiB()), - Persist.IfBlockOlderThan(2), - NullLogManager.Instance), NullLogManager.Instance); + Persist.IfBlockOlderThan(2), NullLogManager.Instance); + StateTree tempTree = new StateTree(trieStore, NullLogManager.Instance); for (int i = 0; i < _largerEntryCount; i++) { if (i % 2000 == 0) { - tempTree.Commit(i / 2000); + using IBlockCommitter _ = trieStore.BeginBlockCommit(i / 2000); + tempTree.Commit(); } (bool isWrite, Hash256 address, Account value) = _largerEntriesAccess[i]; diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs b/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs index aa440794481..64b9d22d833 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs @@ -66,7 +66,7 @@ public static void FillStateTreeWithTestAccounts(StateTree stateTree) stateTree.Set(AccountsWithPaths[3].Path, AccountsWithPaths[3].Account); stateTree.Set(AccountsWithPaths[4].Path, AccountsWithPaths[4].Account); stateTree.Set(AccountsWithPaths[5].Path, AccountsWithPaths[5].Account); - stateTree.Commit(0); + stateTree.Commit(); } public static void FillStateTreeMultipleAccount(StateTree stateTree, int accountNumber) @@ -76,7 +76,7 @@ public static void FillStateTreeMultipleAccount(StateTree stateTree, int account Account acc = Build.An.Account.WithBalance((UInt256)i).TestObject; stateTree.Set(Keccak.Compute(i.ToBigEndianByteArray()), acc); } - stateTree.Commit(0); + stateTree.Commit(); } public static (StateTree stateTree, StorageTree storageTree, Hash256 accountAddr) GetTrees(ITrieStore? store) @@ -92,13 +92,13 @@ public static (StateTree stateTree, StorageTree storageTree, Hash256 accountAddr storageTree.Set(SlotsWithPaths[4].Path, SlotsWithPaths[4].SlotRlpValue, false); storageTree.Set(SlotsWithPaths[5].Path, SlotsWithPaths[5].SlotRlpValue, false); - storageTree.Commit(0); + storageTree.Commit(); var account = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; var stateTree = new StateTree(store.GetTrieStore(null), LimboLogs.Instance); stateTree.Set(AccountAddress0, account); - stateTree.Commit(0); + stateTree.Commit(); return (stateTree, storageTree, AccountAddress0); } @@ -117,13 +117,13 @@ public static (StateTree stateTree, StorageTree storageTree, Hash256 accountAddr false); } - storageTree.Commit(0); + storageTree.Commit(); var account = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; var stateTree = new StateTree(store, LimboLogs.Instance); stateTree.Set(AccountAddress0, account); - stateTree.Commit(0); + stateTree.Commit(); return (stateTree, storageTree, AccountAddress0); } diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/TrieBuilder.cs b/src/Nethermind/Nethermind.Core.Test/Builders/TrieBuilder.cs index 691f2debad1..3f2bdcf9688 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/TrieBuilder.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/TrieBuilder.cs @@ -36,7 +36,7 @@ public TrieBuilder WithAccountsByIndex(int start, int count) TestObjectInternal.Set(key.Bytes, value); } - TestObjectInternal.Commit(0); + TestObjectInternal.Commit(); TestObjectInternal.UpdateRootHash(); return this; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Eip1186/ProofConverterTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Eip1186/ProofConverterTests.cs index cb0b0637829..986fd030146 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Eip1186/ProofConverterTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Eip1186/ProofConverterTests.cs @@ -38,14 +38,14 @@ public void Storage_proofs_have_values_set_complex_3_setup() storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA, new byte[][] { a, b, c, d, e }); tree.Accept(accountProofCollector, tree.RootHash); @@ -72,14 +72,14 @@ public void Does_not_fail_when_proofs_are_longer_than_number_of_proofs_regressio storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA, new byte[][] { a }); tree.Accept(accountProofCollector, tree.RootHash); diff --git a/src/Nethermind/Nethermind.State.Test/PatriciaTreeTests.cs b/src/Nethermind/Nethermind.State.Test/PatriciaTreeTests.cs index 9f1108a9f33..0dd057454aa 100644 --- a/src/Nethermind/Nethermind.State.Test/PatriciaTreeTests.cs +++ b/src/Nethermind/Nethermind.State.Test/PatriciaTreeTests.cs @@ -24,11 +24,11 @@ public void Create_commit_change_balance_get() Account account = new(1); StateTree stateTree = new(); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); account = account.WithChangedBalance(2); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); Account accountRestored = stateTree.Get(TestItem.AddressA); Assert.That(accountRestored.Balance, Is.EqualTo((UInt256)2)); @@ -41,11 +41,11 @@ public void Create_create_commit_change_balance_get() StateTree stateTree = new(); stateTree.Set(TestItem.AddressA, account); stateTree.Set(TestItem.AddressB, account); - stateTree.Commit(0); + stateTree.Commit(); account = account.WithChangedBalance(2); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); Account accountRestored = stateTree.Get(TestItem.AddressA); Assert.That(accountRestored.Balance, Is.EqualTo((UInt256)2)); @@ -58,7 +58,7 @@ public void Create_commit_reset_change_balance_get() Account account = new(1); StateTree stateTree = new(new TrieStore(db, LimboLogs.Instance), LimboLogs.Instance); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); Hash256 rootHash = stateTree.RootHash; stateTree.RootHash = null; @@ -67,7 +67,7 @@ public void Create_commit_reset_change_balance_get() stateTree.Get(TestItem.AddressA); account = account.WithChangedBalance(2); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); Assert.That(db.Keys.Count, Is.EqualTo(2)); } @@ -84,7 +84,7 @@ public void Commit_with_skip_root_should_skip_root(bool skipRoot, bool hasRoot) stateTree.Set(TestItem.AddressA, account); stateTree.UpdateRootHash(); Hash256 stateRoot = stateTree.RootHash; - stateTree.Commit(0, skipRoot); + stateTree.Commit(skipRoot); if (hasRoot) { diff --git a/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs b/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs index 2e82c29714a..503368f463a 100644 --- a/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs +++ b/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs @@ -53,7 +53,7 @@ public void Non_existing_account_is_valid_on_non_empty_tree_with_branch_without_ Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressC, new UInt256[] { 1, 2, 3 }); tree.Accept(accountProofCollector, tree.RootHash); @@ -75,7 +75,7 @@ public void Non_existing_account_is_valid_even_when_leaf_is_the_last_part_of_the Account account1 = Build.An.Account.WithBalance(1).TestObject; tree.Set(TestItem.AddressA, account1); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressC, new UInt256[] { 1, 2, 3 }); tree.Accept(accountProofCollector, tree.RootHash); @@ -106,7 +106,7 @@ public void Non_existing_account_is_valid_even_when_extension_on_the_way_is_not_ Account account = Build.An.Account.WithBalance(1).TestObject; tree.Set(c.AsSpan(), Rlp.Encode(account.WithChangedBalance(3))); tree.Set(d.AsSpan(), Rlp.Encode(account.WithChangedBalance(4))); - tree.Commit(0); + tree.Commit(); // now wer are looking for a trying to trick the code to think that the extension of c and d is a good match // if everything is ok the proof length of 1 is enough since the extension from the root is not matched @@ -128,7 +128,7 @@ public void Addresses_are_correct() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -150,7 +150,7 @@ public void Balance_is_correct() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -173,7 +173,7 @@ public void Code_hash_is_correct() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -196,7 +196,7 @@ public void Nonce_is_correct() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -219,7 +219,7 @@ public void Storage_root_is_correct() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -242,7 +242,7 @@ public void Proof_path_is_filled() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -260,7 +260,7 @@ public void Storage_proofs_length_is_as_expected() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA, new[] { Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"), Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000001") }); tree.Accept(accountProofCollector, tree.RootHash); @@ -277,14 +277,14 @@ public void Storage_proofs_have_values_set() StorageTree storageTree = new(trieStore.GetTrieStore(TestItem.AddressA.ToAccountPath), Keccak.EmptyTreeHash, LimboLogs.Instance); storageTree.Set(UInt256.Zero, Bytes.FromHexString("0xab12000000000000000000000000000000000000000000000000000000000000000000000000000000")); storageTree.Set(UInt256.One, Bytes.FromHexString("0xab34000000000000000000000000000000000000000000000000000000000000000000000000000000")); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA, new[] { Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"), Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000001") }); tree.Accept(accountProofCollector, tree.RootHash); @@ -302,14 +302,14 @@ public void Storage_proofs_have_keys_set() StorageTree storageTree = new(trieStore.GetTrieStore(TestItem.AddressA.ToAccountPath), Keccak.EmptyTreeHash, LimboLogs.Instance); storageTree.Set(UInt256.Zero, Bytes.FromHexString("0xab12000000000000000000000000000000000000000000000000000000000000000000000000000000")); storageTree.Set(UInt256.One, Bytes.FromHexString("0xab34000000000000000000000000000000000000000000000000000000000000000000000000000000")); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA, new[] { Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"), Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000001") }); tree.Accept(accountProofCollector, tree.RootHash); @@ -332,14 +332,14 @@ public void Storage_proofs_have_values_set_complex_setup() storageTree.Set(Keccak.Compute(a).Bytes, Rlp.Encode(Bytes.FromHexString("0xab12000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(b).Bytes, Rlp.Encode(Bytes.FromHexString("0xab34000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -371,14 +371,14 @@ public void Storage_proofs_have_values_set_complex_2_setup() storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -412,14 +412,14 @@ public void Storage_proofs_have_values_set_complex_3_setup() storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -451,14 +451,14 @@ public void Storage_proofs_when_values_are_missing_setup() storageTree.Set(Keccak.Compute(a).Bytes, Rlp.Encode(Bytes.FromHexString("0xab12000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -486,7 +486,7 @@ public void Shows_empty_values_when_account_is_missing() byte[] code = new byte[] { 1, 2, 3 }; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -519,14 +519,14 @@ public void Storage_proofs_have_values_set_selective_setup() storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -629,14 +629,14 @@ public void _Test_storage_failed_case(string historicallyFailingCase) TestContext.Out.WriteLine($"Set {Keccak.Compute(rawKey).Bytes.ToHexString()}"); storageTree.Set(addressWithStorage.StorageCells[j].Index, new byte[] { 1 }); storageTree.UpdateRootHash(); - storageTree.Commit(0); + storageTree.Commit(); } Account account = Build.An.Account.WithBalance((UInt256)accountIndex).WithStorageRoot(storageTree.RootHash).TestObject; tree.Set(addressWithStorage.Address, account); tree.UpdateRootHash(); - tree.Commit(0); + tree.Commit(); TreeDumper treeDumper = new(); tree.Accept(treeDumper, tree.RootHash); @@ -705,14 +705,14 @@ public void Chaotic_test() } storageTree.UpdateRootHash(); - storageTree.Commit(0); + storageTree.Commit(); account = account.WithChangedStorageRoot(storageTree.RootHash); tree.Set(addressesWithStorage[i].Address, account); } tree.UpdateRootHash(); - tree.Commit(0); + tree.Commit(); for (int i = 0; i < accountsCount; i++) { diff --git a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs index ff3f2558ef5..0b0ce1d8b2b 100644 --- a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs +++ b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs @@ -93,7 +93,7 @@ public void Test01() tree.Set(TestItem.Tree.AccountsWithPaths[4].Path, TestItem.Tree.AccountsWithPaths[4].Account); tree.Set(TestItem.Tree.AccountsWithPaths[5].Path, TestItem.Tree.AccountsWithPaths[5].Account); - tree.Commit(0); + tree.Commit(); Assert.That(tree.RootHash, Is.EqualTo(_inputTree.RootHash)); Assert.That(db.Keys.Count, Is.EqualTo(6)); // we don't persist proof nodes (boundary nodes) diff --git a/src/Nethermind/Nethermind.State.Test/StateTreeTests.cs b/src/Nethermind/Nethermind.State.Test/StateTreeTests.cs index 70e0244ef4d..c333a35b5fa 100644 --- a/src/Nethermind/Nethermind.State.Test/StateTreeTests.cs +++ b/src/Nethermind/Nethermind.State.Test/StateTreeTests.cs @@ -36,7 +36,7 @@ public void No_reads_when_setting_on_empty() tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(db.ReadsCount, Is.EqualTo(0), "reads"); } @@ -48,7 +48,7 @@ public void Minimal_writes_when_setting_on_empty() tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(5), "writes"); // branch, branch, two leaves (one is stored as RLP) } @@ -60,7 +60,7 @@ public void Minimal_writes_when_setting_on_empty_scenario_2() tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(7), "writes"); // extension, branch, leaf, extension, branch, 2x same leaf Assert.That(Trie.Metrics.TreeNodeHashCalculations, Is.EqualTo(7), "hashes"); Assert.That(Trie.Metrics.TreeNodeRlpEncodings, Is.EqualTo(7), "encodings"); @@ -75,7 +75,7 @@ public void Minimal_writes_when_setting_on_empty_scenario_3() tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(4), "writes"); // extension, branch, 2x leaf Assert.That(Trie.Metrics.TreeNodeHashCalculations, Is.EqualTo(4), "hashes"); Assert.That(Trie.Metrics.TreeNodeRlpEncodings, Is.EqualTo(4), "encodings"); @@ -91,7 +91,7 @@ public void Minimal_writes_when_setting_on_empty_scenario_4() tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(1), "writes"); // extension, branch, 2x leaf Assert.That(Trie.Metrics.TreeNodeHashCalculations, Is.EqualTo(1), "hashes"); Assert.That(Trie.Metrics.TreeNodeRlpEncodings, Is.EqualTo(1), "encodings"); @@ -108,7 +108,7 @@ public void Minimal_writes_when_setting_on_empty_scenario_5() tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), null); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(0), "writes"); // extension, branch, 2x leaf Assert.That(Trie.Metrics.TreeNodeHashCalculations, Is.EqualTo(0), "hashes"); Assert.That(Trie.Metrics.TreeNodeRlpEncodings, Is.EqualTo(0), "encodings"); @@ -127,7 +127,7 @@ public void Scenario_traverse_extension_read_full_match() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); } @@ -143,7 +143,7 @@ public void Scenario_traverse_extension_read_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); } @@ -158,7 +158,7 @@ public void Scenario_traverse_extension_new_branching() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x543c960143a2a06b685d6b92f0c37000273e616bc23888521e7edf15ad06da46")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x543c960143a2a06b685d6b92f0c37000273e616bc23888521e7edf15ad06da46")); } @@ -174,7 +174,7 @@ public void Scenario_traverse_extension_delete_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); } @@ -191,7 +191,7 @@ public void Scenario_traverse_extension_create_new_extension() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x0918112fc898173562441709a2c1cbedb80d1aaecaeadf2f3e9492eeaa568c67")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x0918112fc898173562441709a2c1cbedb80d1aaecaeadf2f3e9492eeaa568c67")); } @@ -205,7 +205,7 @@ public void Scenario_traverse_leaf_update_new_value() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0xaa5c248d4b4b8c27a654296a8e0cc51131eb9011d9166fa0fca56a966489e169")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0xaa5c248d4b4b8c27a654296a8e0cc51131eb9011d9166fa0fca56a966489e169")); } @@ -219,7 +219,7 @@ public void Scenario_traverse_leaf_update_no_change() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); } @@ -233,7 +233,7 @@ public void Scenario_traverse_leaf_read_matching_leaf() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")); } @@ -247,7 +247,7 @@ public void Scenario_traverse_leaf_delete_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); } @@ -261,7 +261,7 @@ public void Scenario_traverse_leaf_update_with_extension() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x215a4bab4cf2d5ebbaa59c82ae94c9707fcf4cc0ca1fe7e18f918e46db428ef9")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x215a4bab4cf2d5ebbaa59c82ae94c9707fcf4cc0ca1fe7e18f918e46db428ef9")); } @@ -276,7 +276,7 @@ public void Scenario_traverse_leaf_delete_matching_leaf() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); } @@ -291,7 +291,7 @@ public void Scenario_traverse_leaf_read_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); } @@ -306,7 +306,7 @@ public void Scenario_traverse_branch_update_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0xc063af0bd3dd88320bc852ff8452049c42fbc06d1a69661567bd427572824cbf")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0xc063af0bd3dd88320bc852ff8452049c42fbc06d1a69661567bd427572824cbf")); } @@ -322,7 +322,7 @@ public void Scenario_traverse_branch_read_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x94a193704e99c219d9a21428eb37d6d2d71b3d2cea80c77ff0e201c0df70a283")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x94a193704e99c219d9a21428eb37d6d2d71b3d2cea80c77ff0e201c0df70a283")); } @@ -337,7 +337,7 @@ public void Scenario_traverse_branch_delete_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x94a193704e99c219d9a21428eb37d6d2d71b3d2cea80c77ff0e201c0df70a283")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x94a193704e99c219d9a21428eb37d6d2d71b3d2cea80c77ff0e201c0df70a283")); } @@ -349,7 +349,7 @@ public void Minimal_hashes_when_setting_on_empty() tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(Trie.Metrics.TreeNodeHashCalculations, Is.EqualTo(5), "hashes"); // branch, branch, three leaves } @@ -361,7 +361,7 @@ public void Minimal_encodings_when_setting_on_empty() tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(Trie.Metrics.TreeNodeRlpEncodings, Is.EqualTo(5), "encodings"); // branch, branch, three leaves } @@ -373,7 +373,7 @@ public void Zero_decodings_when_setting_on_empty() tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(Trie.Metrics.TreeNodeRlpDecodings, Is.EqualTo(0), "decodings"); } @@ -386,7 +386,7 @@ public void No_writes_on_continues_update() tree.Set(TestItem.AddressA, _account1); tree.Set(TestItem.AddressA, _account2); tree.Set(TestItem.AddressA, _account3); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(1), "writes"); // extension, branch, two leaves } @@ -397,11 +397,11 @@ public void No_writes_on_reverted_update() MemDb db = new(); StateTree tree = new(new TrieStore(db, LimboLogs.Instance).GetTrieStore(null), LimboLogs.Instance); tree.Set(TestItem.AddressA, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(1), "writes before"); // extension, branch, two leaves tree.Set(TestItem.AddressA, _account1); tree.Set(TestItem.AddressA, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(1), "writes after"); // extension, branch, two leaves } @@ -447,7 +447,7 @@ public void Can_ask_about_root_hash_without_when_emptied() tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), null); tree.UpdateRootHash(); Assert.That(tree.RootHash, Is.EqualTo(PatriciaTree.EmptyTreeHash)); - tree.Commit(0); + tree.Commit(); Assert.That(tree.RootHash, Is.EqualTo(PatriciaTree.EmptyTreeHash)); } diff --git a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs index af65595b304..27f814222f8 100644 --- a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs +++ b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs @@ -5,6 +5,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -318,17 +319,36 @@ private void SaveToTree(HashSet toUpdateRoots, Change change) /// Commit persistent storage trees /// /// Current block number - public void CommitTrees(long blockNumber) + public void CommitTrees(IBlockCommitter blockCommitter) { + // Note: These all runs in about 0.4ms. So the little overhead like attempting to sort the tasks + // may make it worst. Always check on mainnet. + + using ArrayPoolList commitTask = new ArrayPoolList(_storages.Count); foreach (KeyValuePair storage in _storages) { if (!_toUpdateRoots.Contains(storage.Key)) { continue; } - storage.Value.Commit(blockNumber); + + if (blockCommitter.TryRequestConcurrencyQuota()) + { + commitTask.Add(Task.Factory.StartNew((ctx) => + { + StorageTree st = (StorageTree)ctx; + st.Commit(); + blockCommitter.ReturnConcurrencyQuota(); + }, storage.Value)); + } + else + { + storage.Value.Commit(); + } } + Task.WaitAll(commitTask.ToArray()); + _toUpdateRoots.Clear(); // only needed here as there is no control over cached storage size otherwise _storages.Clear(); diff --git a/src/Nethermind/Nethermind.State/StateProvider.cs b/src/Nethermind/Nethermind.State/StateProvider.cs index c4689da5d04..e8afceafeb8 100644 --- a/src/Nethermind/Nethermind.State/StateProvider.cs +++ b/src/Nethermind/Nethermind.State/StateProvider.cs @@ -850,14 +850,14 @@ public void Reset(bool resizeCollections = true) _needsStateRootUpdate = false; } - public void CommitTree(long blockNumber) + public void CommitTree() { if (_needsStateRootUpdate) { RecalculateStateRoot(); } - _tree.Commit(blockNumber); + _tree.Commit(); } public static void CommitBranch() diff --git a/src/Nethermind/Nethermind.State/WorldState.cs b/src/Nethermind/Nethermind.State/WorldState.cs index c18fb219381..8df066b9292 100644 --- a/src/Nethermind/Nethermind.State/WorldState.cs +++ b/src/Nethermind/Nethermind.State/WorldState.cs @@ -174,8 +174,11 @@ public void DecrementNonce(Address address, UInt256 delta) public void CommitTree(long blockNumber) { - _persistentStorageProvider.CommitTrees(blockNumber); - _stateProvider.CommitTree(blockNumber); + using (IBlockCommitter committer = _trieStore.BeginBlockCommit(blockNumber)) + { + _persistentStorageProvider.CommitTrees(committer); + _stateProvider.CommitTree(); + } _persistentStorageProvider.StateRoot = _stateProvider.StateRoot; } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedHealingTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedHealingTests.cs index 3c7a2ea1cb2..4c4c3f6a91f 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedHealingTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedHealingTests.cs @@ -71,7 +71,7 @@ public async Task HealBigSqueezedRandomTree() accounts[path] = account; } - dbContext.RemoteStateTree.Commit(0); + dbContext.RemoteStateTree.Commit(); int startingHashIndex = 0; int endHashIndex; @@ -115,7 +115,7 @@ public async Task HealBigSqueezedRandomTree() } } - dbContext.RemoteStateTree.Commit(blockNumber); + dbContext.RemoteStateTree.Commit(); } endHashIndex = startingHashIndex + 1000; diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs index 07505497bea..8450719c629 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs @@ -76,7 +76,7 @@ public async Task Big_test((string Name, Action Setu .WithChangedStorageRoot(SetStorage(dbContext.RemoteTrieStore, i, TestItem.Addresses[i]).RootHash)); dbContext.RemoteStateTree.UpdateRootHash(); - dbContext.RemoteStateTree.Commit(0); + dbContext.RemoteStateTree.Commit(); ctx.Feed.FallAsleep(); ctx.Pool.WakeUpAll(); @@ -94,7 +94,7 @@ public async Task Big_test((string Name, Action Setu .WithChangedStorageRoot(SetStorage(dbContext.RemoteTrieStore, (byte)(i % 7), TestItem.Addresses[i]).RootHash)); dbContext.RemoteStateTree.UpdateRootHash(); - dbContext.RemoteStateTree.Commit(0); + dbContext.RemoteStateTree.Commit(); ctx.Feed.FallAsleep(); @@ -193,7 +193,7 @@ public async Task When_saving_root_goes_asleep() { DbContext dbContext = new(_logger, _logManager); dbContext.RemoteStateTree.Set(TestItem.KeccakA, Build.An.Account.TestObject); - dbContext.RemoteStateTree.Commit(0); + dbContext.RemoteStateTree.Commit(); dbContext.CompareTrees("BEGIN"); @@ -234,7 +234,7 @@ public async Task Can_download_with_moving_target((string Name, Action accountWithStorage = new(); for (int i = 1000; i < 10000; i += 1000) @@ -353,12 +353,12 @@ public void TestWithHugeTree() { storageTree.Set(TestItem.GetRandomKeccak(), TestItem.GetRandomKeccak().Bytes.ToArray()); } - storageTree.Commit(1); + storageTree.Commit(); var account = TestItem.GenerateRandomAccount().WithChangedStorageRoot(storageTree.RootHash); stateTree.Set(address, account); accountWithStorage.Add(new PathWithAccount() { Path = Keccak.Compute(address.Bytes), Account = account }); } - stateTree.Commit(1); + stateTree.Commit(); SnapServer server = new(store.AsReadOnly(), codeDb, CreateConstantStateRootTracker(true), LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs index 95cde2d1a4e..5d48d0a61dc 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs @@ -702,10 +702,13 @@ public void GetNodeData_returns_cached_trie_nodes() Hash256 nodeKey = TestItem.KeccakA; TrieNode node = new(NodeType.Leaf, nodeKey, TestItem.KeccakB.Bytes); IScopedTrieStore scopedTrieStore = trieStore.GetTrieStore(null); - using (ICommitter committer = scopedTrieStore.BeginCommit(TrieType.State, 1, node)) + using (IBlockCommitter _ = trieStore.BeginBlockCommit(1)) { - TreePath path = TreePath.Empty; - committer.CommitNode(ref path, new NodeCommitInfo(node)); + using (ICommitter committer = scopedTrieStore.BeginCommit(node)) + { + TreePath path = TreePath.Empty; + committer.CommitNode(ref path, new NodeCommitInfo(node)); + } } stateDb.KeyExists(nodeKey).Should().BeFalse(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/TrieScenarios.cs b/src/Nethermind/Nethermind.Synchronization.Test/TrieScenarios.cs index 5d3c8cde41d..5f8d57a1820 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/TrieScenarios.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/TrieScenarios.cs @@ -69,7 +69,7 @@ private static (string Name, Action Action)[] InitSc ("empty", (tree, _, codeDb) => { codeDb[Keccak.Compute(Code0).Bytes] = Code0; - tree.Commit(0); + tree.Commit(); }), ("set_3_via_address", (tree, stateDb, codeDb) => { @@ -77,7 +77,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, TestItem.AddressA, Account0); SetStorage(tree, stateDb, TestItem.AddressB, Account0); SetStorage(tree, stateDb, TestItem.AddressC, Account0); - tree.Commit(0); + tree.Commit(); }), ("storage_hash_and_code_hash_same", (tree, stateDb, codeDb) => { @@ -86,11 +86,11 @@ private static (string Name, Action Action)[] InitSc Hash256 account = new Hash256("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); StorageTree remoteStorageTree = new(stateDb.GetTrieStore(account), Keccak.EmptyTreeHash, LimboLogs.Instance); remoteStorageTree.Set((UInt256) 1, new byte[] {1}); - remoteStorageTree.Commit(0); + remoteStorageTree.Commit(); remoteStorageTree.UpdateRootHash(); codeDb[codeHash.Bytes] = code; tree.Set(account, AccountJustState0.WithChangedStorageRoot(remoteStorageTree.RootHash).WithChangedCodeHash(codeHash)); - tree.Commit(0); + tree.Commit(); }), ("storage_hash_and_code_hash_same_with_additional_account_of_same_storage_root", (tree, stateDb, codeDb) => { @@ -100,19 +100,19 @@ private static (string Name, Action Action)[] InitSc Hash256 account1 = new Hash256("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); StorageTree remoteStorageTree1 = new(stateDb.GetTrieStore(account1), Keccak.EmptyTreeHash, LimboLogs.Instance); remoteStorageTree1.Set((UInt256) 1, new byte[] {1}); - remoteStorageTree1.Commit(0); + remoteStorageTree1.Commit(); remoteStorageTree1.UpdateRootHash(); Hash256 account2 = new Hash256("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); StorageTree remoteStorageTree2 = new(stateDb.GetTrieStore(account2), Keccak.EmptyTreeHash, LimboLogs.Instance); remoteStorageTree2.Set((UInt256) 1, new byte[] {1}); - remoteStorageTree2.Commit(0); + remoteStorageTree2.Commit(); remoteStorageTree2.UpdateRootHash(); codeDb[codeHash.Bytes] = code; tree.Set(account1, AccountJustState0.WithChangedStorageRoot(remoteStorageTree1.RootHash)); tree.Set(account2, AccountJustState0.WithChangedStorageRoot(remoteStorageTree2.RootHash).WithChangedCodeHash(codeHash)); - tree.Commit(0); + tree.Commit(); }), ("storage_hash_and_code_hash_same_with_additional_account_of_same_code", (tree, stateDb, codeDb) => { @@ -123,20 +123,20 @@ private static (string Name, Action Action)[] InitSc new Hash256("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); StorageTree remoteStorageTree = new(stateDb.GetTrieStore(accountWithStorage), Keccak.EmptyTreeHash, LimboLogs.Instance); remoteStorageTree.Set((UInt256) 1, new byte[] {1}); - remoteStorageTree.Commit(0); + remoteStorageTree.Commit(); remoteStorageTree.UpdateRootHash(); codeDb[codeHash.Bytes] = code; tree.Set(new Hash256("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), AccountJustState0.WithChangedCodeHash(codeHash)); tree.Set(accountWithStorage, AccountJustState0.WithChangedStorageRoot(remoteStorageTree.RootHash).WithChangedCodeHash(codeHash)); - tree.Commit(0); + tree.Commit(); }), ("branch_with_same_accounts_at_different_addresses", (tree, _, codeDb) => { codeDb[Keccak.Compute(Code0).Bytes] = Code0; tree.Set(new Hash256("1baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), AccountJustState0); tree.Set(new Hash256("2baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), AccountJustState0); - tree.Commit(0); + tree.Commit(); }), ("set_3_delete_1", (tree, stateDb, codeDb) => { @@ -145,7 +145,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), Account0); SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), Account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(0); + tree.Commit(); }), ("set_3_delete_2", (tree, stateDb, codeDb) => { @@ -155,7 +155,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), Account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(0); + tree.Commit(); }), ("set_3_delete_all", (tree, _, _) => { @@ -166,7 +166,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), null); - tree.Commit(0); + tree.Commit(); }), ("extension_read_full_match", (tree, stateDb, codeDb) => { @@ -177,7 +177,7 @@ private static (string Name, Action Action)[] InitSc Account _ = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111"))!; tree.UpdateRootHash(); Hash256 __ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("extension_read_missing", (tree, stateDb, codeDb) => { @@ -188,7 +188,7 @@ private static (string Name, Action Action)[] InitSc Account _ = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"))!; tree.UpdateRootHash(); Hash256 __ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("extension_new_branch", (tree, stateDb, codeDb) => { @@ -200,7 +200,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"), Account2); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("just_state", (tree, _, _) => { @@ -209,7 +209,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"), AccountJustState2); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("extension_delete_missing", (tree, stateDb, codeDb) => { @@ -220,7 +220,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeddddddddddddddddddddddddd"), null); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("extension_create_new_extension", (tree, stateDb, codeDb) => { @@ -234,7 +234,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeaaaaaaaaaaaaaaaab11111111"), Account3); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_new_value", (tree, stateDb, codeDb) => { @@ -243,7 +243,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), Account1); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_no_change", (tree, stateDb, codeDb) => { @@ -252,7 +252,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), Account0); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_delete", (tree, _, _) => { @@ -260,7 +260,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), null); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_delete_missing", (tree, stateDb, codeDb) => { @@ -269,7 +269,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("1111111111111111111111111111111ddddddddddddddddddddddddddddddddd"), null); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_update_extension", (tree, stateDb, codeDb) => { @@ -279,7 +279,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000000000000000000000000000"), Account1); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_read", (tree, stateDb, codeDb) => { @@ -288,7 +288,7 @@ private static (string Name, Action Action)[] InitSc Account _ = tree.Get(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"))!; tree.UpdateRootHash(); Hash256 __ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_update_missing", (tree, stateDb, codeDb) => { @@ -297,7 +297,7 @@ private static (string Name, Action Action)[] InitSc Account _ = tree.Get(new Hash256("111111111111111111111111111111111111111111111111111111111ddddddd"))!; tree.UpdateRootHash(); Hash256 __ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("branch_update_missing", (tree, stateDb, codeDb) => { @@ -309,7 +309,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), Account2); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("branch_read_missing", (tree, stateDb, codeDb) => { @@ -320,7 +320,7 @@ private static (string Name, Action Action)[] InitSc Account _ = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"))!; tree.UpdateRootHash(); Hash256 __ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("branch_delete_missing", (tree, stateDb, codeDb) => { @@ -331,7 +331,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), null); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }) }; } @@ -354,7 +354,7 @@ private static StorageTree SetStorage(ITrieStore trieStore, Hash256 account) remoteStorageTree.Set((UInt256)1007, new byte[] { 7 }); remoteStorageTree.Set((UInt256)1008, new byte[] { 8 }); - remoteStorageTree.Commit(0); + remoteStorageTree.Commit(); return remoteStorageTree; } diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs index ea9dfc1f70a..504d66fb884 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs @@ -76,7 +76,7 @@ public static (AddRangeResult result, bool moreChildrenToRight, List reorgBoundaryCount += e.BlockNumber; - trieStore.BeginCommit(TrieType.State, 1, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(1, trieNode).Dispose(); reorgBoundaryCount.Should().Be(0); - trieStore.BeginCommit(TrieType.State, 2, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(2, trieNode).Dispose(); reorgBoundaryCount.Should().Be(1); - trieStore.BeginCommit(TrieType.State, 3, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(3, trieNode).Dispose(); reorgBoundaryCount.Should().Be(3); - trieStore.BeginCommit(TrieType.State, 4, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(4, trieNode).Dispose(); reorgBoundaryCount.Should().Be(6); } @@ -154,12 +150,11 @@ public void Should_always_announce_zero_when_not_persisting() long reorgBoundaryCount = 0L; using TrieStore fullTrieStore = CreateTrieStore(); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); fullTrieStore.ReorgBoundaryReached += (_, e) => reorgBoundaryCount += e.BlockNumber; - trieStore.BeginCommit(TrieType.State, 1, trieNode).Dispose(); - trieStore.BeginCommit(TrieType.State, 2, trieNode).Dispose(); - trieStore.BeginCommit(TrieType.State, 3, trieNode).Dispose(); - trieStore.BeginCommit(TrieType.State, 4, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(1, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(2, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(3, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(4, trieNode).Dispose(); reorgBoundaryCount.Should().Be(0L); } @@ -202,9 +197,8 @@ public void Memory_with_two_nodes_is_correct() TrieNode trieNode2 = new(NodeType.Leaf, TestItem.KeccakB); using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new TestPruningStrategy(true)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TreePath emptyPath = TreePath.Empty; - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1234, null)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1234, null)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode1)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode2)); @@ -234,7 +228,10 @@ public void Memory_with_concurrent_commits_is_correct() tree.Set(key, value.ToArray()); } - tree.Commit(0); + using (fullTrieStore.BeginBlockCommit(0)) + { + tree.Commit(); + } fullTrieStore.MemoryUsedByDirtyCache.Should().Be(_scheme == INodeStorage.KeyScheme.Hash ? 591672L : 661820L); fullTrieStore.CommittedNodesCount.Should().Be(1349); @@ -249,15 +246,14 @@ public void Memory_with_two_times_two_nodes_is_correct() TrieNode trieNode4 = new(NodeType.Leaf, TestItem.KeccakB); using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new TestPruningStrategy(true)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TreePath emptyPath = TreePath.Empty; - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, 1234, trieNode2)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(1234, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode1)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode2)); } - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, 1235, trieNode2)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(1235, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode3)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode4)); @@ -286,21 +282,20 @@ public void Dispatcher_will_try_to_clear_memory() trieNode4.ResolveKey(null!, ref emptyPath, true); using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new MemoryLimit(640)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1234, trieNode2)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1234, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode1)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode2)); } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1235, trieNode2)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1235, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode3)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode4)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 1236, trieNode2)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(1236, trieNode2)) { } fullTrieStore.MemoryUsedByDirtyCache.Should().Be( trieNode1.GetMemorySize(false) + ExpectedPerNodeKeyMemorySize + @@ -325,15 +320,14 @@ public void Dispatcher_will_try_to_clear_memory_the_soonest_possible() trieNode4.ResolveKey(null!, ref emptyPath, true); using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new MemoryLimit(512)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1234, trieNode2)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1234, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode1)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode2)); } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1235, trieNode2)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1235, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode3)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode4)); @@ -350,13 +344,12 @@ public void Dispatcher_will_try_to_clear_memory_the_soonest_possible() public void Dispatcher_will_always_try_to_clear_memory() { TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new MemoryLimit(512)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TreePath emptyPath = TreePath.Empty; for (int i = 0; i < 1024; i++) { TrieNode fakeRoot = new(NodeType.Leaf, new byte[0]); // 192B fakeRoot.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, i, fakeRoot)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(i, fakeRoot)) { for (int j = 0; j < 1 + i % 3; j++) { @@ -384,16 +377,15 @@ public void Dispatcher_will_save_to_db_everything_from_snapshot_blocks() pruningStrategy: new MemoryLimit(16.MB()), kvStore: memDb, persistenceStrategy: new ConstantInterval(4)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 0, a)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(0, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 1, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(1, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, a)) { } storage.Get(null, TreePath.Empty, a.Keccak).Should().NotBeNull(); fullTrieStore.IsNodeCached(null, TreePath.Empty, a.Keccak).Should().BeTrue(); @@ -410,15 +402,14 @@ public void Stays_in_memory_until_persisted() NodeStorage storage = new NodeStorage(memDb); using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new MemoryLimit(16.MB())); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 0, a)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(0, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 1, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(1, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, a)) { } // <- do not persist in this test storage.Get(null, TreePath.Empty, a.Keccak).Should().BeNull(); @@ -451,20 +442,19 @@ public void Will_get_persisted_on_snapshot_if_referenced() pruningStrategy: new MemoryLimit(16.MB()), persistenceStrategy: new ConstantInterval(4) ); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1, a)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(0, null)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 7, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(7, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, a)) { } storage.Get(null, TreePath.Empty, a.Keccak).Should().NotBeNull(); fullTrieStore.IsNodeCached(null, TreePath.Empty, a.Keccak).Should().BeTrue(); @@ -488,23 +478,21 @@ public void Will_not_get_dropped_on_snapshot_if_unreferenced_in_later_blocks() pruningStrategy: new MemoryLimit(16.MB()), persistenceStrategy: new ConstantInterval(4)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1, a)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(0, null)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, a)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 7, a)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, a)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(7, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(b)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, a)) { } nodeStorage.Get(null, TreePath.Empty, a.Keccak).Should().NotBeNull(); fullTrieStore.IsNodeCached(null, TreePath.Empty, a.Keccak).Should().BeTrue(); @@ -527,23 +515,21 @@ public void Will_get_dropped_on_snapshot_if_it_was_a_transient_node() pruningStrategy: new MemoryLimit(16.MB()), persistenceStrategy: new ConstantInterval(4)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1, a)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(0, null)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 3, a)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(3, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(b)); // <- new root } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, b)) { } // should be 'a' to test properly - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, b)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, b)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 7, b)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, b)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, b)) { } // should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, b)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, b)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(7, b)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, b)) { } memDb[a.Keccak!.Bytes].Should().BeNull(); fullTrieStore.IsNodeCached(null, TreePath.Empty, a.Keccak).Should().BeTrue(); @@ -625,25 +611,28 @@ public void Will_store_storage_on_snapshot() pruningStrategy: new MemoryLimit(16.MB()), persistenceStrategy: new ConstantInterval(4)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); + using (fullTrieStore.BeginStateBlockCommit(0, null)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } - using (ICommitter committer = fullTrieStore.GetTrieStore(TestItem.KeccakA).BeginCommit(TrieType.Storage, 1, storage1)) - { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); - } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1, a)) + using (fullTrieStore.BeginBlockCommit(1)) { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); + using (ICommitter committer = fullTrieStore.GetTrieStore(TestItem.KeccakA).BeginCommit(storage1)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); + } + + using (ICommitter committer = fullTrieStore.GetTrieStore(null).BeginCommit(a)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); + } } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 7, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(7, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, a)) { } asStorage.Get(null, TreePath.Empty, a.Keccak).Should().NotBeNull(); asStorage.Get(TestItem.KeccakA, TreePath.Empty, storage1.Keccak).Should().NotBeNull(); @@ -676,27 +665,31 @@ public void Will_drop_transient_storage() IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(0, null)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.Storage, 1, storage1)) + using (fullTrieStore.BeginBlockCommit(1)) { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); - committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); - } + using (ICommitter committer = fullTrieStore.GetTrieStore(Hash256.Zero).BeginCommit(storage1)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); + committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); + } + + using (ICommitter _ = trieStore.BeginCommit(a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 1, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } + } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 2, b)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(2, b)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(b)); // <- new root } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, b)) { } // Should be 'a' to test properly - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, b)) { } // Should be 'a' to test properly - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, b)) { } // Should be 'a' to test properly - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 7, b)) { } // Should be 'a' to test properly - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, b)) { } // Should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, b)) { } // Should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, b)) { } // Should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, b)) { } // Should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(7, b)) { } // Should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, b)) { } // Should be 'a' to test properly memDb[a.Keccak!.Bytes].Should().BeNull(); memDb[storage1.Keccak!.Bytes].Should().BeNull(); @@ -744,34 +737,35 @@ public void Will_combine_same_storage() pruningStrategy: new MemoryLimit(16.MB()), persistenceStrategy: new ConstantInterval(4)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(0, null)) { } - using (ICommitter committer = fullTrieStore.GetTrieStore(new Hash256(Nibbles.ToBytes(storage1Nib))).BeginCommit(TrieType.Storage, 1, storage1)) + using (fullTrieStore.BeginBlockCommit(1)) { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); - } + using (ICommitter committer = fullTrieStore.GetTrieStore(new Hash256(Nibbles.ToBytes(storage1Nib))).BeginCommit(storage1)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); + } - using (ICommitter committer = fullTrieStore.GetTrieStore(new Hash256(Nibbles.ToBytes(storage2Nib))).BeginCommit(TrieType.Storage, 1, storage2)) - { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage2)); - } + using (ICommitter committer = fullTrieStore.GetTrieStore(new Hash256(Nibbles.ToBytes(storage2Nib))).BeginCommit(storage2)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage2)); + } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1, branch)) - { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); - committer.CommitNode(ref emptyPath, new NodeCommitInfo(b)); - committer.CommitNode(ref emptyPath, new NodeCommitInfo(branch)); + using (ICommitter committer = fullTrieStore.GetTrieStore(null).BeginCommit(branch)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); + committer.CommitNode(ref emptyPath, new NodeCommitInfo(b)); + committer.CommitNode(ref emptyPath, new NodeCommitInfo(branch)); + } } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 7, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(7, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, branch)) { } storage.Get(null, TreePath.FromNibble(new byte[] { 0 }), a.Keccak).Should().NotBeNull(); storage.Get(new Hash256(Nibbles.ToBytes(storage1Nib)), TreePath.Empty, storage1.Keccak).Should().NotBeNull(); @@ -800,7 +794,7 @@ public async Task Read_only_trie_store_is_allowing_many_thread_to_work_with_the_ IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TreePath emptyPath = TreePath.Empty; trieNode.ResolveKey(trieStore, ref emptyPath, false); - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, 0, trieNode)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(0, trieNode)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode)); } @@ -857,7 +851,7 @@ public void ReadOnly_store_returns_copies(bool pruning) using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new TestPruningStrategy(pruning)); IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, 0, node)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(0, node)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(node)); } @@ -891,13 +885,13 @@ public void After_commit_should_have_has_root() Account account = new(1); StateTree stateTree = new(trieStore, LimboLogs.Instance); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); trieStore.HasRoot(stateTree.RootHash).Should().BeTrue(); stateTree.Get(TestItem.AddressA); account = account.WithChangedBalance(2); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); trieStore.HasRoot(stateTree.RootHash).Should().BeTrue(); } @@ -911,13 +905,12 @@ public async Task Will_RemovePastKeys_OnSnapshot() pruningStrategy: new TestPruningStrategy(true, true, 2, 100000), persistenceStrategy: No.Persistence); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TreePath emptyPath = TreePath.Empty; for (int i = 0; i < 64; i++) { TrieNode node = new(NodeType.Leaf, TestItem.Keccaks[i], new byte[2]); - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, i, node)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(i, node)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(node)); } @@ -955,7 +948,7 @@ public async Task Will_Trigger_ReorgBoundaryEvent_On_Prune() for (int i = 0; i < 64; i++) { TrieNode node = new(NodeType.Leaf, TestItem.Keccaks[i], new byte[2]); - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, i, node)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(i, node)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(node)); } @@ -991,7 +984,7 @@ public async Task Will_Not_RemovePastKeys_OnSnapshot_DuringFullPruning() for (int i = 0; i < 64; i++) { TrieNode node = new(NodeType.Leaf, TestItem.Keccaks[i], new byte[2]); - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, i, node)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(i, node)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(node)); } @@ -1021,7 +1014,7 @@ public async Task Will_NotRemove_ReCommittedNode() for (int i = 0; i < 64; i++) { TrieNode node = new(NodeType.Leaf, TestItem.Keccaks[i % 4], new byte[2]); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, i, node)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(i, node)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(node)); } diff --git a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs index 001be11d7d7..a2acfd0be71 100644 --- a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs @@ -925,7 +925,8 @@ void CheckChildren() [Test] public void Rlp_is_cloned_when_cloning() { - IScopedTrieStore trieStore = new TrieStore(new MemDb(), NullLogManager.Instance).GetTrieStore(null); + ITrieStore fullTrieStore = new TrieStore(new MemDb(), NullLogManager.Instance); + IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TrieNode leaf1 = new(NodeType.Leaf); leaf1.Key = Bytes.FromHexString("abc"); @@ -942,10 +943,13 @@ public void Rlp_is_cloned_when_cloning() TreePath path = TreePath.Empty; - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, 0, leaf2)) + using (IBlockCommitter _ = fullTrieStore.BeginBlockCommit(0)) { - committer.CommitNode(ref path, new NodeCommitInfo(leaf1)); - committer.CommitNode(ref path, new NodeCommitInfo(leaf2)); + using (ICommitter? committer = trieStore.BeginCommit(leaf2)) + { + committer.CommitNode(ref path, new NodeCommitInfo(leaf1)); + committer.CommitNode(ref path, new NodeCommitInfo(leaf2)); + } } TrieNode trieNode = new(NodeType.Branch); diff --git a/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs b/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs index 0d9e05c1e1f..6f61ae6b0c1 100644 --- a/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs @@ -61,7 +61,7 @@ public void Single_leaf() using TrieStore trieStore = new(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager); PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(1); @@ -75,7 +75,7 @@ public void Single_leaf_update_same_block() PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyA, _longLeaf2); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(1); @@ -92,9 +92,9 @@ public void Single_leaf_update_next_blocks() using TrieStore trieStore = new(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager); PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, _longLeaf2); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); patriciaTree.UpdateRootHash(); // leaf (root) @@ -113,7 +113,7 @@ public void Single_leaf_delete_same_block() PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyA, Array.Empty()); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(0); @@ -129,9 +129,9 @@ public void Single_leaf_delete_next_block() using TrieStore trieStore = new(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager); PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, Array.Empty()); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); patriciaTree.UpdateRootHash(); // leaf (root) @@ -147,24 +147,24 @@ public void Single_leaf_and_keep_for_multiple_dispatches_then_delete() MemDb memDb = new(); using TrieStore trieStore = new(memDb, Prune.WhenCacheReaches(1.MB()), new ConstantInterval(4), LimboLogs.Instance); PatriciaTree patriciaTree = new(trieStore, _logManager); - patriciaTree.Commit(0); - patriciaTree.Commit(1); - patriciaTree.Commit(2); + trieStore.CommitPatriciaTrie(0, patriciaTree); + trieStore.CommitPatriciaTrie(1, patriciaTree); + trieStore.CommitPatriciaTrie(2, patriciaTree); patriciaTree.Set(_keyA, _longLeaf1); - patriciaTree.Commit(3); - patriciaTree.Commit(4); + trieStore.CommitPatriciaTrie(3, patriciaTree); + trieStore.CommitPatriciaTrie(4, patriciaTree); patriciaTree.Set(_keyA, Array.Empty()); - patriciaTree.Commit(5); + trieStore.CommitPatriciaTrie(5, patriciaTree); patriciaTree.Set(_keyB, _longLeaf2); - patriciaTree.Commit(6); - patriciaTree.Commit(7); - patriciaTree.Commit(8); - patriciaTree.Commit(9); - patriciaTree.Commit(10); - patriciaTree.Commit(11); + trieStore.CommitPatriciaTrie(6, patriciaTree); + trieStore.CommitPatriciaTrie(7, patriciaTree); + trieStore.CommitPatriciaTrie(8, patriciaTree); + trieStore.CommitPatriciaTrie(9, patriciaTree); + trieStore.CommitPatriciaTrie(10, patriciaTree); + trieStore.CommitPatriciaTrie(11, patriciaTree); patriciaTree.Set(_keyB, Array.Empty()); - patriciaTree.Commit(12); - patriciaTree.Commit(13); + trieStore.CommitPatriciaTrie(12, patriciaTree); + trieStore.CommitPatriciaTrie(13, patriciaTree); patriciaTree.UpdateRootHash(); // leaf (root) @@ -184,7 +184,7 @@ public void Branch_with_branch_and_leaf() patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf1); patriciaTree.Set(_keyC, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(6); @@ -218,7 +218,7 @@ public void GetBranchNodesWithPartialPath() patriciaTree.Set(_keysA, _longLeaf1); patriciaTree.Set(_keysB, _longLeaf1); patriciaTree.Set(_keysC, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -252,11 +252,11 @@ public void Branch_with_branch_and_leaf_then_deleted() patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf1); patriciaTree.Set(_keyC, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, Array.Empty()); patriciaTree.Set(_keyB, Array.Empty()); patriciaTree.Set(_keyC, Array.Empty()); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); patriciaTree.UpdateRootHash(); // leaf (root) @@ -280,7 +280,8 @@ public void Test_add_many(int i) patriciaTree.Set(key.Bytes, value); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); + patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -312,7 +313,7 @@ public void Test_try_delete_and_read_missing_nodes(int i) patriciaTree.Set(key.Bytes, Array.Empty()); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -353,7 +354,7 @@ public void Test_update_many(int i) patriciaTree.Set(key.Bytes, value); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -378,7 +379,7 @@ public void Test_update_many_next_block(int i) patriciaTree.Set(key.Bytes, value); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); for (int j = 0; j < i; j++) { @@ -388,7 +389,7 @@ public void Test_update_many_next_block(int i) _logger.Trace($"Setting {key.Bytes.ToHexString()} = {value.ToHexString()}"); } - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -423,7 +424,7 @@ public void Test_add_and_delete_many_same_block(int i) patriciaTree.Set(key.Bytes, Array.Empty()); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -447,7 +448,7 @@ public void Test_add_and_delete_many_next_block(int i) patriciaTree.Set(key.Bytes, value); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); for (int j = 0; j < i; j++) { @@ -455,7 +456,7 @@ public void Test_add_and_delete_many_next_block(int i) patriciaTree.Set(key.Bytes, Array.Empty()); } - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -493,7 +494,7 @@ public void Two_branches_exactly_same_leaf() patriciaTree.Set(_keyB, _longLeaf1); patriciaTree.Set(_keyC, _longLeaf1); patriciaTree.Set(_keyD, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(8); @@ -519,7 +520,7 @@ public void Two_branches_exactly_same_leaf_then_one_removed() patriciaTree.Set(_keyC, _longLeaf1); patriciaTree.Set(_keyD, _longLeaf1); patriciaTree.Set(_keyA, Array.Empty()); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(6); @@ -545,7 +546,7 @@ public void Extension_with_branch_with_two_different_children() PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf2); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); memDb.Keys.Should().HaveCount(4); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); checkTree.Get(_keyA).ToArray().Should().BeEquivalentTo(_longLeaf1); @@ -560,7 +561,7 @@ public void Extension_with_branch_with_two_same_children() PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); memDb.Keys.Should().HaveCount(4); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); checkTree.Get(_keyA).ToArray().Should().BeEquivalentTo(_longLeaf1); @@ -576,11 +577,11 @@ public void When_branch_with_two_different_children_change_one_and_change_back_n patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf2); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, _longLeaf3); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); // extension // branch @@ -597,11 +598,11 @@ public void When_branch_with_two_same_children_change_one_and_change_back_next_b patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, _longLeaf3); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); memDb.Keys.Should().HaveCount(4); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -631,7 +632,7 @@ L L - - - - - - - - - - - - - - */ patriciaTree.Set(key2, _longLeaf1); patriciaTree.Set(key3, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); memDb.Keys.Should().HaveCount(7); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -679,10 +680,10 @@ L L - - - - - - - - - - - - - - */ patriciaTree.Set(key2, _longLeaf1); patriciaTree.Set(key3, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(key3, Array.Empty()); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); memDb.Keys.Should().HaveCount(8); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -702,11 +703,11 @@ public void When_two_branches_with_two_same_children_change_one_and_change_back_ patriciaTree.Set(_keyC, _longLeaf1); patriciaTree.Set(_keyD, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, _longLeaf3); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); memDb.Keys.Should().HaveCount(8); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -788,7 +789,7 @@ public void Fuzz_accounts( streamWriter.WriteLine( $"Commit block {blockNumber} | empty: {isEmptyBlock}"); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(blockNumber); + trieStore.CommitPatriciaTrie(blockNumber, patriciaTree); rootQueue.Enqueue(patriciaTree.RootHash); } @@ -930,7 +931,7 @@ public void Fuzz_accounts_with_reorganizations( streamWriter.WriteLine( $"Commit block {blockCount} | empty: {isEmptyBlock}"); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(blockCount); + trieStore.CommitPatriciaTrie(blockNumber, patriciaTree); rootQueue.Enqueue(patriciaTree.RootHash); rootStack.Push(patriciaTree.RootHash); blockCount++; diff --git a/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs b/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs index 28a7d2bb514..1f773711e92 100644 --- a/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs @@ -39,7 +39,7 @@ public void Visitors_state(VisitingOptions options) patriciaTree.Set(raw, Rlp.Encode(new Account(10, (UInt256)(10_000_000 + i)))); } - patriciaTree.Commit(0); + using (trieStore.BeginBlockCommit(0)) { patriciaTree.Commit(); } var visitor = new AppendingVisitor(); @@ -70,6 +70,8 @@ public void Visitors_storage(VisitingOptions options) byte[] value = Enumerable.Range(1, 32).Select(i => (byte)i).ToArray(); Hash256 stateRootHash = Keccak.Zero; + var blockCommit = trieStore.BeginBlockCommit(0); + for (int outi = 0; outi < 64; outi++) { ValueHash256 stateKey = default; @@ -82,7 +84,7 @@ public void Visitors_storage(VisitingOptions options) storageKey.BytesAsSpan[i / 2] = (byte)(1 << (4 * (1 - i % 2))); storage.Set(storageKey, value); } - storage.Commit(0); + storage.Commit(); stateRootHash = storage.RootHash; } @@ -99,7 +101,8 @@ public void Visitors_storage(VisitingOptions options) new Account(10, (UInt256)(10_000_000 + i), stateRootHash, Keccak.OfAnEmptySequenceRlp)); } - stateTree.Commit(0); + stateTree.Commit(); + blockCommit.Dispose(); var visitor = new AppendingVisitor(); diff --git a/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs b/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs index 5b23ba6ad92..984ab33ec07 100644 --- a/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs @@ -34,8 +34,8 @@ public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address) => public INodeStorage.KeyScheme Scheme => @base.Scheme; - public ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => - @base.BeginCommit(trieType, blockNumber, root, writeFlags); + public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => + @base.BeginCommit(root, writeFlags); public bool IsPersisted(in TreePath path, in ValueHash256 keccak) => @base.IsPersisted(in path, in keccak); diff --git a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs index 88b443a3983..2ff2a442f5b 100644 --- a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs +++ b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs @@ -131,7 +131,7 @@ public PatriciaTree( _bufferPool = bufferPool; } - public void Commit(long blockNumber, bool skipRoot = false, WriteFlags writeFlags = WriteFlags.None) + public void Commit(bool skipRoot = false, WriteFlags writeFlags = WriteFlags.None) { if (!_allowCommits) { @@ -140,14 +140,15 @@ public void Commit(long blockNumber, bool skipRoot = false, WriteFlags writeFlag int maxLevelForConcurrentCommit = _writeBeforeCommit switch { - > 64 * 16 => 1, // we separate at two top levels - > 64 => 0, // we separate at top level + > 4 * 16 * 16 => 2, // we separate at three top levels + > 4 * 16 => 1, // we separate at two top levels + > 4 => 0, // we separate at top level _ => -1 }; _writeBeforeCommit = 0; - using (ICommitter committer = TrieStore.BeginCommit(TrieType, blockNumber, RootRef, writeFlags)) + using (ICommitter committer = TrieStore.BeginCommit(RootRef, writeFlags)) { if (RootRef is not null && RootRef.IsDirty) { @@ -159,14 +160,6 @@ public void Commit(long blockNumber, bool skipRoot = false, WriteFlags writeFlag SetRootHash(RootRef.Keccak!, true); } } - - if (_logger.IsDebug) Debug(blockNumber); - - [MethodImpl(MethodImplOptions.NoInlining)] - void Debug(long blockNumber) - { - _logger.Debug($"Finished committing {RootRef?.Keccak} in block {blockNumber}"); - } } private void Commit(ICommitter committer, ref TreePath path, NodeCommitInfo nodeCommitInfo, int maxLevelForConcurrentCommit, bool skipSelf = false) @@ -214,7 +207,7 @@ Task CreateTaskForPath(TreePath childPath, TrieNode childNode, int idx) => Task. { if (node.IsChildDirty(i)) { - if (i < 15 && committer.CanSpawnTask()) + if (i < 15 && committer.TryRequestConcurrentQuota()) { childTasks ??= new ArrayPoolList(15); TreePath childPath = path.Append(i); diff --git a/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs index 82d83a3e8f3..ece7d00245b 100644 --- a/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs @@ -5,7 +5,6 @@ using System.Collections.Concurrent; using System.Numerics; using Nethermind.Core; -using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Trie.Pruning; @@ -34,9 +33,14 @@ public void Dispose() _inner.Dispose(); } - public ICommitter BeginCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags) + public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) { - return _inner.BeginCommit(trieType, blockNumber, address, root, writeFlags); + return _inner.BeginCommit(address, root, writeFlags); + } + + public IBlockCommitter BeginBlockCommit(long blockNumber) + { + return _inner.BeginBlockCommit(blockNumber); } public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) diff --git a/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs b/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs index 3ea7daf397a..c5993090cac 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Diagnostics; + namespace Nethermind.Trie.Pruning { internal class BlockCommitSet @@ -25,5 +27,23 @@ public override string ToString() { return $"{BlockNumber}({Root})"; } + + /// + /// Prunes persisted branches of the current commit set root. + /// + public void Prune() + { + long start = Stopwatch.GetTimestamp(); + + // We assume that the most recent package very likely resolved many persisted nodes and only replaced + // some top level branches. Any of these persisted nodes are held in cache now so we just prune them here + // to avoid the references still being held after we prune the cache. + // We prune them here but just up to two levels deep which makes it a very lightweight operation. + // Note that currently the TrieNode ResolveChild un-resolves any persisted child immediately which + // may make this call unnecessary. + Root?.PrunePersistedRecursively(2); + Metrics.DeepPruningTime = (long)Stopwatch.GetElapsedTime(start).TotalMilliseconds; + } + } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/IScopedTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/IScopedTrieStore.cs index 3dec5d49e25..6c4a445d86b 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/IScopedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/IScopedTrieStore.cs @@ -13,7 +13,8 @@ namespace Nethermind.Trie.Pruning; /// public interface IScopedTrieStore : ITrieNodeResolver { - ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None); + // Begins a commit to update the trie store. The `ICommitter` provide `CommitNode` to add node into. + ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None); // Only used by snap provider, so ValueHash instead of Hash bool IsPersisted(in TreePath path, in ValueHash256 keccak); @@ -26,6 +27,6 @@ public interface ICommitter : IDisposable { void CommitNode(ref TreePath path, NodeCommitInfo nodeCommitInfo); - bool CanSpawnTask() => false; + bool TryRequestConcurrentQuota() => false; void ReturnConcurrencyQuota() { } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ITrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/ITrieStore.cs index f9f2f59d55e..3178da3fd6d 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/ITrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/ITrieStore.cs @@ -11,10 +11,8 @@ namespace Nethermind.Trie.Pruning /// /// Full traditional trie store. /// - public interface ITrieStore : IDisposable + public interface ITrieStore : IDisposable, ITrieStoreInternal { - bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak); - IReadOnlyTrieStore AsReadOnly(INodeStorage? keyValueStore = null); event EventHandler? ReorgBoundaryReached; @@ -22,22 +20,49 @@ public interface ITrieStore : IDisposable // Used for serving via hash IReadOnlyKeyValueStore TrieNodeRlpStore { get; } - // Used by healing - void Set(Hash256? address, in TreePath path, in ValueHash256 keccak, byte[] rlp); - bool HasRoot(Hash256 stateRoot); IScopedTrieStore GetTrieStore(Hash256? address); + INodeStorage.KeyScheme Scheme { get; } + + /// + /// Begin a block commit for this block number. This call may be blocked if a memory pruning is currently happening. + /// This call is required during block processing for memory pruning and reorg boundary to function. + /// + /// + /// + IBlockCommitter BeginBlockCommit(long blockNumber); + } + /// + /// These methods are to be used by ScopedTrieStore. + /// It should be considered internal to TrieStore. + /// It should not be used directly, nor intercepted. + /// + public interface ITrieStoreInternal + { + // Used by healing + void Set(Hash256? address, in TreePath path, in ValueHash256 keccak, byte[] rlp); + ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags); TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash); byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None); byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None); - INodeStorage.KeyScheme Scheme { get; } - ICommitter BeginCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags); + bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak); } public interface IPruningTrieStore { public void PersistCache(CancellationToken cancellationToken); } + + /// + /// A block committer identifies the scope at which a commit for a block should happen. + /// The commit started via which is called by + /// Depending on , multiple patricia trie commit may run at the same time. + /// + public interface IBlockCommitter : IDisposable + { + bool TryRequestConcurrencyQuota() => false; + void ReturnConcurrencyQuota() { } + } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/NullCommitter.cs b/src/Nethermind/Nethermind.Trie/Pruning/NullCommitter.cs new file mode 100644 index 00000000000..c39ddc374bf --- /dev/null +++ b/src/Nethermind/Nethermind.Trie/Pruning/NullCommitter.cs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.Trie.Pruning; + +internal class NullCommitter : ICommitter, IBlockCommitter +{ + public static NullCommitter Instance = new NullCommitter(); + + private NullCommitter() + { + } + + public void Dispose() { } + + public void CommitNode(ref TreePath path, NodeCommitInfo nodeCommitInfo) { } + public ICommitter GetTrieCommitter(Hash256? address, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => this; +} diff --git a/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs index 585050f7150..5e96cb610ea 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Nethermind.Core; @@ -21,7 +22,7 @@ private NullTrieStore() { } public byte[]? TryLoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => []; - public ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => new NullCommitter(); + public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => NullCommitter.Instance; public bool IsPersisted(in TreePath path, in ValueHash256 keccak) => true; @@ -30,12 +31,5 @@ public void Set(in TreePath path, in ValueHash256 keccak, byte[] rlp) { } public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256 storageRoot) => this; public INodeStorage.KeyScheme Scheme => INodeStorage.KeyScheme.HalfPath; - - internal class NullCommitter : ICommitter - { - public void Dispose() { } - - public void CommitNode(ref TreePath path, NodeCommitInfo nodeCommitInfo) { } - } } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs index 4bb2c8d537f..4f66fb73e70 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs @@ -29,8 +29,12 @@ public byte[] LoadRlp(Hash256? address, in TreePath treePath, Hash256 hash, Read public IReadOnlyTrieStore AsReadOnly(INodeStorage nodeStore) => new ReadOnlyTrieStore(_trieStore, nodeStore); - public ICommitter BeginCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags) => - new NullTrieStore.NullCommitter(); + public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) => NullCommitter.Instance; + + public IBlockCommitter BeginBlockCommit(long blockNumber) + { + return NullCommitter.Instance; + } public event EventHandler ReorgBoundaryReached { @@ -63,8 +67,7 @@ public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address1) => public INodeStorage.KeyScheme Scheme => fullTrieStore.Scheme; - public ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => - new NullTrieStore.NullCommitter(); + public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => NullCommitter.Instance; public bool IsPersisted(in TreePath path, in ValueHash256 keccak) => fullTrieStore.IsPersisted(address, path, in keccak); diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs index 650ee399086..66a61af6f74 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs @@ -22,8 +22,8 @@ public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address1) => public INodeStorage.KeyScheme Scheme => fullTrieStore.Scheme; - public ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => - fullTrieStore.BeginCommit(trieType, blockNumber, address, root, writeFlags); + public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => + fullTrieStore.BeginCommit(address, root, writeFlags); public bool IsPersisted(in TreePath path, in ValueHash256 keccak) => fullTrieStore.IsPersisted(address, path, in keccak); diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index ac44f42a3e0..b0cacf09bf0 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -27,8 +27,6 @@ public class TrieStore : ITrieStore, IPruningTrieStore private int _isFirst; - private INodeStorage.WriteBatch? _currentBatch = null; - private readonly TrieStoreDirtyNodesCache[] _dirtyNodes = []; private readonly Task[] _dirtyNodesTasks = []; private readonly ConcurrentDictionary[] _persistedHashes = []; @@ -164,8 +162,10 @@ public int CachedNodesCount } } - private void CommitNode(long blockNumber, Hash256? address, ref TreePath path, in NodeCommitInfo nodeCommitInfo, WriteFlags writeFlags = WriteFlags.None) + private void CommitAndInsertToDirtyNodes(long blockNumber, Hash256? address, ref TreePath path, in NodeCommitInfo nodeCommitInfo) { + Debug.Assert(_pruningStrategy.PruningEnabled); + if (_logger.IsTrace) Trace(blockNumber, in nodeCommitInfo); if (!nodeCommitInfo.IsEmptyBlockMarker && !nodeCommitInfo.Node.IsBoundaryProofNode) { @@ -181,18 +181,9 @@ private void CommitNode(long blockNumber, Hash256? address, ref TreePath path, i ThrowNodeHasBeenSeen(blockNumber, node); } - if (_pruningStrategy.PruningEnabled) - { - node = SaveOrReplaceInDirtyNodesCache(address, ref path, nodeCommitInfo, node); - } - + node = SaveOrReplaceInDirtyNodesCache(address, ref path, nodeCommitInfo, node); node.LastSeen = Math.Max(blockNumber, node.LastSeen); - if (!_pruningStrategy.PruningEnabled) - { - PersistNode(address, path, node, blockNumber, writeFlags); - } - IncrementCommittedNodesCount(); } @@ -211,6 +202,29 @@ void Trace(long blockNumber, in NodeCommitInfo nodeCommitInfo) static void ThrowNodeHasBeenSeen(long blockNumber, TrieNode node) => throw new TrieStoreException($"{nameof(TrieNode.LastSeen)} set on {node} committed at {blockNumber}."); } + private void CommitAndPersistNode(Hash256? address, ref TreePath path, in NodeCommitInfo nodeCommitInfo, WriteFlags writeFlags, INodeStorage.WriteBatch? writeBatch) + { + Debug.Assert(!_pruningStrategy.PruningEnabled); + + if (!nodeCommitInfo.IsEmptyBlockMarker && !nodeCommitInfo.Node.IsBoundaryProofNode) + { + TrieNode node = nodeCommitInfo.Node; + + if (node.Keccak is null) + { + ThrowUnknownHash(node); + } + + PersistNode(address, path, node, writeBatch!, writeFlags); + + IncrementCommittedNodesCount(); + } + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowUnknownHash(TrieNode node) => throw new TrieStoreException($"The hash of {node} should be known at the time of committing."); + } + private int GetNodeShardIdx(in TreePath path, Hash256 hash) => // When enabled, the shard have dictionaries for tracking past path hash also. // So the same path need to be in the same shard for the remove logic to work. @@ -289,70 +303,61 @@ static void ThrowNodeIsNotSame(TrieNode node, TrieNode cachedNodeCopy) => throw new InvalidOperationException($"The hash of replacement node {cachedNodeCopy} is not the same as the original {node}."); } - public ICommitter BeginCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags) + public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) { - ArgumentOutOfRangeException.ThrowIfNegative(blockNumber); - EnsureCommitSetExistsForBlock(blockNumber); - - int concurrency = _pruningStrategy.PruningEnabled - ? Environment.ProcessorCount - : 0; // The write batch when pruning is not enabled is not concurrent safe + if (_pruningStrategy.PruningEnabled) + { + if (_currentBlockCommitter is null) throw new InvalidOperationException($"With pruning triestore, {nameof(BeginBlockCommit)} must be called."); + } - return new TrieStoreCommitter(this, trieType, blockNumber, address, root, writeFlags, concurrency); + return _currentBlockCommitter is not null + // Note, archive node still use this path. This is because it needs the block commit set to handle reorg announcement. + ? _currentBlockCommitter.GetTrieCommitter(address, root, writeFlags) + // This happens when there are no block involved, such as during snap sync or just calculating patricia root. + : new NonPruningTrieStoreCommitter(this, address, _nodeStorage.StartWriteBatch(), writeFlags); } - private void FinishBlockCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) + public IBlockCommitter BeginBlockCommit(long blockNumber) { - try + if (_pruningStrategy.PruningEnabled) { - if (trieType == TrieType.State) // storage tries happen before state commits + if (_currentBlockCommitter is not null) { - if (_logger.IsTrace) _logger.Trace($"Enqueued blocks {_commitSetQueue?.Count ?? 0}"); - BlockCommitSet set = CurrentPackage; - if (set is not null) - { - if (_logger.IsTrace) _logger.Trace($"Current root (block {blockNumber}): {root}, block {set.BlockNumber}"); - set.Seal(root); - } - - bool shouldPersistSnapshot = _persistenceStrategy.ShouldPersist(set.BlockNumber); - if (shouldPersistSnapshot) - { - _currentBatch ??= _nodeStorage.StartWriteBatch(); - try - { - PersistBlockCommitSet(address, set, _currentBatch, writeFlags: writeFlags); - PruneCurrentSet(); - } - finally - { - // For safety we prefer to commit half of the batch rather than not commit at all. - // Generally hanging nodes are not a problem in the DB but anything missing from the DB is. - _currentBatch?.Dispose(); - _currentBatch = null; - } - } - else - { - PruneCurrentSet(); - } - - CurrentPackage = null; - if (_pruningStrategy.PruningEnabled && Monitor.IsEntered(_dirtyNodesLock)) - { - Monitor.Exit(_dirtyNodesLock); - } + throw new InvalidOperationException("Cannot start a new block commit when an existing one is still not closed"); } + + Monitor.Enter(_dirtyNodesLock); } - finally + + _currentBlockCommitter = new BlockCommitter(this, CreateCommitSet(blockNumber)); + return _currentBlockCommitter; + } + + private void FinishBlockCommit(BlockCommitSet set, TrieNode? root) + { + if (_logger.IsTrace) _logger.Trace($"Enqueued blocks {_commitSetQueue?.Count ?? 0}"); + set.Seal(root); + + bool shouldPersistSnapshot = _persistenceStrategy.ShouldPersist(set.BlockNumber); + if (shouldPersistSnapshot) { - _currentBatch?.Dispose(); - _currentBatch = null; + // For safety we prefer to commit half of the batch rather than not commit at all. + // Generally hanging nodes are not a problem in the DB but anything missing from the DB is. + using INodeStorage.WriteBatch currentBatch = _nodeStorage.StartWriteBatch(); + ParallelPersistBlockCommitSet(set); } + set.Prune(); + + _currentBlockCommitter = null; + + if (_pruningStrategy.PruningEnabled) + Monitor.Exit(_dirtyNodesLock); + Prune(); } + public event EventHandler? ReorgBoundaryReached; public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 keccak, INodeStorage? nodeStorage, ReadFlags readFlags = ReadFlags.None) @@ -544,7 +549,7 @@ private bool SaveSnapshot() { BlockCommitSet blockCommitSet = candidateSets[index]; if (_logger.IsDebug) _logger.Debug($"Elevated pruning for candidate {blockCommitSet.BlockNumber}"); - ParallelPersistBlockCommitSet(null, blockCommitSet, persistedNodeRecorder); + ParallelPersistBlockCommitSet(blockCommitSet, persistedNodeRecorder); } Task deleteTask = shouldDeletePersistedNode ? RemovePastKeys() : Task.CompletedTask; @@ -599,23 +604,6 @@ private void PersistedNodeRecorder(TreePath treePath, Hash256 address, TrieNode } } - /// - /// Prunes persisted branches of the current commit set root. - /// - private void PruneCurrentSet() - { - long start = Stopwatch.GetTimestamp(); - - // We assume that the most recent package very likely resolved many persisted nodes and only replaced - // some top level branches. Any of these persisted nodes are held in cache now so we just prune them here - // to avoid the references still being held after we prune the cache. - // We prune them here but just up to two levels deep which makes it a very lightweight operation. - // Note that currently the TrieNode ResolveChild un-resolves any persisted child immediately which - // may make this call unnecessary. - CurrentPackage?.Root?.PrunePersistedRecursively(2); - Metrics.DeepPruningTime = (long)Stopwatch.GetElapsedTime(start).TotalMilliseconds; - } - /// /// This method is responsible for reviewing the nodes that are directly in the cache and /// removing ones that are either no longer referenced or already persisted. @@ -682,6 +670,9 @@ public void WaitForPruning() private ConcurrentQueue _commitSetQueue; + private ConcurrentQueue CommitSetQueue => + (_commitSetQueue ?? CreateQueueAtomic(ref _commitSetQueue)); + private long _memoryUsedByDirtyCache; private int _committedNodesCount; @@ -690,9 +681,8 @@ public void WaitForPruning() private long _latestPersistedBlockNumber; - private BlockCommitSet? CurrentPackage { get; set; } + private BlockCommitter? _currentBlockCommitter = null; - private bool IsCurrentListSealed => CurrentPackage is null || CurrentPackage.IsSealed; private long LatestCommittedBlockNumber { get; set; } public INodeStorage.KeyScheme Scheme => _nodeStorage.Scheme; @@ -705,22 +695,23 @@ private static ConcurrentQueue CreateQueueAtomic(ref ConcurrentQ return prior ?? instance; } - private void CreateCommitSet(long blockNumber) + private BlockCommitSet CreateCommitSet(long blockNumber) { if (_logger.IsDebug) _logger.Debug($"Beginning new {nameof(BlockCommitSet)} - {blockNumber}"); // TODO: this throws on reorgs, does it not? let us recreate it in test - Debug.Assert(CurrentPackage is null || blockNumber == CurrentPackage.BlockNumber + 1, "Newly begun block is not a successor of the last one"); - Debug.Assert(IsCurrentListSealed, "Not sealed when beginning new block"); + Debug.Assert(!CommitSetQueue.TryPeek(out BlockCommitSet lastSet) || blockNumber == lastSet.BlockNumber + 1, $"Newly begun block is not a successor of the last one."); + Debug.Assert(!CommitSetQueue.TryPeek(out lastSet) || lastSet.IsSealed, "Not sealed when beginning new block"); BlockCommitSet commitSet = new(blockNumber); - (_commitSetQueue ?? CreateQueueAtomic(ref _commitSetQueue)).Enqueue(commitSet); + CommitSetQueue.Enqueue(commitSet); LatestCommittedBlockNumber = Math.Max(blockNumber, LatestCommittedBlockNumber); + // Why are we announcing **before** committing next block?? + // Should it be after commit? AnnounceReorgBoundaries(); DequeueOldCommitSets(); - CurrentPackage = commitSet; - Debug.Assert(ReferenceEquals(CurrentPackage, commitSet), $"Current {nameof(BlockCommitSet)} is not same as the new package just after adding"); + return commitSet; } /// @@ -728,40 +719,10 @@ private void CreateCommitSet(long blockNumber) /// Already persisted nodes are skipped. After this action we are sure that the full state is available /// for the block represented by this commit set. /// - /// /// A commit set of a block which root is to be persisted. - /// The write batch to write to - /// Track persisted hashes in this dictionary if not null + /// Special action to be called on each persist. Used to track which node to remove. /// - private void PersistBlockCommitSet( - Hash256? address, - BlockCommitSet commitSet, - INodeStorage.WriteBatch writeBatch, - Action? persistedNodeRecorder = null, - WriteFlags writeFlags = WriteFlags.None - ) - { - void PersistNode(TrieNode tn, Hash256? address2, TreePath path) - { - persistedNodeRecorder?.Invoke(path, address2, tn); - this.PersistNode(address2, path, tn, commitSet.BlockNumber, writeFlags, writeBatch); - } - - if (_logger.IsDebug) _logger.Debug($"Persisting from root {commitSet.Root} in {commitSet.BlockNumber}"); - - long start = Stopwatch.GetTimestamp(); - TreePath path = TreePath.Empty; - commitSet.Root?.CallRecursively(PersistNode, address, ref path, GetTrieStore(null), true, _logger); - long elapsedMilliseconds = (long)Stopwatch.GetElapsedTime(start).TotalMilliseconds; - Metrics.SnapshotPersistenceTime = elapsedMilliseconds; - - if (_logger.IsDebug) _logger.Debug($"Persisted trie from {commitSet.Root} at {commitSet.BlockNumber} in {elapsedMilliseconds}ms (cache memory {MemoryUsedByDirtyCache})"); - - LastPersistedBlockNumber = commitSet.BlockNumber; - } - private void ParallelPersistBlockCommitSet( - Hash256? address, BlockCommitSet commitSet, Action? persistedNodeRecorder = null, WriteFlags writeFlags = WriteFlags.None @@ -777,7 +738,7 @@ void TopLevelPersist(TrieNode tn, Hash256? address2, TreePath path) if (path.Length < parallelBoundaryPathLength) { persistedNodeRecorder?.Invoke(path, address2, tn); - PersistNode(address2, path, tn, commitSet.BlockNumber, writeFlags, topLevelWriteBatch); + PersistNode(address2, path, tn, topLevelWriteBatch, writeFlags); } else { @@ -791,7 +752,7 @@ void TopLevelPersist(TrieNode tn, Hash256? address2, TreePath path) // The first CallRecursive stop at two level, yielding 256 node in parallelStartNodes, which is run concurrently TreePath path = TreePath.Empty; - commitSet.Root?.CallRecursively(TopLevelPersist, address, ref path, GetTrieStore(null), true, _logger, maxPathLength: parallelBoundaryPathLength); + commitSet.Root?.CallRecursively(TopLevelPersist, null, ref path, GetTrieStore(null), true, _logger, maxPathLength: parallelBoundaryPathLength); // The amount of change in the subtrees are not balanced at all. So their writes ares buffered here // which get disposed in parallel instead of being disposed in `PersistNodeStartingFrom`. @@ -815,7 +776,7 @@ void TopLevelPersist(TrieNode tn, Hash256? address2, TreePath path) Task.WaitAll(parallelStartNodes.Select(entry => Task.Run(() => { (TrieNode trieNode, Hash256? address2, TreePath path2) = entry; - PersistNodeStartingFrom(trieNode, address2, path2, commitSet, persistedNodeRecorder, writeFlags, disposeQueue); + PersistNodeStartingFrom(trieNode, address2, path2, persistedNodeRecorder, writeFlags, disposeQueue); })).ToArray()); disposeQueue.CompleteAdding(); @@ -832,7 +793,7 @@ void TopLevelPersist(TrieNode tn, Hash256? address2, TreePath path) LastPersistedBlockNumber = commitSet.BlockNumber; } - private void PersistNodeStartingFrom(TrieNode tn, Hash256 address2, TreePath path, BlockCommitSet commitSet, + private void PersistNodeStartingFrom(TrieNode tn, Hash256 address2, TreePath path, Action? persistedNodeRecorder, WriteFlags writeFlags, BlockingCollection disposeQueue) { @@ -842,7 +803,7 @@ private void PersistNodeStartingFrom(TrieNode tn, Hash256 address2, TreePath pat void DoPersist(TrieNode node, Hash256? address3, TreePath path2) { persistedNodeRecorder?.Invoke(path2, address3, node); - PersistNode(address3, path2, node, commitSet.BlockNumber, writeFlags, writeBatch); + PersistNode(address3, path2, node, writeBatch, writeFlags); persistedNodeCount++; if (persistedNodeCount % 512 == 0) @@ -856,23 +817,20 @@ void DoPersist(TrieNode node, Hash256? address3, TreePath path2) disposeQueue.Add(writeBatch); } - private void PersistNode(Hash256? address, in TreePath path, TrieNode currentNode, long blockNumber, WriteFlags writeFlags = WriteFlags.None, INodeStorage.WriteBatch? writeBatch = null) + private void PersistNode(Hash256? address, in TreePath path, TrieNode currentNode, INodeStorage.WriteBatch writeBatch, WriteFlags writeFlags = WriteFlags.None) { - writeBatch ??= _currentBatch ??= _nodeStorage.StartWriteBatch(); ArgumentNullException.ThrowIfNull(currentNode); if (currentNode.Keccak is not null) { - Debug.Assert(currentNode.LastSeen >= 0, $"Cannot persist a dangling node (without {(nameof(TrieNode.LastSeen))} value set)."); // Note that the LastSeen value here can be 'in the future' (greater than block number // if we replaced a newly added node with an older copy and updated the LastSeen value. // Here we reach it from the old root so it appears to be out of place but it is correct as we need // to prevent it from being removed from cache and also want to have it persisted. - if (_logger.IsTrace) _logger.Trace($"Persisting {nameof(TrieNode)} {currentNode} in snapshot {blockNumber}."); + if (_logger.IsTrace) _logger.Trace($"Persisting {nameof(TrieNode)} {currentNode}."); writeBatch.Set(address, path, currentNode.Keccak, currentNode.FullRlp, writeFlags); currentNode.IsPersisted = true; - currentNode.LastSeen = Math.Max(blockNumber, currentNode.LastSeen); IncrementPersistedNodesCount(); } else @@ -912,19 +870,6 @@ private void DequeueOldCommitSets() } } - private void EnsureCommitSetExistsForBlock(long blockNumber) - { - if (CurrentPackage is null) - { - if (_pruningStrategy.PruningEnabled && !Monitor.IsEntered(_dirtyNodesLock)) - { - Monitor.Enter(_dirtyNodesLock); - } - - CreateCommitSet(blockNumber); - } - } - private void AnnounceReorgBoundaries() { if (LatestCommittedBlockNumber < 1) @@ -993,7 +938,7 @@ private void PersistOnShutdown() { BlockCommitSet blockCommitSet = candidateSets[index]; if (_logger.IsDebug) _logger.Debug($"Persisting on disposal {blockCommitSet} (cache memory at {MemoryUsedByDirtyCache})"); - PersistBlockCommitSet(null, blockCommitSet, writeBatch); + ParallelPersistBlockCommitSet(blockCommitSet); } writeBatch.Dispose(); @@ -1031,10 +976,8 @@ void ClearCommitSetQueue() } commitSetCount++; - using INodeStorage.WriteBatch writeBatch = _nodeStorage.StartWriteBatch(); - PersistBlockCommitSet(null, commitSet, writeBatch); + ParallelPersistBlockCommitSet(commitSet); } - PruneCurrentSet(); } if (!(_commitSetQueue?.IsEmpty ?? true)) @@ -1130,19 +1073,50 @@ public bool HasRoot(Hash256 stateRoot) return true; } - private class TrieStoreCommitter( + private class BlockCommitter( + TrieStore trieStore, + BlockCommitSet commitSet + ) : IBlockCommitter + { + internal TrieNode? StateRoot = null; + private int _concurrency = trieStore._pruningStrategy.PruningEnabled ? Environment.ProcessorCount : 0; + + public void Dispose() + { + trieStore.FinishBlockCommit(commitSet, StateRoot); + } + + public ICommitter GetTrieCommitter(Hash256? address, TrieNode? root, WriteFlags writeFlags) + { + if (address is null) StateRoot = root; + return trieStore._pruningStrategy.PruningEnabled + ? new PruningTrieStoreCommitter(this, trieStore, commitSet.BlockNumber, address, root) + : new NonPruningTrieStoreCommitter(trieStore, address, trieStore._nodeStorage.StartWriteBatch(), writeFlags); + } + + public bool TryRequestConcurrencyQuota() + { + if (Interlocked.Decrement(ref _concurrency) >= 0) + { + return true; + } + + ReturnConcurrencyQuota(); + return false; + } + + public void ReturnConcurrencyQuota() => Interlocked.Increment(ref _concurrency); + } + + private class PruningTrieStoreCommitter( + BlockCommitter blockCommitter, TrieStore trieStore, - TrieType trieType, long blockNumber, Hash256? address, - TrieNode? root, - WriteFlags writeFlags, - int concurrency + TrieNode root ) : ICommitter { private readonly bool _needToResetRoot = root is not null && root.IsDirty; - private int _concurrency = concurrency; - private TrieNode? _root = root; public void Dispose() { @@ -1150,29 +1124,40 @@ public void Dispose() { // During commit it PatriciaTrie, the root may get resolved to an existing node (same keccak). // This ensure that the root that we use here is the same. - _root = trieStore.FindCachedOrUnknown(address, TreePath.Empty, _root?.Keccak); + // This is only needed for state tree as the root need to be put in the block commit set. + if (address == null) blockCommitter.StateRoot = trieStore.FindCachedOrUnknown(address, TreePath.Empty, root?.Keccak); } - - trieStore.FinishBlockCommit(trieType, blockNumber, address, _root, writeFlags); } public void CommitNode(ref TreePath path, NodeCommitInfo nodeCommitInfo) => - trieStore.CommitNode(blockNumber, address, ref path, nodeCommitInfo, writeFlags: writeFlags); + trieStore.CommitAndInsertToDirtyNodes(blockNumber, address, ref path, nodeCommitInfo); + + public bool TryRequestConcurrentQuota() => blockCommitter.TryRequestConcurrencyQuota(); - public bool CanSpawnTask() + public void ReturnConcurrencyQuota() => blockCommitter.ReturnConcurrencyQuota(); + } + + private class NonPruningTrieStoreCommitter( + TrieStore trieStore, + Hash256? address, + INodeStorage.WriteBatch writeBatch, + WriteFlags writeFlags = WriteFlags.None + ) : ICommitter + { + public void Dispose() { - if (Interlocked.Decrement(ref _concurrency) >= 0) - { - return true; - } + writeBatch.Dispose(); + } - ReturnConcurrencyQuota(); - return false; + public void CommitNode(ref TreePath path, NodeCommitInfo nodeCommitInfo) + { + trieStore.CommitAndPersistNode(address, ref path, nodeCommitInfo, writeFlags: writeFlags, writeBatch: writeBatch); } - public void ReturnConcurrencyQuota() => Interlocked.Increment(ref _concurrency); - } + public bool TryRequestConcurrentQuota() => false; + public void ReturnConcurrencyQuota() { } + } internal static class HashHelpers { diff --git a/src/Nethermind/Nethermind.Trie/TrieStoreWithReadFlags.cs b/src/Nethermind/Nethermind.Trie/TrieStoreWithReadFlags.cs index e93f1096d06..499f6f18501 100644 --- a/src/Nethermind/Nethermind.Trie/TrieStoreWithReadFlags.cs +++ b/src/Nethermind/Nethermind.Trie/TrieStoreWithReadFlags.cs @@ -10,8 +10,8 @@ namespace Nethermind.Trie; public class TrieStoreWithReadFlags(IScopedTrieStore implementation, ReadFlags flags) : TrieNodeResolverWithReadFlags(implementation, flags), IScopedTrieStore { - public ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => - implementation.BeginCommit(trieType, blockNumber, root, writeFlags); + public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => + implementation.BeginCommit(root, writeFlags); public bool IsPersisted(in TreePath path, in ValueHash256 keccak) => implementation.IsPersisted(in path, in keccak);