Skip to content

Commit

Permalink
Perf/parallelize storage commit (#7605)
Browse files Browse the repository at this point in the history 
Co-authored-by: Lukasz Rozmej <lukasz.rozmej@gmail.com>
  • Loading branch information
@asdacap @LukaszRozmej
asdacap and LukaszRozmej authored Oct 16, 2024
1 parent 4f0e596 commit b0e8c2c
Show file tree
Hide file tree
Showing 38 changed files with 680 additions and 570 deletions.
2 changes: 1 addition & 1 deletion src/Nethermind/Ethereum.Trie.Test/TrieTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ public void Delete_on_empty()
{
PatriciaTree patriciaTree = new PatriciaTree(_db, Keccak.EmptyTreeHash, false, true, NullLogManager.Instance);
patriciaTree.Set(Keccak.Compute("1").Bytes, new byte[0]);
patriciaTree.Commit(0);
patriciaTree.Commit();
Assert.That(patriciaTree.RootHash, Is.EqualTo(PatriciaTree.EmptyTreeHash));
}

Expand Down
56 changes: 28 additions & 28 deletions src/Nethermind/Nethermind.Benchmark/Store/PatriciaTreeBenchmarks.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,22 +56,22 @@ public class PatriciaTreeBenchmarks
tree.Set(TestItem.AddressA, _account0);
tree.Set(TestItem.AddressB, _account0);
tree.Set(TestItem.AddressC, _account0);
tree.Commit(1);
tree.Commit();
}),
("set_3_via_hash", tree =>
{
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0);
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0);
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0);
tree.Commit(1);
tree.Commit();
}),
("set_3_delete_1", tree =>
{
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0);
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0);
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0);
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null);
tree.Commit(1);
tree.Commit();
}),
("set_3_delete_2", tree =>
{
Expand All @@ -80,7 +80,7 @@ public class PatriciaTreeBenchmarks
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0);
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null);
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null);
tree.Commit(1);
tree.Commit();
}),
("set_3_delete_all", tree =>
{
Expand All @@ -90,7 +90,7 @@ public class PatriciaTreeBenchmarks
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null);
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null);
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), null);
tree.Commit(1);
tree.Commit();
}),
("extension_read_full_match", tree =>
{
Expand All @@ -99,7 +99,7 @@ public class PatriciaTreeBenchmarks
Account account = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111"));
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("extension_read_missing", tree =>
{
Expand All @@ -108,7 +108,7 @@ public class PatriciaTreeBenchmarks
Account account = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"));
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("extension_new_branch", tree =>
{
Expand All @@ -117,7 +117,7 @@ public class PatriciaTreeBenchmarks
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"), _account2);
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("extension_delete_missing", tree =>
{
Expand All @@ -126,7 +126,7 @@ public class PatriciaTreeBenchmarks
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeddddddddddddddddddddddddd"), null);
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("extenson_create_new_extension", tree =>
{
Expand All @@ -136,63 +136,63 @@ public class PatriciaTreeBenchmarks
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeaaaaaaaaaaaaaaaab11111111"), _account3);
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("leaf_new_value", tree =>
{
tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account0);
tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account1);
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("leaf_no_change", tree =>
{
tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account0);
tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account0);
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("leaf_delete", tree =>
{
tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account0);
tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), null);
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("leaf_delete_missing", tree =>
{
tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account0);
tree.Set(new Hash256("1111111111111111111111111111111ddddddddddddddddddddddddddddddddd"), null);
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("leaf_update_extension", tree =>
{
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111111111111111111111111111"), _account0);
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000000000000000000000000000"), _account1);
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("leaf_read", tree =>
{
tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account0);
Account account = tree.Get(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"));
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("leaf_update_missing", tree =>
{
tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account0);
Account account = tree.Get(new Hash256("111111111111111111111111111111111111111111111111111111111ddddddd"));
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("branch_update_missing", tree =>
{
Expand All @@ -201,7 +201,7 @@ public class PatriciaTreeBenchmarks
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), _account2);
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("branch_read_missing", tree =>
{
Expand All @@ -210,7 +210,7 @@ public class PatriciaTreeBenchmarks
Account account = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"));
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
("branch_delete_missing", tree =>
{
Expand All @@ -219,7 +219,7 @@ public class PatriciaTreeBenchmarks
tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), null);
tree.UpdateRootHash();
Hash256 rootHash = tree.RootHash;
tree.Commit(1);
tree.Commit();
}),
};

Expand Down Expand Up @@ -247,15 +247,15 @@ public void Setup()
{
tempTree.Set(_entries[i].Item1, _entries[i].Item2);
}
tempTree.Commit(0);
tempTree.Commit();
_rootHash = tempTree.RootHash;

_fullTree = new StateTree();
for (int i = 0; i < _entryCount; i++)
{
_fullTree.Set(_entries[i].Item1, _entries[i].Item2);
}
_fullTree.Commit(0);
_fullTree.Commit();

_uncommittedFullTree = new StateTree();
for (int i = 0; i < _entryCount; i++)
Expand Down Expand Up @@ -330,23 +330,23 @@ public void InsertAndCommit()
{
tempTree.Set(_entries[i].Item1, _entries[i].Item2);
}
tempTree.Commit(0);
tempTree.Commit();
}

[Benchmark]
public void InsertAndCommitRepeatedlyTimes()
{
StateTree tempTree = new StateTree(
new TrieStore(new MemDb(),
TrieStore trieStore = new TrieStore(new MemDb(),
Prune.WhenCacheReaches(1.MiB()),
Persist.IfBlockOlderThan(2),
NullLogManager.Instance), NullLogManager.Instance);
Persist.IfBlockOlderThan(2), NullLogManager.Instance);
StateTree tempTree = new StateTree(trieStore, NullLogManager.Instance);

for (int i = 0; i < _largerEntryCount; i++)
{
if (i % 2000 == 0)
{
tempTree.Commit(i / 2000);
using IBlockCommitter _ = trieStore.BeginBlockCommit(i / 2000);
tempTree.Commit();
}

(bool isWrite, Hash256 address, Account value) = _largerEntriesAccess[i];
Expand Down
12 changes: 6 additions & 6 deletions src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public static void FillStateTreeWithTestAccounts(StateTree stateTree)
stateTree.Set(AccountsWithPaths[3].Path, AccountsWithPaths[3].Account);
stateTree.Set(AccountsWithPaths[4].Path, AccountsWithPaths[4].Account);
stateTree.Set(AccountsWithPaths[5].Path, AccountsWithPaths[5].Account);
stateTree.Commit(0);
stateTree.Commit();
}

public static void FillStateTreeMultipleAccount(StateTree stateTree, int accountNumber)
Expand All @@ -76,7 +76,7 @@ public static void FillStateTreeMultipleAccount(StateTree stateTree, int account
Account acc = Build.An.Account.WithBalance((UInt256)i).TestObject;
stateTree.Set(Keccak.Compute(i.ToBigEndianByteArray()), acc);
}
stateTree.Commit(0);
stateTree.Commit();
}

public static (StateTree stateTree, StorageTree storageTree, Hash256 accountAddr) GetTrees(ITrieStore? store)
Expand All @@ -92,13 +92,13 @@ public static (StateTree stateTree, StorageTree storageTree, Hash256 accountAddr
storageTree.Set(SlotsWithPaths[4].Path, SlotsWithPaths[4].SlotRlpValue, false);
storageTree.Set(SlotsWithPaths[5].Path, SlotsWithPaths[5].SlotRlpValue, false);

storageTree.Commit(0);
storageTree.Commit();

var account = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject;

var stateTree = new StateTree(store.GetTrieStore(null), LimboLogs.Instance);
stateTree.Set(AccountAddress0, account);
stateTree.Commit(0);
stateTree.Commit();

return (stateTree, storageTree, AccountAddress0);
}
Expand All @@ -117,13 +117,13 @@ public static (StateTree stateTree, StorageTree storageTree, Hash256 accountAddr
false);
}

storageTree.Commit(0);
storageTree.Commit();

var account = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject;

var stateTree = new StateTree(store, LimboLogs.Instance);
stateTree.Set(AccountAddress0, account);
stateTree.Commit(0);
stateTree.Commit();

return (stateTree, storageTree, AccountAddress0);
}
Expand Down
2 changes: 1 addition & 1 deletion src/Nethermind/Nethermind.Core.Test/Builders/TrieBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public TrieBuilder WithAccountsByIndex(int start, int count)
TestObjectInternal.Set(key.Bytes, value);
}

TestObjectInternal.Commit(0);
TestObjectInternal.Commit();
TestObjectInternal.UpdateRootHash();

return this;
Expand Down
8 changes: 4 additions & 4 deletions src/Nethermind/Nethermind.JsonRpc.Test/Eip1186/ProofConverterTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,14 @@ public void Storage_proofs_have_values_set_complex_3_setup()
storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000")));
storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000")));
storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000")));
storageTree.Commit(0);
storageTree.Commit();

byte[] code = new byte[] { 1, 2, 3 };
Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject;
Account account2 = Build.An.Account.WithBalance(2).TestObject;
tree.Set(TestItem.AddressA, account1);
tree.Set(TestItem.AddressB, account2);
tree.Commit(0);
tree.Commit();

AccountProofCollector accountProofCollector = new(TestItem.AddressA, new byte[][] { a, b, c, d, e });
tree.Accept(accountProofCollector, tree.RootHash);
Expand All @@ -72,14 +72,14 @@ public void Does_not_fail_when_proofs_are_longer_than_number_of_proofs_regressio
storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000")));
storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000")));
storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000")));
storageTree.Commit(0);
storageTree.Commit();

byte[] code = new byte[] { 1, 2, 3 };
Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject;
Account account2 = Build.An.Account.WithBalance(2).TestObject;
tree.Set(TestItem.AddressA, account1);
tree.Set(TestItem.AddressB, account2);
tree.Commit(0);
tree.Commit();

AccountProofCollector accountProofCollector = new(TestItem.AddressA, new byte[][] { a });
tree.Accept(accountProofCollector, tree.RootHash);
Expand Down
14 changes: 7 additions & 7 deletions src/Nethermind/Nethermind.State.Test/PatriciaTreeTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ public void Create_commit_change_balance_get()
Account account = new(1);
StateTree stateTree = new();
stateTree.Set(TestItem.AddressA, account);
stateTree.Commit(0);
stateTree.Commit();

account = account.WithChangedBalance(2);
stateTree.Set(TestItem.AddressA, account);
stateTree.Commit(0);
stateTree.Commit();

Account accountRestored = stateTree.Get(TestItem.AddressA);
Assert.That(accountRestored.Balance, Is.EqualTo((UInt256)2));
Expand All @@ -41,11 +41,11 @@ public void Create_create_commit_change_balance_get()
StateTree stateTree = new();
stateTree.Set(TestItem.AddressA, account);
stateTree.Set(TestItem.AddressB, account);
stateTree.Commit(0);
stateTree.Commit();

account = account.WithChangedBalance(2);
stateTree.Set(TestItem.AddressA, account);
stateTree.Commit(0);
stateTree.Commit();

Account accountRestored = stateTree.Get(TestItem.AddressA);
Assert.That(accountRestored.Balance, Is.EqualTo((UInt256)2));
Expand All @@ -58,7 +58,7 @@ public void Create_commit_reset_change_balance_get()
Account account = new(1);
StateTree stateTree = new(new TrieStore(db, LimboLogs.Instance), LimboLogs.Instance);
stateTree.Set(TestItem.AddressA, account);
stateTree.Commit(0);
stateTree.Commit();

Hash256 rootHash = stateTree.RootHash;
stateTree.RootHash = null;
Expand All @@ -67,7 +67,7 @@ public void Create_commit_reset_change_balance_get()
stateTree.Get(TestItem.AddressA);
account = account.WithChangedBalance(2);
stateTree.Set(TestItem.AddressA, account);
stateTree.Commit(0);
stateTree.Commit();

Assert.That(db.Keys.Count, Is.EqualTo(2));
}
Expand All @@ -84,7 +84,7 @@ public void Commit_with_skip_root_should_skip_root(bool skipRoot, bool hasRoot)
stateTree.Set(TestItem.AddressA, account);
stateTree.UpdateRootHash();
Hash256 stateRoot = stateTree.RootHash;
stateTree.Commit(0, skipRoot);
stateTree.Commit(skipRoot);

if (hasRoot)
{
Expand Down
Loading

0 comments on commit b0e8c2c

Please sign in to comment.