Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Perf/dont redownload downloaded code #6873

Merged
merged 7 commits into from
Mar 28, 2024
Merged

Perf/dont redownload downloaded code #6873

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a856c6b
Dont redownload downloaded code
asdacap Mar 26, 2024
08a030b
Add test
asdacap Mar 27, 2024
3aea262
Minor comment
asdacap Mar 27, 2024
2495d9d
Whitespace
asdacap Mar 27, 2024
35658f1
Clear code lru after snap finished
asdacap Mar 28, 2024
c23459b
Fix dispose
asdacap Mar 28, 2024
2457ae7
Merge remote-tracking branch 'origin/master' into perf/dont-redownloa…
asdacap Mar 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@

using System.Collections.Generic;
using System.Linq;
using FluentAssertions;
using Nethermind.Blockchain;
using Nethermind.Core;
using Nethermind.Core.Crypto;
using Nethermind.Core.Test.Builders;
using Nethermind.Db;
using Nethermind.Logging;
using Nethermind.State;
using Nethermind.State.Proofs;
using Nethermind.State.Snap;
using Nethermind.Synchronization.SnapSync;
using Nethermind.Trie;
using Nethermind.Trie.Pruning;
Expand Down Expand Up @@ -260,5 +263,52 @@ public void MissingAccountFromRange()
Assert.That(db.Keys.Count, Is.EqualTo(6));
Assert.IsFalse(db.KeyExists(rootHash));
}

[Test]
public void Will_not_redownload_persisted_code()
{
MemDb db = new();
MemDb codeDb = new();
DbProvider dbProvider = new();
dbProvider.RegisterDb(DbNames.State, db);
dbProvider.RegisterDb(DbNames.Code, codeDb);

BlockTree tree = Build.A.BlockTree().OfChainLength(5).TestObject;
using ProgressTracker progressTracker = new(tree, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance, accountRangePartitionCount: 1);
SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance);

PathWithAccount[] accountsWithPath =
[
new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001112345"),
new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[0])),
new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001113456"),
new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[1])),
new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001114567"),
new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[2])),
new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001123456"),
new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[3])),
new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001123457"),
new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[4]))
];

codeDb[TestItem.Keccaks[1].Bytes] = [1];
codeDb[TestItem.Keccaks[2].Bytes] = [1];

StateTree stateTree = new StateTree();
foreach (PathWithAccount pathWithAccount in accountsWithPath)
{
stateTree.Set(pathWithAccount.Path, pathWithAccount.Account);
}
stateTree.UpdateRootHash();

snapProvider.AddAccountRange(1,
stateTree.RootHash,
accountsWithPath[0].Path,
accountsWithPath);

progressTracker.IsFinished(out SnapSyncBatch nextRequest).Should().BeFalse();
progressTracker.IsFinished(out nextRequest).Should().BeFalse();
nextRequest.CodesRequest.Count.Should().Be(3);
}
}
}
17 changes: 16 additions & 1 deletion src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.Threading;
using Microsoft.Extensions.ObjectPool;
using Nethermind.Core;
using Nethermind.Core.Caching;
using Nethermind.Core.Collections;
using Nethermind.Core.Crypto;
using Nethermind.Core.Extensions;
Expand All @@ -29,6 +30,9 @@ public class SnapProvider : ISnapProvider

private readonly ProgressTracker _progressTracker;

// This is actually close to 97% effective.
private readonly LruKeyCache<ValueHash256> _codeExistKeyCache = new(1024 * 16, "");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

512kb of Keccaks; does/can this get dumped after sync?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can drop/clean it after the sync finishes.


public SnapProvider(ProgressTracker progressTracker, IDbProvider dbProvider, ILogManager logManager)
{
_dbProvider = dbProvider ?? throw new ArgumentNullException(nameof(dbProvider));
Expand Down Expand Up @@ -88,7 +92,18 @@ public AddRangeResult AddAccountRange(long blockNumber, in ValueHash256 expected
_progressTracker.EnqueueAccountStorage(item);
}

_progressTracker.EnqueueCodeHashes(CollectionsMarshal.AsSpan(codeHashes));

using ArrayPoolList<ValueHash256> filteredCodeHashes = codeHashes.AsParallel().Where((code) =>
{
if (_codeExistKeyCache.Get(code)) return false;

bool exist = _dbProvider.CodeDb.KeyExists(code.Bytes);
if (exist) _codeExistKeyCache.Set(code);
return !exist;
}).ToPooledList(codeHashes.Count);

_progressTracker.EnqueueCodeHashes(filteredCodeHashes.AsSpan());

_progressTracker.UpdateAccountRangePartitionProgress(effectiveHashLimit, accounts[^1].Path, moreChildrenToRight);
}
else if (result == AddRangeResult.MissingRootHashInProofs)
Expand Down