From a856c6bd2661b70069ed647601418e7a93a55d0b Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Tue, 26 Mar 2024 23:46:49 +0800 Subject: [PATCH 1/6] Dont redownload downloaded code --- .../SnapSync/SnapProvider.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index 6ef0027bd0a..0157c77e2f8 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -8,6 +8,7 @@ using System.Threading; using Microsoft.Extensions.ObjectPool; using Nethermind.Core; +using Nethermind.Core.Caching; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -28,6 +29,7 @@ public class SnapProvider : ISnapProvider private readonly ILogger _logger; private readonly ProgressTracker _progressTracker; + private readonly LruKeyCache _codeExistKeyCache = new(1024*16, ""); public SnapProvider(ProgressTracker progressTracker, IDbProvider dbProvider, ILogManager logManager) { @@ -88,7 +90,18 @@ public AddRangeResult AddAccountRange(long blockNumber, in ValueHash256 expected _progressTracker.EnqueueAccountStorage(item); } - _progressTracker.EnqueueCodeHashes(CollectionsMarshal.AsSpan(codeHashes)); + + using ArrayPoolList filteredCodeHashes = codeHashes.AsParallel().Where((code) => + { + if (_codeExistKeyCache.Get(code)) return false; + + bool exist = _dbProvider.CodeDb.KeyExists(code.Bytes); + if (exist) _codeExistKeyCache.Set(code); + return !exist; + }).ToPooledList(codeHashes.Count); + + _progressTracker.EnqueueCodeHashes(filteredCodeHashes.AsSpan()); + _progressTracker.UpdateAccountRangePartitionProgress(effectiveHashLimit, accounts[^1].Path, moreChildrenToRight); } else if (result == AddRangeResult.MissingRootHashInProofs) From 08a030baca71d7811c2bf8adb5d5af6187619edb Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 27 Mar 2024 22:23:04 +0800 Subject: [PATCH 2/6] Add test --- .../RecreateStateFromAccountRangesTests.cs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs index 9b4e65b8ad2..286215fd88e 100644 --- a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs +++ b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs @@ -5,6 +5,8 @@ using System.Collections.Generic; using System.Linq; +using FluentAssertions; +using Nethermind.Blockchain; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Test.Builders; @@ -12,6 +14,7 @@ using Nethermind.Logging; using Nethermind.State; using Nethermind.State.Proofs; +using Nethermind.State.Snap; using Nethermind.Synchronization.SnapSync; using Nethermind.Trie; using Nethermind.Trie.Pruning; @@ -260,5 +263,52 @@ public void MissingAccountFromRange() Assert.That(db.Keys.Count, Is.EqualTo(6)); Assert.IsFalse(db.KeyExists(rootHash)); } + + [Test] + public void Will_not_redownload_persisted_code() + { + MemDb db = new(); + MemDb codeDb = new(); + DbProvider dbProvider = new(); + dbProvider.RegisterDb(DbNames.State, db); + dbProvider.RegisterDb(DbNames.Code, codeDb); + + BlockTree tree = Build.A.BlockTree().OfChainLength(5).TestObject; + using ProgressTracker progressTracker = new(tree, dbProvider.GetDb(DbNames.State), LimboLogs.Instance, accountRangePartitionCount: 1); + SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + + PathWithAccount[] accountsWithPath = + [ + new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001112345"), + new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[0])), + new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001113456"), + new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[1])), + new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001114567"), + new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[2])), + new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001123456"), + new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[3])), + new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001123457"), + new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[4])) + ]; + + codeDb[TestItem.Keccaks[1].Bytes] = [1]; + codeDb[TestItem.Keccaks[2].Bytes] = [1]; + + StateTree stateTree = new StateTree(); + foreach (PathWithAccount pathWithAccount in accountsWithPath) + { + stateTree.Set(pathWithAccount.Path, pathWithAccount.Account); + } + stateTree.UpdateRootHash(); + + snapProvider.AddAccountRange(1, + stateTree.RootHash, + accountsWithPath[0].Path, + accountsWithPath); + + progressTracker.IsFinished(out SnapSyncBatch nextRequest).Should().BeFalse(); + progressTracker.IsFinished(out nextRequest).Should().BeFalse(); + nextRequest.CodesRequest.Count.Should().Be(3); + } } } From 3aea262f20e270db0f6f024bf51298a9b7fe59b3 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 27 Mar 2024 22:28:50 +0800 Subject: [PATCH 3/6] Minor comment --- .../Nethermind.Synchronization/SnapSync/SnapProvider.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index 0157c77e2f8..c33d52962dd 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -29,6 +29,8 @@ public class SnapProvider : ISnapProvider private readonly ILogger _logger; private readonly ProgressTracker _progressTracker; + + // This is actually close to 97% effective. private readonly LruKeyCache _codeExistKeyCache = new(1024*16, ""); public SnapProvider(ProgressTracker progressTracker, IDbProvider dbProvider, ILogManager logManager) From 2495d9dc319c28386b85cad2bd9ac4fa205c3e00 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 27 Mar 2024 22:35:44 +0800 Subject: [PATCH 4/6] Whitespace --- .../Nethermind.Synchronization/SnapSync/SnapProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index c33d52962dd..ca968647191 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -31,7 +31,7 @@ public class SnapProvider : ISnapProvider private readonly ProgressTracker _progressTracker; // This is actually close to 97% effective. - private readonly LruKeyCache _codeExistKeyCache = new(1024*16, ""); + private readonly LruKeyCache _codeExistKeyCache = new(1024 * 16, ""); public SnapProvider(ProgressTracker progressTracker, IDbProvider dbProvider, ILogManager logManager) { From 35658f1b72f437d31282bb299d26268ef7f417f7 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 28 Mar 2024 20:26:59 +0800 Subject: [PATCH 5/6] Clear code lru after snap finished --- .../Nethermind.Synchronization/SnapSync/ISnapProvider.cs | 1 + .../Nethermind.Synchronization/SnapSync/SnapProvider.cs | 5 +++++ .../Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs | 1 + 3 files changed, 7 insertions(+) diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapProvider.cs index c55ff822c79..ce1ae75d614 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapProvider.cs @@ -26,5 +26,6 @@ public interface ISnapProvider bool IsSnapGetRangesFinished(); void UpdatePivot(); + void Dispose(); } } diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index ca968647191..e31f83147f3 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -339,6 +339,11 @@ public void UpdatePivot() _progressTracker.UpdatePivot(); } + public void Dispose() + { + _codeExistKeyCache.Clear(); + } + private class TrieStorePoolPolicy : IPooledObjectPolicy { private readonly IKeyValueStoreWithBatching _stateDb; diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs index abcdf0b9f09..66146c9a209 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs @@ -205,6 +205,7 @@ public override void SyncModeSelectorOnChanged(SyncMode current) if (_disposed) return; if (CurrentState == SyncFeedState.Dormant) { + _snapProvider.Dispose(); if ((current & SyncMode.SnapSync) == SyncMode.SnapSync) { if (_snapProvider.CanSync()) From c23459b357b7fcb5ec372a8b8b9fb19c0afbfeb2 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 28 Mar 2024 21:04:55 +0800 Subject: [PATCH 6/6] Fix dispose --- .../Nethermind.Synchronization/ParallelSync/SyncFeed.cs | 2 +- .../Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs index 1d887c70c36..d3d258f80ec 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs @@ -36,7 +36,7 @@ private void ChangeState(SyncFeedState newState) public void Activate() => ChangeState(SyncFeedState.Active); - public void Finish() + public virtual void Finish() { ChangeState(SyncFeedState.Finished); GC.Collect(2, GCCollectionMode.Aggressive, true, true); diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs index 66146c9a209..aa56e36552f 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs @@ -205,7 +205,6 @@ public override void SyncModeSelectorOnChanged(SyncMode current) if (_disposed) return; if (CurrentState == SyncFeedState.Dormant) { - _snapProvider.Dispose(); if ((current & SyncMode.SnapSync) == SyncMode.SnapSync) { if (_snapProvider.CanSync()) @@ -216,6 +215,12 @@ public override void SyncModeSelectorOnChanged(SyncMode current) } } + public override void Finish() + { + _snapProvider.Dispose(); + base.Finish(); + } + public override bool IsFinished => _snapProvider.IsSnapGetRangesFinished(); } }