Finally it works

src/Nethermind/Nethermind.State/StateReaderExtensions.cs

@@ -53,7 +53,7 @@ public static TrieStats CollectStats(this IStateReader stateProvider, Hash256 ro
             TrieStatsCollector collector = new(codeStorage, logManager, cancellationToken);
             stateProvider.RunTreeVisitor(collector, root, new VisitingOptions
             {
-                MaxDegreeOfParallelism = Environment.ProcessorCount,
+                MaxDegreeOfParallelism = 1,
                 FullScanMemoryBudget = 16.GiB(), // Gonna guess that if you are running this, you have a decent setup.
             });
             return collector.Stats;

src/Nethermind/Nethermind.Synchronization.Test/Nethermind.Synchronization.Test.csproj

@@ -29,8 +29,25 @@
 
   <ItemGroup>
     <None Remove="SnapSync\TestFixtures\badreq.json" />
-    <EmbeddedResource Include="SnapSync\TestFixtures\badreq.json">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    <None Remove="SnapSync\TestFixtures\badreq-roothash.zip" />
+    <EmbeddedResource Include="SnapSync\TestFixtures\badreq-roothash.zip">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </EmbeddedResource>
+    <None Remove="SnapSync\TestFixtures\badreq-roothash-2.zip" />
+    <EmbeddedResource Include="SnapSync\TestFixtures\badreq-roothash-2.zip">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </EmbeddedResource>
+    <None Remove="SnapSync\TestFixtures\badreq-roothash-3.zip" />
+    <EmbeddedResource Include="SnapSync\TestFixtures\badreq-roothash-3.zip">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </EmbeddedResource>
+    <None Remove="SnapSync\TestFixtures\badreq-trieexception.zip" />
+    <EmbeddedResource Include="SnapSync\TestFixtures\badreq-trieexception.zip">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </EmbeddedResource>
+    <None Remove="SnapSync\TestFixtures\badreq-roothash-4.zip" />
+    <EmbeddedResource Include="SnapSync\TestFixtures\badreq-roothash-4.zip">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </EmbeddedResource>
   </ItemGroup>
 

src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs

@@ -16,10 +16,14 @@
 using System.Collections.Generic;
 using System.IO;
 using System.IO.Abstractions;
+using System.IO.Compression;
 using System.Linq;
 using System.Text.Json;
 using Nethermind.Core;
+using Nethermind.Core.Test;
 using Nethermind.Serialization.Rlp;
+using Nethermind.State;
+using Nethermind.Trie.Pruning;
 
 namespace Nethermind.Synchronization.Test.SnapSync;
 
@@ -65,18 +69,21 @@ public void AddAccountRange_ResponseHasEmptyListOfAccountsAndOneProof_ReturnsExp
         sut.AddAccountRange(accountRange, accountsAndProofs).Should().Be(AddRangeResult.ExpiredRootHash);
     }
 
-    [Test]
-    public void TestStrangeCase()
+    [TestCase("badreq-roothash.zip")]
+    [TestCase("badreq-roothash-2.zip")]
+    [TestCase("badreq-roothash-3.zip")]
+    [TestCase("badreq-roothash-4.zip")]
+    [TestCase("badreq-trieexception.zip")]
+    public void TestStrangeCase2(string testFileName)
     {
-        string asStr = new StreamReader(GetType().Assembly.GetManifestResourceStream("Nethermind.Synchronization.Test.SnapSync.TestFixtures.badreq.json")!).ReadToEnd();
+        DeflateStream decompressor =
+            new DeflateStream(
+                GetType().Assembly
+                    .GetManifestResourceStream($"Nethermind.Synchronization.Test.SnapSync.TestFixtures.{testFileName}")!,
+                CompressionMode.Decompress);
+        string asStr = new StreamReader(decompressor).ReadToEnd();
         BadReq asReq = JsonSerializer.Deserialize<BadReq>(asStr)!;
 
-        MemDb db = new();
-        IDbProvider dbProvider = new DbProvider();
-        dbProvider.RegisterDb(DbNames.State, db);
-        dbProvider.RegisterDb(DbNames.Code, new MemDb());
-        using ProgressTracker progressTracker = new(Substitute.For<IBlockTree>(), dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
-
         AccountDecoder acd = new AccountDecoder();
         Account[] accounts = asReq.Accounts.Select((bt) => acd.Decode(new RlpStream(Bytes.FromHexString(bt)))!).ToArray();
         ValueHash256[] paths = asReq.Paths.Select((bt) => new ValueHash256(Bytes.FromHexString(bt))).ToArray();
@@ -86,10 +93,16 @@ public void TestStrangeCase()
         accountsAndProofs.PathAndAccounts = accounts.Select((acc, idx) => new PathWithAccount(paths[idx], acc)).ToPooledList(1);
         accountsAndProofs.Proofs = asReq.Proofs.Select((str) => Bytes.FromHexString(str)).ToPooledList(1);
 
-        SnapProvider sut = new(progressTracker, dbProvider.CodeDb, new NodeStorage(dbProvider.StateDb), LimboLogs.Instance);
-
-        sut.AddAccountRange(accountRange, accountsAndProofs).Should().Be(AddRangeResult.ExpiredRootHash);
-
+        StateTree stree = new StateTree(new TrieStore(new TestMemDb(), LimboLogs.Instance), LimboLogs.Instance);
+        SnapProviderHelper.AddAccountRange(
+                stree,
+                0,
+                accountRange.RootHash,
+                accountRange.StartingHash,
+                accountRange.LimitHash!.Value,
+                accountsAndProofs.PathAndAccounts,
+                accountsAndProofs.Proofs,
+                null).result.Should().Be(AddRangeResult.OK);
     }
 
     private record BadReq(

src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs

@@ -89,7 +89,7 @@ private void SetupAccountRangePartition()
             // The mismatch happens on exactly the same partition every time, suggesting tome kind of boundary issues
             // either on proof generation or validation.
             uint curStartingPath = 0;
-            uint partitionSize = (uint.MaxValue / (uint)_accountRangePartitionCount);
+            uint partitionSize = (uint)(((ulong)uint.MaxValue + 1) / (uint)_accountRangePartitionCount);
 
             for (var i = 0; i < _accountRangePartitionCount; i++)
             {

src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs

@@ -2,8 +2,10 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Threading;
@@ -16,6 +18,7 @@
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Extensions;
 using Nethermind.Db;
+using Nethermind.Int256;
 using Nethermind.Logging;
 using Nethermind.Serialization.Rlp;
 using Nethermind.State;
@@ -61,7 +64,6 @@ public AddRangeResult AddAccountRange(AccountRange request, AccountsAndProofs re
 
             if (response.PathAndAccounts.Count == 0)
             {
-                _logger.Trace($"SNAP - GetAccountRange - requested expired RootHash:{request.RootHash}");
 
                 result = AddRangeResult.ExpiredRootHash;
             }
@@ -81,6 +83,33 @@ public AddRangeResult AddAccountRange(AccountRange request, AccountsAndProofs re
                     {
                         Interlocked.Add(ref Metrics.SnapSyncedAccounts, response.PathAndAccounts.Count);
                     }
+                    else
+                    {
+                        _logger.Warn($"Resutl is {result} from {peerInfo} " +
+                                     $"{peerInfo?.PeerClientType}. Req " +
+                                     $"{request.StartingHash} " +
+                                     $"{request.LimitHash} Resp " +
+                                     $"{response.PathAndAccounts[^1].Path} " +
+                                     $"{request.LimitHash} Resp {request.RootHash} ");
+
+                        List<string> proofs = response.Proofs.Select(p => p.ToHexString()).ToList();
+                        List<string> paths = response.PathAndAccounts.Select(p => p.Path.ToString()).ToList();
+                        AccountDecoder acd = new AccountDecoder();
+                        List<string> accounts = response.PathAndAccounts.Select(p => acd.Encode(p.Account).Bytes.ToHexString()).ToList();
+
+                        string jsonified = JsonSerializer.Serialize(new BadReq(
+                            request.RootHash.ToString(), request.StartingHash.ToString(), request.LimitHash.ToString(),
+                            proofs, paths, accounts
+                        ));
+
+                        string fileName = $"badreq-{result}-{Random.Shared.Next()}.zip";
+                        using FileStream fileOutStream = File.OpenWrite(fileName);
+                        using DeflateStream deflateStream = new DeflateStream(fileOutStream, CompressionLevel.Optimal);
+                        using StreamWriter streamWriter = new StreamWriter(deflateStream);
+                        streamWriter.Write(jsonified);
+                        streamWriter.Close();
+                        _logger.Warn($"Dumped to {fileName}");
+                    }
                 }
                 catch (MissingTrieNodeException)
                 {
@@ -89,8 +118,25 @@ public AddRangeResult AddAccountRange(AccountRange request, AccountsAndProofs re
                                  $"{request.StartingHash} " +
                                  $"{request.LimitHash} Resp " +
                                  $"{response.PathAndAccounts[^1].Path} " +
-                                 $"{response.PathAndAccounts[0].Path} " +
-                                 $"{response.PathAndAccounts.Count} ");
+                                 $"{request.LimitHash} Resp {request.RootHash} ");
+
+                    List<string> proofs = response.Proofs.Select(p => p.ToHexString()).ToList();
+                    List<string> paths = response.PathAndAccounts.Select(p => p.Path.ToString()).ToList();
+                    AccountDecoder acd = new AccountDecoder();
+                    List<string> accounts = response.PathAndAccounts.Select(p => acd.Encode(p.Account).Bytes.ToHexString()).ToList();
+
+                    string jsonified = JsonSerializer.Serialize(new BadReq(
+                        request.RootHash.ToString(), request.StartingHash.ToString(), request.LimitHash.ToString(),
+                        proofs, paths, accounts
+                    ));
+
+                    string fileName = $"badreq-trieexception-{Random.Shared.Next()}.zip";
+                    using FileStream fileOutStream = File.OpenWrite(fileName);
+                    using DeflateStream deflateStream = new DeflateStream(fileOutStream, CompressionLevel.Optimal);
+                    using StreamWriter streamWriter = new StreamWriter(deflateStream);
+                    streamWriter.Write(jsonified);
+                    streamWriter.Close();
+
                     result = AddRangeResult.InternalError;
                 }
             }
@@ -149,7 +195,11 @@ public AddRangeResult AddAccountRange(
 
                     _progressTracker.EnqueueCodeHashes(filteredCodeHashes.AsSpan());
 
-                    _progressTracker.UpdateAccountRangePartitionProgress(effectiveHashLimit, accounts[^1].Path, moreChildrenToRight);
+                    UInt256 nextPath = new UInt256(accounts[^1].Path.Bytes, true);
+                    nextPath += UInt256.One;
+                    ValueHash256 asValueHash = new ValueHash256(nextPath.ToBigEndian());
+
+                    _progressTracker.UpdateAccountRangePartitionProgress(effectiveHashLimit, asValueHash, moreChildrenToRight);
                 }
                 else if (result == AddRangeResult.MissingRootHashInProofs)
                 {
@@ -254,6 +304,10 @@ public AddRangeResult AddStorageRange(long blockNumber, PathWithAccount pathWith
 
                     _progressTracker.EnqueueAccountRefresh(pathWithAccount, startingHash);
                 }
+                else if (result == AddRangeResult.ExpiredRootHash)
+                {
+                    _logger.Warn($"Exxpired root hash does happens");
+                }
 
                 return result;
             }

src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs

@@ -50,7 +50,7 @@ public static (AddRangeResult result, bool moreChildrenToRight, List<PathWithAcc
             for (var index = 0; index < accounts.Count; index++)
             {
                 PathWithAccount account = accounts[index];
-                if (account.Account.HasStorage)
+                if (account.Account.HasStorage && account.Path < limitHash)
                 {
                     accountsWithStorage.Add(account);
                 }
@@ -190,8 +190,10 @@ private static (AddRangeResult result, List<(TrieNode, TreePath)> sortedBoundary
                         }
                         else
                         {
-                            Span<byte> pathSpan = CollectionsMarshal.AsSpan(path);
-                            if (Bytes.BytesComparer.Compare(pathSpan, leftBoundary[0..path.Count]) >= 0
+                            TreePath extensionChildPath = TreePath.FromNibble(CollectionsMarshal.AsSpan(path));
+                            extensionChildPath = extensionChildPath.Append(node.Key);
+                            TreePath firstKeyPath = TreePath.FromPath(effectiveStartingHAsh.Bytes).Truncate(extensionChildPath.Length);
+                            if (extensionChildPath.CompareTo(firstKeyPath) >= 0
                                 && parent is not null
                                 && parent.IsBranch)
                             {
@@ -245,6 +247,26 @@ private static (AddRangeResult result, List<(TrieNode, TreePath)> sortedBoundary
                                 proofNodesToProcess.Push((node, child, pathIndex, newPath));
                                 sortedBoundaryList.Add((child, TreePath.FromNibble(CollectionsMarshal.AsSpan(newPath))));
                             }
+                            else
+                            {
+                                // Sometimes a leaf becomes a proof.
+                                // we add them.
+                                TreePath tPath = TreePath.FromNibble(CollectionsMarshal.AsSpan(path));
+                                tPath.AppendMut(ci);
+
+                                List<byte> newPath = new(path)
+                                {
+                                    (byte)ci
+                                };
+
+                                TreePath wholePath = tPath.Append(child.Key);
+                                if (wholePath.Path < startingHash || wholePath.Path > endHash)
+                                {
+                                    node.SetChild(ci, child);
+                                    proofNodesToProcess.Push((node, child, pathIndex, newPath));
+                                    sortedBoundaryList.Add((child, tPath));
+                                }
+                            }
                         }
                     }
                 }

src/Nethermind/Nethermind.Trie/Pruning/TreePath.cs

@@ -136,7 +136,7 @@ internal void AppendMut(ReadOnlySpan<byte> nibbles)
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    internal void AppendMut(int nib)
+    public void AppendMut(int nib)
     {
         this[Length] = nib;
         Length++;

src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs

@@ -10,7 +10,7 @@
 
 namespace Nethermind.Trie
 {
-    public class TrieStatsCollector : ITreeVisitor
+    public class TrieStatsCollector : ITreeVisitor<TreePathContextWithStorage>
     {
         private readonly ClockCache<ValueHash256, int> _existingCodeHash = new ClockCache<ValueHash256, int>(1024 * 8);
         private readonly IKeyValueStore _codeKeyValueStore;
@@ -29,16 +29,18 @@ public TrieStatsCollector(IKeyValueStore codeKeyValueStore, ILogManager logManag
         public TrieStats Stats { get; } = new();
 
         public bool IsFullDbScan => true;
+        public void VisitTree(in TreePathContextWithStorage nodeContext, Hash256 rootHash, TrieVisitContext trieVisitContext)
+        {
+        }
 
-        public bool ShouldVisit(Hash256 nextNode)
+        public bool ShouldVisit(in TreePathContextWithStorage nodeContext, Hash256 nextNode)
         {
             return true;
         }
 
-        public void VisitTree(Hash256 rootHash, TrieVisitContext trieVisitContext) { }
-
-        public void VisitMissingNode(Hash256 nodeHash, TrieVisitContext trieVisitContext)
+        public void VisitMissingNode(in TreePathContextWithStorage nodeContext, Hash256 nodeHash, TrieVisitContext trieVisitContext)
         {
+            _logger.Warn($"Missing node. S {nodeContext.Storage} P {nodeContext.Path} H {nodeHash}");
             if (trieVisitContext.IsStorage)
             {
                 Interlocked.Increment(ref Stats._missingStorage);
@@ -51,7 +53,7 @@ public void VisitMissingNode(Hash256 nodeHash, TrieVisitContext trieVisitContext
             IncrementLevel(trieVisitContext);
         }
 
-        public void VisitBranch(TrieNode node, TrieVisitContext trieVisitContext)
+        public void VisitBranch(in TreePathContextWithStorage nodeContext, TrieNode node, TrieVisitContext trieVisitContext)
         {
             _cancellationToken.ThrowIfCancellationRequested();
 
@@ -69,7 +71,7 @@ public void VisitBranch(TrieNode node, TrieVisitContext trieVisitContext)
             IncrementLevel(trieVisitContext);
         }
 
-        public void VisitExtension(TrieNode node, TrieVisitContext trieVisitContext)
+        public void VisitExtension(in TreePathContextWithStorage nodeContext, TrieNode node, TrieVisitContext trieVisitContext)
         {
             if (trieVisitContext.IsStorage)
             {
@@ -85,7 +87,7 @@ public void VisitExtension(TrieNode node, TrieVisitContext trieVisitContext)
             IncrementLevel(trieVisitContext);
         }
 
-        public void VisitLeaf(TrieNode node, TrieVisitContext trieVisitContext, ReadOnlySpan<byte> value)
+        public void VisitLeaf(in TreePathContextWithStorage nodeContext, TrieNode node, TrieVisitContext trieVisitContext, ReadOnlySpan<byte> value)
         {
             long lastAccountNodeCount = _lastAccountNodeCount;
             long currentNodeCount = Stats.NodesCount;
@@ -108,7 +110,7 @@ public void VisitLeaf(TrieNode node, TrieVisitContext trieVisitContext, ReadOnly
             IncrementLevel(trieVisitContext);
         }
 
-        public void VisitCode(Hash256 codeHash, TrieVisitContext trieVisitContext)
+        public void VisitCode(in TreePathContextWithStorage nodeContext, Hash256 codeHash, TrieVisitContext trieVisitContext)
         {
             ValueHash256 key = new ValueHash256(codeHash.Bytes);
             bool codeExist = _existingCodeHash.TryGet(key, out int codeLength);