Feature/debug get bad blocks (#3838)

Co-authored-by: lukasz.rozmej <lukasz.rozmej@gmail.com> Co-authored-by: Marc <Marchhill@users.noreply.github.com>
NethermindEth · Jan 5, 2024 · 9b5a3f2 · 9b5a3f2
1 parent 7d2958f
commit 9b5a3f2
Show file tree

Hide file tree

Showing 23 changed files with 191 additions and 59 deletions.
diff --git a/src/Nethermind/Nethermind.Api/IApiWithStores.cs b/src/Nethermind/Nethermind.Api/IApiWithStores.cs
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using Nethermind.Blockchain;
+using Nethermind.Blockchain.Blocks;
 using Nethermind.Blockchain.Find;
 using Nethermind.Blockchain.Receipts;
 using Nethermind.Consensus;
@@ -27,5 +28,6 @@ public interface IApiWithStores : IBasicApi
         IReceiptFinder? ReceiptFinder { get; set; }
         IReceiptMonitor? ReceiptMonitor { get; set; }
         IWallet? Wallet { get; set; }
+        IBlockStore? BadBlocksStore { get; set; }
     }
 }
diff --git a/src/Nethermind/Nethermind.Api/IInitConfig.cs b/src/Nethermind/Nethermind.Api/IInitConfig.cs
@@ -74,6 +74,9 @@ public interface IInitConfig : IConfig
     [ConfigItem(Description = "The hint on the max memory limit, in bytes, to configure the database and networking memory allocations.", DefaultValue = "null")]
     long? MemoryHint { get; set; }
 
+    [ConfigItem(Description = "The maximum number of bad blocks observed on the network that will be stored to disk.", DefaultValue = "100")]
+    long? BadBlocksStored { get; set; }
+
     [ConfigItem(Description = "[TECHNICAL] Disable garbage collector on newPayload", DefaultValue = "true", HiddenFromDocs = true)]
     bool DisableGcOnNewPayload { get; set; }
 

diff --git a/src/Nethermind/Nethermind.Api/InitConfig.cs b/src/Nethermind/Nethermind.Api/InitConfig.cs
@@ -31,6 +31,7 @@ public class InitConfig : IInitConfig
 
         public string RpcDbUrl { get; set; } = String.Empty;
         public long? MemoryHint { get; set; }
+        public long? BadBlocksStored { get; set; } = 100;
         public bool DisableGcOnNewPayload { get; set; } = true;
         public bool DisableMallocOpts { get; set; } = false;
         public long? ExitOnBlockNumber { get; set; } = null;

diff --git a/src/Nethermind/Nethermind.Api/NethermindApi.cs b/src/Nethermind/Nethermind.Api/NethermindApi.cs
@@ -7,6 +7,7 @@
 using Nethermind.Abi;
 using Nethermind.Api.Extensions;
 using Nethermind.Blockchain;
+using Nethermind.Blockchain.Blocks;
 using Nethermind.Blockchain.Filters;
 using Nethermind.Blockchain.Find;
 using Nethermind.Blockchain.FullPruning;
@@ -210,6 +211,7 @@ public ISealEngine SealEngine
         public IEthSyncingInfo? EthSyncingInfo { get; set; }
         public IBlockProductionPolicy? BlockProductionPolicy { get; set; }
         public IWallet? Wallet { get; set; }
+        public IBlockStore? BadBlocksStore { get; set; }
         public ITransactionComparerProvider? TransactionComparerProvider { get; set; }
         public IWebSocketsManager WebSocketsManager { get; set; } = new WebSocketsManager();
 

diff --git a/src/Nethermind/Nethermind.Blockchain/BlockTree.cs b/src/Nethermind/Nethermind.Blockchain/BlockTree.cs
@@ -43,6 +43,7 @@ public partial class BlockTree : IBlockTree
         private readonly IHeaderStore _headerStore;
         private readonly IDb _blockInfoDb;
         private readonly IDb _metadataDb;
+        private readonly IBlockStore _badBlockStore;
 
         private readonly LruCache<ValueHash256, Block> _invalidBlocks =
             new(128, 128, "invalid blocks");
@@ -110,6 +111,7 @@ public BlockTree(
             IHeaderStore? headerDb,
             IDb? blockInfoDb,
             IDb? metadataDb,
+            IBlockStore? badBlockStore,
             IChainLevelInfoRepository? chainLevelInfoRepository,
             ISpecProvider? specProvider,
             IBloomStorage? bloomStorage,
@@ -121,6 +123,7 @@ public BlockTree(
             _headerStore = headerDb ?? throw new ArgumentNullException(nameof(headerDb));
             _blockInfoDb = blockInfoDb ?? throw new ArgumentNullException(nameof(blockInfoDb));
             _metadataDb = metadataDb ?? throw new ArgumentNullException(nameof(metadataDb));
+            _badBlockStore = badBlockStore ?? throw new ArgumentNullException(nameof(badBlockStore));
             _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider));
             _bloomStorage = bloomStorage ?? throw new ArgumentNullException(nameof(bloomStorage));
             _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig));
@@ -710,6 +713,7 @@ public void DeleteInvalidBlock(Block invalidBlock)
             if (_logger.IsDebug) _logger.Debug($"Deleting invalid block {invalidBlock.ToString(Block.Format.FullHashAndNumber)}");
 
             _invalidBlocks.Set(invalidBlock.Hash, invalidBlock);
+            _badBlockStore.Insert(invalidBlock);
 
             BestSuggestedHeader = Head?.Header;
             BestSuggestedBody = Head;

diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs
@@ -3,6 +3,8 @@
 
 using System;
 using System.Buffers;
+using System.Collections.Generic;
+using System.Linq;
 using Nethermind.Core;
 using Nethermind.Core.Caching;
 using Nethermind.Core.Crypto;
@@ -20,10 +22,12 @@ public class BlockStore : IBlockStore
 
     private readonly LruCache<ValueHash256, Block>
         _blockCache = new(CacheSize, CacheSize, "blocks");
+    private long? _maxSize;
 
-    public BlockStore(IDb blockDb)
+    public BlockStore(IDb blockDb, long? maxSize = null)
     {
         _blockDb = blockDb;
+        _maxSize = maxSize;
     }
 
     public void SetMetadata(byte[] key, byte[] value)
@@ -36,6 +40,18 @@ public void SetMetadata(byte[] key, byte[] value)
         return _blockDb.Get(key);
     }
 
+    private void TruncateToMaxSize()
+    {
+        int toDelete = (int)(_blockDb.GetSize() - _maxSize!);
+        if (toDelete > 0)
+        {
+            foreach (var blockToDelete in GetAll().Take(toDelete))
+            {
+                Delete(blockToDelete.Number, blockToDelete.Hash);
+            }
+        }
+    }
+
     public void Insert(Block block, WriteFlags writeFlags = WriteFlags.None)
     {
         if (block.Hash is null)
@@ -48,6 +64,11 @@ public void Insert(Block block, WriteFlags writeFlags = WriteFlags.None)
         using NettyRlpStream newRlp = _blockDecoder.EncodeToNewNettyStream(block);
 
         _blockDb.Set(block.Number, block.Hash, newRlp.AsSpan(), writeFlags);
+
+        if (_maxSize is not null)
+        {
+            TruncateToMaxSize();
+        }
     }
 
     private static void GetBlockNumPrefixedKey(long blockNumber, Hash256 blockHash, Span<byte> output)
@@ -88,4 +109,9 @@ public void Cache(Block block)
     {
         _blockCache.Set(block.Hash, block);
     }
+
+    public IEnumerable<Block> GetAll()
+    {
+        return _blockDb.GetAllValues(true).Select(bytes => _blockDecoder.Decode(bytes.AsRlpStream()));
+    }
 }
diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs
@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Collections.Generic;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Serialization.Rlp;
@@ -16,6 +17,7 @@ public interface IBlockStore
     void Insert(Block block, WriteFlags writeFlags = WriteFlags.None);
     void Delete(long blockNumber, Hash256 blockHash);
     Block? Get(long blockNumber, Hash256 blockHash, bool shouldCache = true);
+    IEnumerable<Block> GetAll();
     ReceiptRecoveryBlock? GetReceiptRecoveryBlock(long blockNumber, Hash256 blockHash);
     void Cache(Block block);
 

diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeBuilder.cs b/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeBuilder.cs
@@ -47,6 +47,7 @@ public BlockTreeBuilder(Block genesisBlock, ISpecProvider specProvider)
             BlockNumbersDb = new TestMemDb();
             BlockInfoDb = new TestMemDb();
             MetadataDb = new TestMemDb();
+            BadBlocksDb = new TestMemDb();
 
             _genesisBlock = genesisBlock;
             _specProvider = specProvider;
@@ -79,6 +80,7 @@ public BlockTree BlockTree
                         HeaderStore,
                         BlockInfoDb,
                         MetadataDb,
+                        BadBlockStore,
                         ChainLevelInfoRepository,
                         _specProvider,
                         BloomStorage,
@@ -105,6 +107,7 @@ protected override void BeforeReturn()
         public ISyncConfig SyncConfig { get; set; } = new SyncConfig();
 
         public IDb BlocksDb { get; set; }
+        public IDb BadBlocksDb { get; set; }
 
         private IBlockStore? _blockStore;
         public IBlockStore BlockStore
@@ -139,6 +142,18 @@ public IHeaderStore HeaderStore
 
         public IDb MetadataDb { get; set; }
 
+        private IBlockStore? _badBlockStore;
+        public IBlockStore BadBlockStore
+        {
+            get
+            {
+                return _badBlockStore ??= new BlockStore(BadBlocksDb, 100);
+            }
+            set
+            {
+                _badBlockStore = value;
+            }
+        }
         private IChainLevelInfoRepository? _chainLevelInfoRepository;
 
         public IChainLevelInfoRepository ChainLevelInfoRepository
@@ -384,6 +399,18 @@ public BlockTreeBuilder WithBlockStore(IBlockStore blockStore)
             return this;
         }
 
+        public BlockTreeBuilder WithBadBlockStore(IBlockStore blockStore)
+        {
+            BadBlockStore = blockStore;
+            return this;
+        }
+
+        public BlockTreeBuilder WithHeaderStore(IHeaderStore headerStore)
+        {
+            HeaderStore = headerStore;
+            return this;
+        }
+
         public BlockTreeBuilder WithBlocksDb(IDb blocksDb)
         {
             BlocksDb = blocksDb;

diff --git a/src/Nethermind/Nethermind.Core/Caching/LruCache.cs b/src/Nethermind/Nethermind.Core/Caching/LruCache.cs
@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
+using System.Runtime.CompilerServices;
 using Nethermind.Core.Extensions;
 using Nethermind.Core.Threading;
 
@@ -146,6 +147,19 @@ public KeyValuePair<TKey, TValue>[] ToArray()
             return array;
         }
 
+        [MethodImpl(MethodImplOptions.Synchronized)]
+        public TValue[] GetValues()
+        {
+            int i = 0;
+            TValue[] array = new TValue[_cacheMap.Count];
+            foreach (KeyValuePair<TKey, LinkedListNode<LruCacheItem>> kvp in _cacheMap)
+            {
+                array[i++] = kvp.Value.Value.Value;
+            }
+
+            return array;
+        }
+
         private void Replace(TKey key, TValue value)
         {
             LinkedListNode<LruCacheItem>? node = _leastRecentlyUsed;

diff --git a/src/Nethermind/Nethermind.Db/DbNames.cs b/src/Nethermind/Nethermind.Db/DbNames.cs
@@ -13,6 +13,7 @@ public static class DbNames
         public const string BlockNumbers = "blockNumbers";
         public const string Receipts = "receipts";
         public const string BlockInfos = "blockInfos";
+        public const string BadBlocks = "badBlocks";
         public const string Bloom = "bloom";
         public const string Witness = "witness";
         public const string CHT = "canonicalHashTrie";

diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs
@@ -22,6 +22,7 @@ public interface IDbProvider : IDisposable
         public IDb HeadersDb => GetDb<IDb>(DbNames.Headers);
         public IDb BlockNumbersDb => GetDb<IDb>(DbNames.BlockNumbers);
         public IDb BlockInfosDb => GetDb<IDb>(DbNames.BlockInfos);
+        public IDb BadBlocksDb => GetDb<IDb>(DbNames.BadBlocks);
 
         // BloomDB progress / config (does not contain blooms - they are kept in bloom storage)
         public IDb BloomDb => GetDb<IDb>(DbNames.Bloom);

diff --git a/src/Nethermind/Nethermind.Db/Metrics.cs b/src/Nethermind/Nethermind.Db/Metrics.cs
@@ -126,6 +126,14 @@ public static class Metrics
         [Description("Number of Metadata DB writes.")]
         public static long MetadataDbWrites { get; set; }
 
+        [CounterMetric]
+        [Description("Number of BadBlocks DB writes.")]
+        public static long BadBlocksDbWrites { get; set; }
+
+        [CounterMetric]
+        [Description("Number of BadBlocks DB reads.")]
+        public static long BadBlocksDbReads { get; set; }
+
         [CounterMetric]
         [Description("Number of BlobTransactions DB reads.")]
         public static long BlobTransactionsDbReads { get; set; }

diff --git a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs
@@ -41,6 +41,7 @@ private void RegisterAll(bool useReceiptsDb, bool useBlobsDb)
             RegisterDb(BuildRocksDbSettings(DbNames.Headers, () => Metrics.HeaderDbReads++, () => Metrics.HeaderDbWrites++));
             RegisterDb(BuildRocksDbSettings(DbNames.BlockNumbers, () => Metrics.BlockNumberDbReads++, () => Metrics.BlockNumberDbWrites++));
             RegisterDb(BuildRocksDbSettings(DbNames.BlockInfos, () => Metrics.BlockInfosDbReads++, () => Metrics.BlockInfosDbWrites++));
+            RegisterDb(BuildRocksDbSettings(DbNames.BadBlocks, () => Metrics.BadBlocksDbReads++, () => Metrics.BadBlocksDbWrites++));
 
             RocksDbSettings stateDbSettings = BuildRocksDbSettings(DbNames.State, () => Metrics.StateDbReads++, () => Metrics.StateDbWrites++);
             RegisterCustomDb(DbNames.State, () => new FullPruningDb(

diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockTree.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockTree.cs
@@ -52,12 +52,14 @@ public Task Execute(CancellationToken cancellationToken)
 
             IBlockStore blockStore = new BlockStore(_get.DbProvider.BlocksDb);
             IHeaderStore headerStore = new HeaderStore(_get.DbProvider.HeadersDb, _get.DbProvider.BlockNumbersDb);
+            IBlockStore badBlockStore = _set.BadBlocksStore = new BlockStore(_get.DbProvider.BadBlocksDb, initConfig.BadBlocksStored);
 
             IBlockTree blockTree = _set.BlockTree = new BlockTree(
                 blockStore,
                 headerStore,
                 _get.DbProvider.BlockInfosDb,
                 _get.DbProvider.MetadataDb,
+                badBlockStore,
                 chainLevelInfoRepository,
                 _get.SpecProvider,
                 bloomStorage,

diff --git a/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs b/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs
@@ -116,6 +116,7 @@ public virtual async Task Execute(CancellationToken cancellationToken)
         if (_api.KeyStore is null) throw new StepDependencyException(nameof(_api.KeyStore));
         if (_api.PeerPool is null) throw new StepDependencyException(nameof(_api.PeerPool));
         if (_api.WitnessRepository is null) throw new StepDependencyException(nameof(_api.WitnessRepository));
+        if (_api.BadBlocksStore is null) throw new StepDependencyException(nameof(_api.BadBlocksStore));
 
         ProofModuleFactory proofModuleFactory = new(_api.WorldStateManager, _api.BlockTree, _api.BlockPreprocessor, _api.ReceiptFinder, _api.SpecProvider, _api.LogManager);
         rpcModuleProvider.RegisterBounded(proofModuleFactory, 2, rpcConfig.Timeout);
@@ -133,6 +134,7 @@ public virtual async Task Execute(CancellationToken cancellationToken)
             _api.ConfigProvider,
             _api.SpecProvider,
             _api.SyncModeSelector,
+            _api.BadBlocksStore,
             _api.FileSystem,
             _api.LogManager);
         rpcModuleProvider.RegisterBoundedByCpuCount(debugModuleFactory, rpcConfig.Timeout);

diff --git a/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs b/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs
@@ -73,6 +73,7 @@ public void GlobalSetup()
                 new HeaderStore(dbProvider.HeadersDb, dbProvider.BlockNumbersDb),
                 dbProvider.BlockInfosDb,
                 dbProvider.MetadataDb,
+                new BlockStore(dbProvider.BadBlocksDb),
                 chainLevelInfoRepository,
                 specProvider,
                 NullBloomStorage.Instance,