From 2a91ca7a54efdab3aaa6d0fbab0d3555e3a2c2ab Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Tue, 23 Jan 2024 12:38:49 +0300 Subject: [PATCH 1/3] Compact dbs manually --- src/Nethermind/Nethermind.Db/IColumnsDb.cs | 8 ++++++++ ...erTests.cs => BlobTransactionsDbCleanerTests.cs} | 4 ++-- ...onsDbCleaner.cs => BlobTransactionsDbCleaner.cs} | 13 ++++++++++--- .../Nethermind.Merge.Plugin/MergePlugin.cs | 2 +- 4 files changed, 21 insertions(+), 6 deletions(-) rename src/Nethermind/Nethermind.Merge.Plugin.Test/{ProcessedTransactionsDbCleanerTests.cs => BlobTransactionsDbCleanerTests.cs} (92%) rename src/Nethermind/Nethermind.Merge.Plugin/{ProcessedTransactionsDbCleaner.cs => BlobTransactionsDbCleaner.cs} (80%) diff --git a/src/Nethermind/Nethermind.Db/IColumnsDb.cs b/src/Nethermind/Nethermind.Db/IColumnsDb.cs index d7531872a6d..085f2de80c3 100644 --- a/src/Nethermind/Nethermind.Db/IColumnsDb.cs +++ b/src/Nethermind/Nethermind.Db/IColumnsDb.cs @@ -13,6 +13,14 @@ public interface IColumnsDb : IDbMeta IEnumerable ColumnKeys { get; } public IReadOnlyColumnDb CreateReadOnly(bool createInMemWriteStore) => new ReadOnlyColumnsDb(this, createInMemWriteStore); IColumnsWriteBatch StartWriteBatch(); + + void CompactColumns() + { + foreach (TKey key in ColumnKeys) + { + GetColumnDb(key).Compact(); + } + } } public interface IColumnsWriteBatch : IDisposable diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlobTransactionsDbCleanerTests.cs similarity index 92% rename from src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs rename to src/Nethermind/Nethermind.Merge.Plugin.Test/BlobTransactionsDbCleanerTests.cs index ee17433c037..e7330504ee7 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlobTransactionsDbCleanerTests.cs @@ -20,7 +20,7 @@ namespace Nethermind.Merge.Plugin.Test; [TestFixture] [Parallelizable(ParallelScope.Self)] -public class ProcessedTransactionsDbCleanerTests +public class BlobTransactionsDbCleanerTests { private readonly ILogManager _logManager = LimboLogs.Instance; private readonly ISpecProvider _specProvider = MainnetSpecProvider.Instance; @@ -48,7 +48,7 @@ Transaction GetTx(PrivateKey sender) returnedTxs!.Length.Should().Be(2); IBlockFinalizationManager finalizationManager = Substitute.For(); - ProcessedTransactionsDbCleaner dbCleaner = new(finalizationManager, columnsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _logManager); + BlobTransactionsDbCleaner dbCleaner = new(finalizationManager, columnsDb, _logManager); finalizationManager.BlocksFinalized += Raise.EventWith( new FinalizeEventArgs(Build.A.BlockHeader.TestObject, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/BlobTransactionsDbCleaner.cs similarity index 80% rename from src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs rename to src/Nethermind/Nethermind.Merge.Plugin/BlobTransactionsDbCleaner.cs index 7babdf12c72..e675cac2656 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/BlobTransactionsDbCleaner.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Linq; using System.Threading.Tasks; using Nethermind.Blockchain; using Nethermind.Core; @@ -11,18 +12,20 @@ namespace Nethermind.Merge.Plugin; -public class ProcessedTransactionsDbCleaner : IDisposable +public class BlobTransactionsDbCleaner : IDisposable { private readonly IBlockFinalizationManager _finalizationManager; private readonly IDb _processedTxsDb; + private readonly IColumnsDb _db; private readonly ILogger _logger; private long _lastFinalizedBlock = 0; public Task CleaningTask { get; private set; } = Task.CompletedTask; - public ProcessedTransactionsDbCleaner(IBlockFinalizationManager finalizationManager, IDb processedTxsDb, ILogManager logManager) + public BlobTransactionsDbCleaner(IBlockFinalizationManager finalizationManager, IColumnsDb db, ILogManager logManager) { _finalizationManager = finalizationManager ?? throw new ArgumentNullException(nameof(finalizationManager)); - _processedTxsDb = processedTxsDb ?? throw new ArgumentNullException(nameof(processedTxsDb)); + _db = db ?? throw new ArgumentNullException(nameof(db)); + _processedTxsDb = db?.GetColumnDb(BlobTxsColumns.ProcessedTxs) ?? throw new ArgumentNullException(nameof(db)); _logger = logManager.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _finalizationManager.BlocksFinalized += OnBlocksFinalized; @@ -55,6 +58,10 @@ private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) if (_logger.IsDebug) _logger.Debug($"Cleaned processed blob txs from block {_lastFinalizedBlock} to block {newlyFinalizedBlockNumber}"); + _db.CompactColumns(); + + if (_logger.IsDebug) _logger.Debug($"Blob transactions database columns have been compacted"); + _lastFinalizedBlock = newlyFinalizedBlockNumber; } catch (Exception exception) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index a8c7db69aaa..593f5dda3fc 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -103,7 +103,7 @@ public virtual Task Init(INethermindApi nethermindApi) _blockFinalizationManager = new ManualBlockFinalizationManager(); if (_txPoolConfig.BlobsSupport.SupportsReorgs()) { - ProcessedTransactionsDbCleaner processedTransactionsDbCleaner = new(_blockFinalizationManager, _api.DbProvider.BlobTransactionsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _api.LogManager); + BlobTransactionsDbCleaner processedTransactionsDbCleaner = new(_blockFinalizationManager, _api.DbProvider.BlobTransactionsDb, _api.LogManager); _api.DisposeStack.Push(processedTransactionsDbCleaner); } From e96c3e1c32cb8dac41b15a3098355807869cc1ee Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Tue, 23 Jan 2024 12:56:45 +0300 Subject: [PATCH 2/3] Override Compact instead of new method usage --- src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs | 8 ++++++++ src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs | 2 +- src/Nethermind/Nethermind.Db/IColumnsDb.cs | 8 -------- .../Nethermind.Merge.Plugin/BlobTransactionsDbCleaner.cs | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs index 71bbc7476c0..23d40d34f11 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs @@ -28,6 +28,14 @@ public ColumnsDb(string basePath, DbSettings settings, IDbConfig dbConfig, ILogM } } + public override void Compact() + { + foreach (T key in ColumnKeys) + { + _columnDbs[key].Compact(); + } + } + private static IReadOnlyList GetEnumKeys(IReadOnlyList keys) { if (typeof(T).IsEnum && keys.Count == 0) diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index 92606c07306..8a31ef7b6ec 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -962,7 +962,7 @@ public void Flush() InnerFlush(); } - public void Compact() + public virtual void Compact() { _db.CompactRange(Keccak.Zero.BytesToArray(), Keccak.MaxValue.BytesToArray()); } diff --git a/src/Nethermind/Nethermind.Db/IColumnsDb.cs b/src/Nethermind/Nethermind.Db/IColumnsDb.cs index 085f2de80c3..d7531872a6d 100644 --- a/src/Nethermind/Nethermind.Db/IColumnsDb.cs +++ b/src/Nethermind/Nethermind.Db/IColumnsDb.cs @@ -13,14 +13,6 @@ public interface IColumnsDb : IDbMeta IEnumerable ColumnKeys { get; } public IReadOnlyColumnDb CreateReadOnly(bool createInMemWriteStore) => new ReadOnlyColumnsDb(this, createInMemWriteStore); IColumnsWriteBatch StartWriteBatch(); - - void CompactColumns() - { - foreach (TKey key in ColumnKeys) - { - GetColumnDb(key).Compact(); - } - } } public interface IColumnsWriteBatch : IDisposable diff --git a/src/Nethermind/Nethermind.Merge.Plugin/BlobTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/BlobTransactionsDbCleaner.cs index 7c0d4fc6871..38f0e28b4d5 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/BlobTransactionsDbCleaner.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/BlobTransactionsDbCleaner.cs @@ -58,7 +58,7 @@ private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) if (_logger.IsDebug) _logger.Debug($"Cleaned processed blob txs from block {_lastFinalizedBlock} to block {newlyFinalizedBlockNumber}"); - _db.CompactColumns(); + _db.Compact(); if (_logger.IsDebug) _logger.Debug($"Blob transactions database columns have been compacted"); From a55635d0e0c9730798495d7b8accefd55615283d Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 29 Jan 2024 13:44:09 +0300 Subject: [PATCH 3/3] Clean just one db and while not in progress --- ...ts.cs => ProcessedTransactionsDbCleanerTests.cs} | 4 ++-- .../Nethermind.Merge.Plugin/MergePlugin.cs | 2 +- ...Cleaner.cs => ProcessedTransactionsDbCleaner.cs} | 13 +++++-------- 3 files changed, 8 insertions(+), 11 deletions(-) rename src/Nethermind/Nethermind.Merge.Plugin.Test/{BlobTransactionsDbCleanerTests.cs => ProcessedTransactionsDbCleanerTests.cs} (92%) rename src/Nethermind/Nethermind.Merge.Plugin/{BlobTransactionsDbCleaner.cs => ProcessedTransactionsDbCleaner.cs} (83%) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/BlobTransactionsDbCleanerTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs similarity index 92% rename from src/Nethermind/Nethermind.Merge.Plugin.Test/BlobTransactionsDbCleanerTests.cs rename to src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs index e7330504ee7..ee17433c037 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/BlobTransactionsDbCleanerTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs @@ -20,7 +20,7 @@ namespace Nethermind.Merge.Plugin.Test; [TestFixture] [Parallelizable(ParallelScope.Self)] -public class BlobTransactionsDbCleanerTests +public class ProcessedTransactionsDbCleanerTests { private readonly ILogManager _logManager = LimboLogs.Instance; private readonly ISpecProvider _specProvider = MainnetSpecProvider.Instance; @@ -48,7 +48,7 @@ Transaction GetTx(PrivateKey sender) returnedTxs!.Length.Should().Be(2); IBlockFinalizationManager finalizationManager = Substitute.For(); - BlobTransactionsDbCleaner dbCleaner = new(finalizationManager, columnsDb, _logManager); + ProcessedTransactionsDbCleaner dbCleaner = new(finalizationManager, columnsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _logManager); finalizationManager.BlocksFinalized += Raise.EventWith( new FinalizeEventArgs(Build.A.BlockHeader.TestObject, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 760f697026c..3a7b584e3a4 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -103,7 +103,7 @@ public virtual Task Init(INethermindApi nethermindApi) _blockFinalizationManager = new ManualBlockFinalizationManager(); if (_txPoolConfig.BlobsSupport.SupportsReorgs()) { - BlobTransactionsDbCleaner processedTransactionsDbCleaner = new(_blockFinalizationManager, _api.DbProvider.BlobTransactionsDb, _api.LogManager); + ProcessedTransactionsDbCleaner processedTransactionsDbCleaner = new(_blockFinalizationManager, _api.DbProvider.BlobTransactionsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _api.LogManager); _api.DisposeStack.Push(processedTransactionsDbCleaner); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/BlobTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs similarity index 83% rename from src/Nethermind/Nethermind.Merge.Plugin/BlobTransactionsDbCleaner.cs rename to src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs index 38f0e28b4d5..fccaeb34a8a 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/BlobTransactionsDbCleaner.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Linq; using System.Threading.Tasks; using Nethermind.Blockchain; using Nethermind.Core; @@ -12,20 +11,18 @@ namespace Nethermind.Merge.Plugin; -public class BlobTransactionsDbCleaner : IDisposable +public class ProcessedTransactionsDbCleaner : IDisposable { private readonly IBlockFinalizationManager _finalizationManager; private readonly IDb _processedTxsDb; - private readonly IColumnsDb _db; private readonly ILogger _logger; private long _lastFinalizedBlock = 0; public Task CleaningTask { get; private set; } = Task.CompletedTask; - public BlobTransactionsDbCleaner(IBlockFinalizationManager finalizationManager, IColumnsDb db, ILogManager logManager) + public ProcessedTransactionsDbCleaner(IBlockFinalizationManager finalizationManager, IDb processedTxsDb, ILogManager logManager) { _finalizationManager = finalizationManager ?? throw new ArgumentNullException(nameof(finalizationManager)); - _db = db ?? throw new ArgumentNullException(nameof(db)); - _processedTxsDb = db?.GetColumnDb(BlobTxsColumns.ProcessedTxs) ?? throw new ArgumentNullException(nameof(db)); + _processedTxsDb = processedTxsDb ?? throw new ArgumentNullException(nameof(processedTxsDb)); _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _finalizationManager.BlocksFinalized += OnBlocksFinalized; @@ -33,7 +30,7 @@ public BlobTransactionsDbCleaner(IBlockFinalizationManager finalizationManager, private void OnBlocksFinalized(object? sender, FinalizeEventArgs e) { - if (e.FinalizedBlocks.Count > 0 && e.FinalizedBlocks[0].Number > _lastFinalizedBlock) + if (e.FinalizedBlocks.Count > 0 && e.FinalizedBlocks[0].Number > _lastFinalizedBlock && CleaningTask.IsCompleted) { CleaningTask = Task.Run(() => CleanProcessedTransactionsDb(e.FinalizedBlocks[0].Number)); } @@ -58,7 +55,7 @@ private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) if (_logger.IsDebug) _logger.Debug($"Cleaned processed blob txs from block {_lastFinalizedBlock} to block {newlyFinalizedBlockNumber}"); - _db.Compact(); + _processedTxsDb.Compact(); if (_logger.IsDebug) _logger.Debug($"Blob transactions database columns have been compacted");