diff --git a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs index 26da7eee7ab..1faf60af176 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; using FluentAssertions; @@ -174,6 +175,8 @@ private class TestContext public FullPruner Pruner { get; } public MemDb TrieDb { get; } public TestMemDb CopyDb { get; } + public IDriveInfo DriveInfo { get; set; } = Substitute.For(); + public IChainEstimations _chainEstimations = ChainSizes.UnknownChain.Instance; public IProcessExitSource ProcessExitSource { get; } = Substitute.For(); @@ -201,7 +204,7 @@ public TestContext( FullPruningMaxDegreeOfParallelism = degreeOfParallelism, FullPruningMemoryBudgetMb = fullScanMemoryBudgetMb, FullPruningCompletionBehavior = completionBehavior - }, BlockTree, StateReader, ProcessExitSource, LimboLogs.Instance); + }, BlockTree, StateReader, ProcessExitSource, _chainEstimations, DriveInfo, LimboLogs.Instance); } public async Task WaitForPruning() diff --git a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs index fdffbf5d9fe..c5af5c5c360 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs @@ -39,6 +39,8 @@ public class PruningTestBlockchain : TestBlockchain public IPruningTrigger PruningTrigger { get; } = Substitute.For(); public FullTestPruner FullPruner { get; private set; } public IPruningConfig PruningConfig { get; set; } = new PruningConfig(); + public IDriveInfo DriveInfo { get; set; } = Substitute.For(); + public IChainEstimations _chainEstimations = Substitute.For(); public IProcessExitSource ProcessExitSource { get; } = Substitute.For(); public PruningTestBlockchain() @@ -50,7 +52,9 @@ protected override async Task Build(ISpecProvider? specProvider { TestBlockchain chain = await base.Build(specProvider, initialValues); PruningDb = (IFullPruningDb)DbProvider.StateDb; - FullPruner = new FullTestPruner(PruningDb, PruningTrigger, PruningConfig, BlockTree, StateReader, ProcessExitSource, LogManager); + DriveInfo.AvailableFreeSpace.Returns(long.MaxValue); + _chainEstimations.StateSize.Returns((long?)null); + FullPruner = new FullTestPruner(PruningDb, PruningTrigger, PruningConfig, BlockTree, StateReader, ProcessExitSource, DriveInfo, _chainEstimations, LogManager); return chain; } @@ -89,8 +93,10 @@ public FullTestPruner( IBlockTree blockTree, IStateReader stateReader, IProcessExitSource processExitSource, + IDriveInfo driveInfo, + IChainEstimations chainEstimations, ILogManager logManager) - : base(pruningDb, pruningTrigger, pruningConfig, blockTree, stateReader, processExitSource, logManager) + : base(pruningDb, pruningTrigger, pruningConfig, blockTree, stateReader, processExitSource, chainEstimations, driveInfo, logManager) { } @@ -122,6 +128,20 @@ public async Task prune_on_disk_only_once() } } + [TestCase(100, 150, false)] + [TestCase(200, 100, true)] + [TestCase(130, 100, true)] + [TestCase(130, 101, false)] + public async Task should_check_available_space_before_running(long availableSpace, long requiredSpace, bool isEnoughSpace) + { + using PruningTestBlockchain chain = await PruningTestBlockchain.Create(); + chain._chainEstimations.PruningSize.Returns(requiredSpace); + chain.DriveInfo.AvailableFreeSpace.Returns(availableSpace); + PruningTriggerEventArgs args = new(); + chain.PruningTrigger.Prune += Raise.Event>(args); + args.Status.Should().Be(isEnoughSpace ? PruningStatus.Starting : PruningStatus.NotEnoughDiskSpace); + } + private static async Task RunPruning(PruningTestBlockchain chain, int time, bool onlyFirstRuns) { chain.FullPruner.WaitHandle.Reset(); diff --git a/src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs new file mode 100644 index 00000000000..9e353b8c63b --- /dev/null +++ b/src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Extensions; +using NUnit.Framework; + +namespace Nethermind.Blockchain.Test; + +public class KnownChainSizesTests +{ + [Test] + public void Update_known_chain_sizes() + { + // Pruning size have to be updated frequently + ChainSizes.CreateChainSizeInfo(BlockchainIds.Mainnet).PruningSize.Should().BeLessThan(200.GB()); + ChainSizes.CreateChainSizeInfo(BlockchainIds.Goerli).PruningSize.Should().BeLessThan(55.GB()); + ChainSizes.CreateChainSizeInfo(BlockchainIds.Sepolia).PruningSize.Should().BeLessThan(8.GB()); + + ChainSizes.CreateChainSizeInfo(BlockchainIds.Chiado).PruningSize.Should().Be(null); + ChainSizes.CreateChainSizeInfo(BlockchainIds.Gnosis).PruningSize.Should().Be(null); + + ChainSizes.CreateChainSizeInfo(BlockchainIds.EnergyWeb).PruningSize.Should().Be(null); + ChainSizes.CreateChainSizeInfo(BlockchainIds.Volta).PruningSize.Should().Be(null); + } +} diff --git a/src/Nethermind/Nethermind.Blockchain/FullPruning/FullPruner.cs b/src/Nethermind/Nethermind.Blockchain/FullPruning/FullPruner.cs index 6a6566906a5..fb27c76f176 100755 --- a/src/Nethermind/Nethermind.Blockchain/FullPruning/FullPruner.cs +++ b/src/Nethermind/Nethermind.Blockchain/FullPruning/FullPruner.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; using Nethermind.Config; @@ -28,6 +29,8 @@ public class FullPruner : IDisposable private readonly IStateReader _stateReader; private readonly IProcessExitSource _processExitSource; private readonly ILogManager _logManager; + private readonly IChainEstimations _chainEstimations; + private readonly IDriveInfo _driveInfo; private IPruningContext? _currentPruning; private int _waitingForBlockProcessed = 0; private int _waitingForStateReady = 0; @@ -44,6 +47,8 @@ public FullPruner( IBlockTree blockTree, IStateReader stateReader, IProcessExitSource processExitSource, + IChainEstimations chainEstimations, + IDriveInfo driveInfo, ILogManager logManager) { _fullPruningDb = fullPruningDb; @@ -53,6 +58,8 @@ public FullPruner( _stateReader = stateReader; _processExitSource = processExitSource; _logManager = logManager; + _chainEstimations = chainEstimations; + _driveInfo = driveInfo; _pruningTrigger.Prune += OnPrune; _logger = _logManager.GetClassLogger(); _minimumPruningDelay = TimeSpan.FromHours(_pruningConfig.FullPruningMinimumDelayHours); @@ -78,8 +85,13 @@ private void OnPrune(object? sender, PruningTriggerEventArgs e) // If we are already pruning, we don't need to do anything else if (CanStartNewPruning()) { + // Check if we have enough disk space to run pruning + if (!HaveEnoughDiskSpaceToRun() && _pruningConfig.AvailableSpaceCheckEnabled) + { + e.Status = PruningStatus.NotEnoughDiskSpace; + } // we mark that we are waiting for block (for thread safety) - if (Interlocked.CompareExchange(ref _waitingForBlockProcessed, 1, 0) == 0) + else if (Interlocked.CompareExchange(ref _waitingForBlockProcessed, 1, 0) == 0) { // we don't want to start pruning in the middle of block processing, lets wait for new head. _blockTree.OnUpdateMainChain += OnUpdateMainChain; @@ -158,6 +170,28 @@ private void SetCurrentPruning(IPruningContext pruningContext) private bool CanStartNewPruning() => _fullPruningDb.CanStartPruning; + private const long ChainSizeThresholdFactor = 130; + + private bool HaveEnoughDiskSpaceToRun() + { + long? currentChainSize = _chainEstimations.PruningSize; + if (currentChainSize is null) + { + if (_logger.IsInfo) _logger.Info("Full Pruning: Chain size estimation is unavailable."); + return true; + } + + long available = _driveInfo.AvailableFreeSpace; + if (available < currentChainSize.Value * ChainSizeThresholdFactor / 100) + { + if (_logger.IsWarn) + _logger.Warn( + $"Not enough disk space to run full pruning. Required {(currentChainSize * ChainSizeThresholdFactor) / 1.GB()} GB. Have {available / 1.GB()} GB"); + return false; + } + return true; + } + private void HandlePruningFinished(object? sender, PruningEventArgs e) { switch (_pruningConfig.FullPruningCompletionBehavior) diff --git a/src/Nethermind/Nethermind.Blockchain/FullPruning/PruningStatus.cs b/src/Nethermind/Nethermind.Blockchain/FullPruning/PruningStatus.cs index 7422ab17999..2685df1a4ee 100644 --- a/src/Nethermind/Nethermind.Blockchain/FullPruning/PruningStatus.cs +++ b/src/Nethermind/Nethermind.Blockchain/FullPruning/PruningStatus.cs @@ -17,6 +17,11 @@ public enum PruningStatus /// Disabled, + /// + /// Pruning failed because of low disk space + /// + NotEnoughDiskSpace, + /// /// Delayed - full pruning is temporary disabled. Too little time from previous successful pruning. /// diff --git a/src/Nethermind/Nethermind.Blockchain/KnownChainSizes.cs b/src/Nethermind/Nethermind.Blockchain/KnownChainSizes.cs index 9a38bf63825..3b25442fb62 100644 --- a/src/Nethermind/Nethermind.Blockchain/KnownChainSizes.cs +++ b/src/Nethermind/Nethermind.Blockchain/KnownChainSizes.cs @@ -2,46 +2,96 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; using Nethermind.Core; using Nethermind.Core.Extensions; namespace Nethermind.Blockchain { - public static class Known + public interface IChainEstimations { - public readonly struct SizeInfo + long? StateSize { get; } + long? PruningSize { get; } + } + + public static class ChainSizes + { + public class UnknownChain : IChainEstimations + { + public long? StateSize => null; + public long? PruningSize => null; + + public static readonly IChainEstimations Instance = new UnknownChain(); + } + + private class ChainEstimations : IChainEstimations { - public SizeInfo( - long sizeAtUpdateDate, - long dailyGrowth, - DateTime updateDate) + private readonly LinearExtrapolation? _stateSizeEstimator; + private readonly LinearExtrapolation? _prunedStateEstimator; + + public ChainEstimations(LinearExtrapolation? stateSizeEstimator = null, LinearExtrapolation? prunedStateEstimator = null) { - SizeAtUpdateDate = sizeAtUpdateDate; - DailyGrowth = dailyGrowth; - UpdateDate = updateDate; + _stateSizeEstimator = stateSizeEstimator; + _prunedStateEstimator = prunedStateEstimator; } - public long SizeAtUpdateDate { get; } - public long DailyGrowth { get; } - public DateTime UpdateDate { get; } + public long? StateSize => _stateSizeEstimator?.Estimate; + public long? PruningSize => _prunedStateEstimator?.Estimate; + } - public long Current => SizeAtUpdateDate + (DateTime.UtcNow - UpdateDate).Days * DailyGrowth; + private class LinearExtrapolation + { + private readonly long _atUpdate; + private readonly long _dailyGrowth; + private readonly DateTime _updateDate; + + public LinearExtrapolation(long atUpdate, long dailyGrowth, DateTime updateDate) + { + _atUpdate = atUpdate; + _dailyGrowth = dailyGrowth; + _updateDate = updateDate; + } + + public LinearExtrapolation(long firstValue, DateTime firstDate, long secondValue, DateTime secondDate) + { + _atUpdate = firstValue; + _dailyGrowth = (long)((secondValue - firstValue) / (secondDate - firstDate).TotalDays); + _updateDate = firstDate; + } + + public long Estimate => _atUpdate + (DateTime.UtcNow - _updateDate).Days * _dailyGrowth; } /// /// Size in bytes, daily growth rate and the date of manual update /// - public static Dictionary ChainSize = new() + public static IChainEstimations CreateChainSizeInfo(ulong chainId) { - { BlockchainIds.Goerli, new SizeInfo(8490.MB(), 15.MB(), new DateTime(2021, 12, 7)) }, - { BlockchainIds.Rinkeby, new SizeInfo(34700.MB(), 20.MB(), new DateTime(2021, 12, 7)) }, - { BlockchainIds.Ropsten, new SizeInfo(35900.MB(), 25.MB(), new DateTime(2021, 12, 7)) }, - { BlockchainIds.Mainnet, new SizeInfo(90000.MB(), 70.MB(), new DateTime(2022, 04, 7)) }, - { BlockchainIds.Gnosis, new SizeInfo(18000.MB(), 48.MB(), new DateTime(2021, 12, 7)) }, - { BlockchainIds.EnergyWeb, new SizeInfo(15300.MB(), 15.MB(), new DateTime(2021, 12, 7)) }, - { BlockchainIds.Volta, new SizeInfo(17500.MB(), 10.MB(), new DateTime(2021, 11, 7)) }, - { BlockchainIds.PoaCore, new SizeInfo(13900.MB(), 4.MB(), new DateTime(2021, 12, 7)) }, - }; + return chainId switch + { + BlockchainIds.Goerli => new ChainEstimations( + new LinearExtrapolation(8490.MB(), 15.MB(), new DateTime(2021, 12, 7)), + new LinearExtrapolation( + 49311060515, new(2023, 05, 20, 1, 31, 00), + 52341479114, new(2023, 06, 07, 20, 12, 00))), + BlockchainIds.Mainnet => new ChainEstimations( + new LinearExtrapolation(90000.MB(), 70.MB(), new DateTime(2022, 04, 7)), + new LinearExtrapolation( + 172553555637, new DateTime(2023, 05, 18, 18, 12, 0), + 177439054863, new DateTime(2023, 06, 8, 02, 36, 0))), + BlockchainIds.Gnosis => new ChainEstimations( + new LinearExtrapolation(18000.MB(), 48.MB(), new DateTime(2021, 12, 7))), + BlockchainIds.EnergyWeb => new ChainEstimations( + new LinearExtrapolation(15300.MB(), 15.MB(), new DateTime(2021, 12, 7))), + BlockchainIds.Volta => new ChainEstimations( + new LinearExtrapolation(17500.MB(), 10.MB(), new DateTime(2021, 11, 7))), + BlockchainIds.PoaCore => new ChainEstimations( + new LinearExtrapolation(13900.MB(), 4.MB(), new DateTime(2021, 12, 7))), + BlockchainIds.Sepolia => new ChainEstimations(null, + new LinearExtrapolation( + 3699505976, new(2023, 04, 28, 20, 18, 0), + 5407426707, new(2023, 06, 07, 23, 10, 0))), + _ => UnknownChain.Instance + }; + } } } diff --git a/src/Nethermind/Nethermind.HealthChecks/DbDriveInfoProvider.cs b/src/Nethermind/Nethermind.Core/Extensions/DbDriveInfoProvider.cs similarity index 95% rename from src/Nethermind/Nethermind.HealthChecks/DbDriveInfoProvider.cs rename to src/Nethermind/Nethermind.Core/Extensions/DbDriveInfoProvider.cs index 0c5c35ce9de..08ea1f1a32e 100644 --- a/src/Nethermind/Nethermind.HealthChecks/DbDriveInfoProvider.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/DbDriveInfoProvider.cs @@ -7,7 +7,7 @@ using System.IO.Abstractions; using System.Linq; -namespace Nethermind.HealthChecks +namespace Nethermind.Core.Extensions { public static class DbDriveInfoProvider { @@ -17,7 +17,7 @@ static IDriveInfo FindDriveForDirectory(IDriveInfo[] drives, DirectoryInfo dir) { string dPath = dir.LinkTarget ?? dir.FullName; IEnumerable candidateDrives = drives.Where(drive => dPath.StartsWith(drive.RootDirectory.FullName)); - IDriveInfo result = null; + IDriveInfo? result = null; foreach (IDriveInfo driveInfo in candidateDrives) { result ??= driveInfo; @@ -27,7 +27,7 @@ static IDriveInfo FindDriveForDirectory(IDriveInfo[] drives, DirectoryInfo dir) } } - return result; + return result!; } DirectoryInfo topDir = new(dbPath); diff --git a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj index 5a7b9a74981..3c7d27cd097 100644 --- a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj +++ b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Nethermind/Nethermind.Db/IPruningConfig.cs b/src/Nethermind/Nethermind.Db/IPruningConfig.cs index 9992f89725f..dc297eb701e 100755 --- a/src/Nethermind/Nethermind.Db/IPruningConfig.cs +++ b/src/Nethermind/Nethermind.Db/IPruningConfig.cs @@ -63,5 +63,8 @@ public interface IPruningConfig : IConfig "'AlwaysShutdown': shuts Nethermind down once the prune completes, whether it succeeded or failed.", DefaultValue = "None")] FullPruningCompletionBehavior FullPruningCompletionBehavior { get; set; } + + [ConfigItem(Description = "Enables available disk space check.", DefaultValue = "true")] + bool AvailableSpaceCheckEnabled { get; set; } } } diff --git a/src/Nethermind/Nethermind.Db/PruningConfig.cs b/src/Nethermind/Nethermind.Db/PruningConfig.cs index 18ee256a1d5..5d894a611af 100755 --- a/src/Nethermind/Nethermind.Db/PruningConfig.cs +++ b/src/Nethermind/Nethermind.Db/PruningConfig.cs @@ -31,5 +31,6 @@ public bool Enabled public bool FullPruningDisableLowPriorityWrites { get; set; } = false; public int FullPruningMinimumDelayHours { get; set; } = 240; public FullPruningCompletionBehavior FullPruningCompletionBehavior { get; set; } = FullPruningCompletionBehavior.None; + public bool AvailableSpaceCheckEnabled { get; set; } = true; } } diff --git a/src/Nethermind/Nethermind.HealthChecks/HealthChecksPlugin.cs b/src/Nethermind/Nethermind.HealthChecks/HealthChecksPlugin.cs index d6bd450be6a..350395ab9dd 100644 --- a/src/Nethermind/Nethermind.HealthChecks/HealthChecksPlugin.cs +++ b/src/Nethermind/Nethermind.HealthChecks/HealthChecksPlugin.cs @@ -14,7 +14,7 @@ using Nethermind.Logging; using Nethermind.JsonRpc; using Nethermind.Monitoring.Config; -using Nethermind.Core.Exceptions; +using Nethermind.Core.Extensions; namespace Nethermind.HealthChecks { diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs index a8c849563b0..d988242765a 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; using Nethermind.Api; @@ -298,13 +299,17 @@ private static void InitializeFullPruning( IDb stateDb = api.DbProvider!.StateDb; if (stateDb is IFullPruningDb fullPruningDb) { - IPruningTrigger? pruningTrigger = CreateAutomaticTrigger(fullPruningDb.GetPath(initConfig.BaseDbPath)); + string pruningDbPath = fullPruningDb.GetPath(initConfig.BaseDbPath); + IPruningTrigger? pruningTrigger = CreateAutomaticTrigger(pruningDbPath); if (pruningTrigger is not null) { api.PruningTrigger.Add(pruningTrigger); } - FullPruner pruner = new(fullPruningDb, api.PruningTrigger, pruningConfig, api.BlockTree!, stateReader, api.ProcessExit!, api.LogManager); + IDriveInfo drive = api.FileSystem.GetDriveInfos(pruningDbPath)[0]; + FullPruner pruner = new(fullPruningDb, api.PruningTrigger, pruningConfig, api.BlockTree!, + stateReader, api.ProcessExit!, ChainSizes.CreateChainSizeInfo(api.ChainSpec.ChainId), + drive, api.LogManager); api.DisposeStack.Push(pruner); } } diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/DetailedProgress.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/DetailedProgress.cs index 6f641772820..41107f73ed5 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/DetailedProgress.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/DetailedProgress.cs @@ -2,8 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; -using System.Linq; using Nethermind.Blockchain; using Nethermind.Logging; using Nethermind.Serialization.Rlp; @@ -46,14 +44,11 @@ public class DetailedProgress internal (DateTime small, DateTime full) LastReportTime = (DateTime.MinValue, DateTime.MinValue); - private Known.SizeInfo? _chainSizeInfo; + private readonly IChainEstimations _chainEstimations; public DetailedProgress(ulong chainId, byte[] serializedInitialState) { - if (Known.ChainSize.TryGetValue(chainId, out Known.SizeInfo value)) - { - _chainSizeInfo = value; - } + _chainEstimations = ChainSizes.CreateChainSizeInfo(chainId); LoadFromSerialized(serializedInitialState); } @@ -75,12 +70,12 @@ internal void DisplayProgressReport(int pendingRequestsCount, BranchProgress bra Metrics.StateSynced = DataSize; string dataSizeInfo = $"{(decimal)DataSize / 1000 / 1000,6:F2}MB"; - if (_chainSizeInfo != null) + if (_chainEstimations.StateSize is not null) { - decimal percentage = Math.Min(1, (decimal)DataSize / _chainSizeInfo.Value.Current); + decimal percentage = Math.Min(1, (decimal)DataSize / _chainEstimations.StateSize.Value); dataSizeInfo = string.Concat( $"~{percentage:P2} | ", dataSizeInfo, - $" / ~{(decimal)_chainSizeInfo.Value.Current / 1000 / 1000,6:F2}MB"); + $" / ~{(decimal)_chainEstimations.StateSize.Value / 1000 / 1000,6:F2}MB"); } if (logger.IsInfo) logger.Info(