Safety check for full pruning

src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO.Abstractions;
 using System.Threading;
 using System.Threading.Tasks;
 using FluentAssertions;
@@ -150,6 +151,8 @@ private class TestContext
             public FullPruner Pruner { get; }
             public MemDb TrieDb { get; }
             public MemDb CopyDb { get; }
+            public IDriveInfo DriveInfo { get; set; } = Substitute.For<IDriveInfo>();
+            public IChainEstimations _chainEstimations = ChainSizes.UnknownChain.Instance;
 
             public TestContext(bool successfulPruning, bool clearPrunedDb = false, int fullScanMemoryBudgetMb = 0, int degreeOfParallelism = 0)
             {
@@ -169,7 +172,7 @@ public TestContext(bool successfulPruning, bool clearPrunedDb = false, int fullS
                 {
                     FullPruningMaxDegreeOfParallelism = degreeOfParallelism,
                     FullPruningMemoryBudgetMb = fullScanMemoryBudgetMb,
-                }, BlockTree, StateReader, LimboLogs.Instance);
+                }, BlockTree, StateReader, _chainEstimations, DriveInfo, LimboLogs.Instance);
             }
 
             public async Task<bool> WaitForPruning()

src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs

@@ -37,6 +37,8 @@ public class PruningTestBlockchain : TestBlockchain
             public IPruningTrigger PruningTrigger { get; } = Substitute.For<IPruningTrigger>();
             public FullTestPruner FullPruner { get; private set; }
             public IPruningConfig PruningConfig { get; set; } = new PruningConfig();
+            public IDriveInfo DriveInfo { get; set; } = Substitute.For<IDriveInfo>();
+            public IChainEstimations _chainEstimations = Substitute.For<IChainEstimations>();
 
             public PruningTestBlockchain()
             {
@@ -47,7 +49,9 @@ protected override async Task<TestBlockchain> Build(ISpecProvider specProvider =
             {
                 TestBlockchain chain = await base.Build(specProvider, initialValues);
                 PruningDb = (IFullPruningDb)DbProvider.StateDb;
-                FullPruner = new FullTestPruner(PruningDb, PruningTrigger, PruningConfig, BlockTree, StateReader, LogManager);
+                DriveInfo.AvailableFreeSpace.Returns(long.MaxValue);
+                _chainEstimations.StateSize.Returns((long?)null);
+                FullPruner = new FullTestPruner(PruningDb, PruningTrigger, PruningConfig, BlockTree, StateReader, DriveInfo, _chainEstimations, LogManager);
                 return chain;
             }
 
@@ -85,8 +89,10 @@ public FullTestPruner(
                     IPruningConfig pruningConfig,
                     IBlockTree blockTree,
                     IStateReader stateReader,
+                    IDriveInfo driveInfo,
+                    IChainEstimations chainEstimations,
                     ILogManager logManager)
-                    : base(pruningDb, pruningTrigger, pruningConfig, blockTree, stateReader, logManager)
+                    : base(pruningDb, pruningTrigger, pruningConfig, blockTree, stateReader, chainEstimations, driveInfo, logManager)
                 {
                 }
 
@@ -118,6 +124,20 @@ public async Task prune_on_disk_only_once()
             }
         }
 
+        [TestCase(100, 150, false)]
+        [TestCase(150, 100, true)]
+        [TestCase(120, 100, true)]
+        [TestCase(120, 101, false)]
+        public async Task should_check_available_space_before_running(long availableSpace, long requiredSpace, bool isEnoughSpace)
+        {
+            using PruningTestBlockchain chain = await PruningTestBlockchain.Create();
+            chain._chainEstimations.StateSize.Returns(requiredSpace);
+            chain.DriveInfo.AvailableFreeSpace.Returns(availableSpace);
+            PruningTriggerEventArgs args = new();
+            chain.PruningTrigger.Prune += Raise.Event<EventHandler<PruningTriggerEventArgs>>(args);
+            args.Status.Should().Be(isEnoughSpace ? PruningStatus.Starting : PruningStatus.NotEnoughDiscSpace);
+        }
+
         private static async Task RunPruning(PruningTestBlockchain chain, int time, bool onlyFirstRuns)
         {
             chain.FullPruner.WaitHandle.Reset();

src/Nethermind/Nethermind.Blockchain/FullPruning/FullPruner.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.IO.Abstractions;
 using System.Threading;
 using System.Threading.Tasks;
 using Nethermind.Core;
@@ -26,6 +27,8 @@ public class FullPruner : IDisposable
         private readonly IBlockTree _blockTree;
         private readonly IStateReader _stateReader;
         private readonly ILogManager _logManager;
+        private readonly IChainEstimations _chainEstimations;
+        private readonly IDriveInfo _driveInfo;
         private IPruningContext? _currentPruning;
         private int _waitingForBlockProcessed = 0;
         private int _waitingForStateReady = 0;
@@ -41,6 +44,8 @@ public FullPruner(
             IPruningConfig pruningConfig,
             IBlockTree blockTree,
             IStateReader stateReader,
+            IChainEstimations chainEstimations,
+            IDriveInfo driveInfo,
             ILogManager logManager)
         {
             _fullPruningDb = fullPruningDb;
@@ -49,6 +54,8 @@ public FullPruner(
             _blockTree = blockTree;
             _stateReader = stateReader;
             _logManager = logManager;
+            _chainEstimations = chainEstimations;
+            _driveInfo = driveInfo;
             _pruningTrigger.Prune += OnPrune;
             _logger = _logManager.GetClassLogger();
             _minimumPruningDelay = TimeSpan.FromHours(_pruningConfig.FullPruningMinimumDelayHours);
@@ -74,8 +81,13 @@ private void OnPrune(object? sender, PruningTriggerEventArgs e)
             // If we are already pruning, we don't need to do anything
             else if (CanStartNewPruning())
             {
+                // Check if we have enough disk space to run pruning
+                if (!HaveEnoughDiskSpaceToRun() && _pruningConfig.AvailableSpaceCheckEnabled)
+                {
+                    e.Status = PruningStatus.NotEnoughDiscSpace;
+                }
                 // we mark that we are waiting for block (for thread safety)
-                if (Interlocked.CompareExchange(ref _waitingForBlockProcessed, 1, 0) == 0)
+                else if (Interlocked.CompareExchange(ref _waitingForBlockProcessed, 1, 0) == 0)
                 {
                     // we don't want to start pruning in the middle of block processing, lets wait for new head.
                     _blockTree.OnUpdateMainChain += OnUpdateMainChain;
@@ -154,6 +166,28 @@ private void SetCurrentPruning(IPruningContext pruningContext)
 
         private bool CanStartNewPruning() => _fullPruningDb.CanStartPruning;
 
+        private const double ChainSizeThresholdFactor = 1.2;
+
+        private bool HaveEnoughDiskSpaceToRun()
+        {
+            long? currentChainSize = _chainEstimations.StateSize;
+            if (currentChainSize is null)
+            {
+                if (_logger.IsWarn) _logger.Warn("Chain size estimation is unavailable.");
+                return true;
+            }
+
+            long available = _driveInfo.AvailableFreeSpace;
+            if (available < currentChainSize * ChainSizeThresholdFactor)
+            {
+                if (_logger.IsError)
+                    _logger.Error(
+                        $"Not enough disk space to run full pruning. Expected {(currentChainSize * ChainSizeThresholdFactor) / 1.GB()} GB. Have {available / 1.GB()} GB");
+                return false;
+            }
+            return true;
+        }
+
         private void HandlePruningFinished(object? sender, PruningEventArgs e)
         {
             switch (_pruningConfig.FullPruningCompletionBehavior)

src/Nethermind/Nethermind.Blockchain/FullPruning/PruningStatus.cs

@@ -31,4 +31,9 @@ public enum PruningStatus
     /// Full pruning was triggered and is starting.
     /// </summary>
     Starting,
+
+    /// <summary>
+    /// Pruning failed because of low disk space
+    /// </summary>
+    NotEnoughDiscSpace,
 }

src/Nethermind/Nethermind.Blockchain/KnownChainSizes.cs

@@ -2,46 +2,78 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
-using System.Collections.Generic;
 using Nethermind.Core;
 using Nethermind.Core.Extensions;
 
 namespace Nethermind.Blockchain
 {
-    public static class Known
+    public interface IChainEstimations
     {
-        public readonly struct SizeInfo
+        long? StateSize { get; }
+    }
+
+    public static class ChainSizes
+    {
+        public class UnknownChain : IChainEstimations
+        {
+            public long? StateSize => null;
+
+            public static readonly IChainEstimations Instance = new UnknownChain();
+        }
+
+        private class ChainEstimations : IChainEstimations
         {
-            public SizeInfo(
-                long sizeAtUpdateDate,
-                long dailyGrowth,
-                DateTime updateDate)
+            private readonly LinearExtrapolation _stateSizeEstimator;
+
+            public ChainEstimations(LinearExtrapolation stateSizeEstimator)
             {
-                SizeAtUpdateDate = sizeAtUpdateDate;
-                DailyGrowth = dailyGrowth;
-                UpdateDate = updateDate;
+                _stateSizeEstimator = stateSizeEstimator;
             }
 
-            public long SizeAtUpdateDate { get; }
-            public long DailyGrowth { get; }
-            public DateTime UpdateDate { get; }
+            public long? StateSize => _stateSizeEstimator.Estimate;
+        }
+
+        private class LinearExtrapolation
+        {
+            private readonly long _atUpdate;
+            private readonly long _dailyGrowth;
+            private readonly DateTime _updateDate;
+
+            public LinearExtrapolation(long atUpdate, long dailyGrowth, DateTime updateDate)
+            {
+                _atUpdate = atUpdate;
+                _dailyGrowth = dailyGrowth;
+                _updateDate = updateDate;
+            }
 
-            public long Current => SizeAtUpdateDate + (DateTime.UtcNow - UpdateDate).Days * DailyGrowth;
+            public long Estimate => _atUpdate + (DateTime.UtcNow - _updateDate).Days * _dailyGrowth;
         }
 
         /// <summary>
         /// Size in bytes, daily growth rate and the date of manual update
         /// </summary>
-        public static Dictionary<ulong, SizeInfo> ChainSize = new()
+        public static IChainEstimations CreateChainSizeInfo(ulong chainId)
         {
-            { BlockchainIds.Goerli, new SizeInfo(8490.MB(), 15.MB(), new DateTime(2021, 12, 7)) },
-            { BlockchainIds.Rinkeby, new SizeInfo(34700.MB(), 20.MB(), new DateTime(2021, 12, 7)) },
-            { BlockchainIds.Ropsten, new SizeInfo(35900.MB(), 25.MB(), new DateTime(2021, 12, 7)) },
-            { BlockchainIds.Mainnet, new SizeInfo(90000.MB(), 70.MB(), new DateTime(2022, 04, 7)) },
-            { BlockchainIds.Gnosis, new SizeInfo(18000.MB(), 48.MB(), new DateTime(2021, 12, 7)) },
-            { BlockchainIds.EnergyWeb, new SizeInfo(15300.MB(), 15.MB(), new DateTime(2021, 12, 7)) },
-            { BlockchainIds.Volta, new SizeInfo(17500.MB(), 10.MB(), new DateTime(2021, 11, 7)) },
-            { BlockchainIds.PoaCore, new SizeInfo(13900.MB(), 4.MB(), new DateTime(2021, 12, 7)) },
-        };
+            return chainId switch
+            {
+                BlockchainIds.Goerli => new ChainEstimations(
+                    new LinearExtrapolation(8490.MB(), 15.MB(), new DateTime(2021, 12, 7))),
+                BlockchainIds.Rinkeby => new ChainEstimations(
+                    new LinearExtrapolation(34700.MB(), 20.MB(), new DateTime(2021, 12, 7))),
+                BlockchainIds.Ropsten => new ChainEstimations(
+                    new LinearExtrapolation(35900.MB(), 25.MB(), new DateTime(2021, 12, 7))),
+                BlockchainIds.Mainnet => new ChainEstimations(
+                    new LinearExtrapolation(90000.MB(), 70.MB(), new DateTime(2022, 04, 7))),
+                BlockchainIds.Gnosis => new ChainEstimations(
+                    new LinearExtrapolation(18000.MB(), 48.MB(), new DateTime(2021, 12, 7))),
+                BlockchainIds.EnergyWeb => new ChainEstimations(
+                    new LinearExtrapolation(15300.MB(), 15.MB(), new DateTime(2021, 12, 7))),
+                BlockchainIds.Volta => new ChainEstimations(
+                    new LinearExtrapolation(17500.MB(), 10.MB(), new DateTime(2021, 11, 7))),
+                BlockchainIds.PoaCore => new ChainEstimations(
+                    new LinearExtrapolation(13900.MB(), 4.MB(), new DateTime(2021, 12, 7))),
+                _ => UnknownChain.Instance
+            };
+        }
     }
 }

src/Nethermind/Nethermind.HealthChecks/DbDriveInfoProvider.cs → src/Nethermind/Nethermind.Core/Extensions/DbDriveInfoProvider.cs

@@ -20,7 +20,7 @@
 using System.IO.Abstractions;
 using System.Linq;
 
-namespace Nethermind.HealthChecks
+namespace Nethermind.Core.Extensions
 {
     public static class DbDriveInfoProvider
     {
@@ -30,7 +30,7 @@ static IDriveInfo FindDriveForDirectory(IDriveInfo[] drives, DirectoryInfo dir)
             {
                 string dPath = dir.LinkTarget ?? dir.FullName;
                 IEnumerable<IDriveInfo> candidateDrives = drives.Where(drive => dPath.StartsWith(drive.RootDirectory.FullName));
-                IDriveInfo result = null;
+                IDriveInfo? result = null;
                 foreach (IDriveInfo driveInfo in candidateDrives)
                 {
                     result ??= driveInfo;
@@ -40,7 +40,7 @@ static IDriveInfo FindDriveForDirectory(IDriveInfo[] drives, DirectoryInfo dir)
                     }
                 }
 
-                return result;
+                return result!;
             }
 
             DirectoryInfo topDir = new(dbPath);

src/Nethermind/Nethermind.Core/Nethermind.Core.csproj

@@ -12,6 +12,7 @@
     <PackageReference Include="FastEnum" />
     <PackageReference Include="Microsoft.Extensions.ObjectPool" />
     <PackageReference Include="Nethermind.Numerics.Int256" />
+    <PackageReference Include="System.IO.Abstractions" />
     <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="System.IdentityModel.Tokens.Jwt" />
   </ItemGroup>

src/Nethermind/Nethermind.Db/IPruningConfig.cs

@@ -58,5 +58,8 @@ public interface IPruningConfig : IConfig
                                   "'AlwaysShutdown': shuts Nethermind down once the prune completes, whether it succeeded or failed.",
             DefaultValue = "None")]
         FullPruningCompletionBehavior FullPruningCompletionBehavior { get; set; }
+
+        [ConfigItem(Description = "Disables available disk space check.", DefaultValue = "true", HiddenFromDocs = true)]
+        bool AvailableSpaceCheckEnabled { get; set; }
     }
 }

src/Nethermind/Nethermind.Db/PruningConfig.cs

@@ -30,5 +30,6 @@ public bool Enabled
         public int FullPruningMemoryBudgetMb { get; set; } = 0;
         public int FullPruningMinimumDelayHours { get; set; } = 240;
         public FullPruningCompletionBehavior FullPruningCompletionBehavior { get; set; } = FullPruningCompletionBehavior.None;
+        public bool AvailableSpaceCheckEnabled { get; set; } = true;
     }
 }

src/Nethermind/Nethermind.HealthChecks/HealthChecksPlugin.cs

@@ -14,7 +14,7 @@
 using Nethermind.Logging;
 using Nethermind.JsonRpc;
 using Nethermind.Monitoring.Config;
-using Nethermind.Core.Exceptions;
+using Nethermind.Core.Extensions;
 
 namespace Nethermind.HealthChecks
 {

src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO.Abstractions;
 using System.Threading;
 using System.Threading.Tasks;
 using Nethermind.Api;
@@ -17,12 +18,10 @@
 using Nethermind.Consensus.Processing;
 using Nethermind.Consensus.Producers;
 using Nethermind.Consensus.Validators;
-using Nethermind.Consensus.Withdrawals;
 using Nethermind.Core;
 using Nethermind.Core.Attributes;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Extensions;
-using Nethermind.Crypto;
 using Nethermind.Db;
 using Nethermind.Db.FullPruning;
 using Nethermind.Evm;
@@ -316,7 +315,10 @@ private static void InitializeFullPruning(
                         api.PruningTrigger.Add(pruningTrigger);
                     }
 
-                    FullPruner pruner = new(fullPruningDb, api.PruningTrigger, pruningConfig, api.BlockTree!, stateReader, api.LogManager);
+                    IDriveInfo drive = api.FileSystem.GetDriveInfos(fullPruningDb.GetPath(initConfig.BaseDbPath))[0];
+                    FullPruner pruner = new(fullPruningDb, api.PruningTrigger, pruningConfig, api.BlockTree!,
+                        stateReader, ChainSizes.CreateChainSizeInfo(api.ChainSpec!.ChainId),
+                        drive, api.LogManager);
                     api.DisposeStack.Push(pruner);
                 }
             }