Safety check for full pruning

src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO.Abstractions;
 using System.Threading;
 using System.Threading.Tasks;
 using FluentAssertions;
@@ -174,6 +175,8 @@ private class TestContext
             public FullPruner Pruner { get; }
             public MemDb TrieDb { get; }
             public MemDb CopyDb { get; }
+            public IDriveInfo DriveInfo { get; set; } = Substitute.For<IDriveInfo>();
+            public IChainEstimations _chainEstimations = ChainSizes.UnknownChain.Instance;
 
             public IProcessExitSource ProcessExitSource { get; } = Substitute.For<IProcessExitSource>();
 
@@ -201,7 +204,7 @@ public TestContext(
                     FullPruningMaxDegreeOfParallelism = degreeOfParallelism,
                     FullPruningMemoryBudgetMb = fullScanMemoryBudgetMb,
                     FullPruningCompletionBehavior = completionBehavior
-                }, BlockTree, StateReader, ProcessExitSource, LimboLogs.Instance);
+                }, BlockTree, StateReader, ProcessExitSource, _chainEstimations, DriveInfo, LimboLogs.Instance);
             }
 
             public async Task<bool> WaitForPruning()

src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs

@@ -38,6 +38,8 @@ public class PruningTestBlockchain : TestBlockchain
             public IPruningTrigger PruningTrigger { get; } = Substitute.For<IPruningTrigger>();
             public FullTestPruner FullPruner { get; private set; }
             public IPruningConfig PruningConfig { get; set; } = new PruningConfig();
+            public IDriveInfo DriveInfo { get; set; } = Substitute.For<IDriveInfo>();
+            public IChainEstimations _chainEstimations = Substitute.For<IChainEstimations>();
             public IProcessExitSource ProcessExitSource { get; } = Substitute.For<IProcessExitSource>();
 
             public PruningTestBlockchain()
@@ -49,7 +51,9 @@ protected override async Task<TestBlockchain> Build(ISpecProvider? specProvider
             {
                 TestBlockchain chain = await base.Build(specProvider, initialValues);
                 PruningDb = (IFullPruningDb)DbProvider.StateDb;
-                FullPruner = new FullTestPruner(PruningDb, PruningTrigger, PruningConfig, BlockTree, StateReader, ProcessExitSource, LogManager);
+                DriveInfo.AvailableFreeSpace.Returns(long.MaxValue);
+                _chainEstimations.StateSize.Returns((long?)null);
+                FullPruner = new FullTestPruner(PruningDb, PruningTrigger, PruningConfig, BlockTree, StateReader, ProcessExitSource, DriveInfo, _chainEstimations, LogManager);
                 return chain;
             }
 
@@ -88,8 +92,10 @@ public FullTestPruner(
                     IBlockTree blockTree,
                     IStateReader stateReader,
                     IProcessExitSource processExitSource,
+                    IDriveInfo driveInfo,
+                    IChainEstimations chainEstimations,
                     ILogManager logManager)
-                    : base(pruningDb, pruningTrigger, pruningConfig, blockTree, stateReader, processExitSource, logManager)
+                    : base(pruningDb, pruningTrigger, pruningConfig, blockTree, stateReader, processExitSource, chainEstimations, driveInfo, logManager)
                 {
                 }
 
@@ -121,6 +127,20 @@ public async Task prune_on_disk_only_once()
             }
         }
 
+        [TestCase(100, 150, false)]
+        [TestCase(200, 100, true)]
+        [TestCase(130, 100, true)]
+        [TestCase(130, 101, false)]
+        public async Task should_check_available_space_before_running(long availableSpace, long requiredSpace, bool isEnoughSpace)
+        {
+            using PruningTestBlockchain chain = await PruningTestBlockchain.Create();
+            chain._chainEstimations.PruningSize.Returns(requiredSpace);
+            chain.DriveInfo.AvailableFreeSpace.Returns(availableSpace);
+            PruningTriggerEventArgs args = new();
+            chain.PruningTrigger.Prune += Raise.Event<EventHandler<PruningTriggerEventArgs>>(args);
+            args.Status.Should().Be(isEnoughSpace ? PruningStatus.Starting : PruningStatus.NotEnoughDiskSpace);
+        }
+
         private static async Task RunPruning(PruningTestBlockchain chain, int time, bool onlyFirstRuns)
         {
             chain.FullPruner.WaitHandle.Reset();

src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using FluentAssertions;
+using Nethermind.Core;
+using Nethermind.Core.Extensions;
+using NUnit.Framework;
+
+namespace Nethermind.Blockchain.Test;
+
+public class KnownChainSizesTests
+{
+    [Test]
+    public void Update_known_chain_sizes()
+    {
+        // Pruning size have to be updated frequently
+        ChainSizes.CreateChainSizeInfo(BlockchainIds.Mainnet).PruningSize.Should().BeLessThan(200.GB());
+        ChainSizes.CreateChainSizeInfo(BlockchainIds.Sepolia).PruningSize.Should().BeLessThan(7.GB());
+    }
+}

src/Nethermind/Nethermind.Blockchain/FullPruning/FullPruner.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.IO.Abstractions;
 using System.Threading;
 using System.Threading.Tasks;
 using Nethermind.Config;
@@ -28,6 +29,8 @@ public class FullPruner : IDisposable
         private readonly IStateReader _stateReader;
         private readonly IProcessExitSource _processExitSource;
         private readonly ILogManager _logManager;
+        private readonly IChainEstimations _chainEstimations;
+        private readonly IDriveInfo _driveInfo;
         private IPruningContext? _currentPruning;
         private int _waitingForBlockProcessed = 0;
         private int _waitingForStateReady = 0;
@@ -44,6 +47,8 @@ public FullPruner(
             IBlockTree blockTree,
             IStateReader stateReader,
             IProcessExitSource processExitSource,
+            IChainEstimations chainEstimations,
+            IDriveInfo driveInfo,
             ILogManager logManager)
         {
             _fullPruningDb = fullPruningDb;
@@ -53,6 +58,8 @@ public FullPruner(
             _stateReader = stateReader;
             _processExitSource = processExitSource;
             _logManager = logManager;
+            _chainEstimations = chainEstimations;
+            _driveInfo = driveInfo;
             _pruningTrigger.Prune += OnPrune;
             _logger = _logManager.GetClassLogger();
             _minimumPruningDelay = TimeSpan.FromHours(_pruningConfig.FullPruningMinimumDelayHours);
@@ -78,8 +85,13 @@ private void OnPrune(object? sender, PruningTriggerEventArgs e)
             // If we are already pruning, we don't need to do anything
             else if (CanStartNewPruning())
             {
+                // Check if we have enough disk space to run pruning
+                if (!HaveEnoughDiskSpaceToRun() && _pruningConfig.AvailableSpaceCheckEnabled)
+                {
+                    e.Status = PruningStatus.NotEnoughDiskSpace;
+                }
                 // we mark that we are waiting for block (for thread safety)
-                if (Interlocked.CompareExchange(ref _waitingForBlockProcessed, 1, 0) == 0)
+                else if (Interlocked.CompareExchange(ref _waitingForBlockProcessed, 1, 0) == 0)
                 {
                     // we don't want to start pruning in the middle of block processing, lets wait for new head.
                     _blockTree.OnUpdateMainChain += OnUpdateMainChain;
@@ -158,6 +170,28 @@ private void SetCurrentPruning(IPruningContext pruningContext)
 
         private bool CanStartNewPruning() => _fullPruningDb.CanStartPruning;
 
+        private const double ChainSizeThresholdFactor = 1.3;
+
+        private bool HaveEnoughDiskSpaceToRun()
+        {
+            long? currentChainSize = _chainEstimations.PruningSize;
+            if (currentChainSize is null)
+            {
+                if (_logger.IsWarn) _logger.Warn("Full Pruning: Chain size estimation is unavailable.");
+                return true;
+            }
+
+            long available = _driveInfo.AvailableFreeSpace;
+            if (available < currentChainSize * ChainSizeThresholdFactor)
+            {
+                if (_logger.IsWarn)
+                    _logger.Warn(
+                        $"Not enough disk space to run full pruning. Required {(currentChainSize * ChainSizeThresholdFactor) / 1.GB()} GB. Have {available / 1.GB()} GB");
+                return false;
+            }
+            return true;
+        }
+
         private void HandlePruningFinished(object? sender, PruningEventArgs e)
         {
             switch (_pruningConfig.FullPruningCompletionBehavior)

src/Nethermind/Nethermind.Blockchain/FullPruning/PruningStatus.cs

@@ -31,4 +31,9 @@ public enum PruningStatus
     /// Full pruning was triggered and is starting.
     /// </summary>
     Starting,
+
+    /// <summary>
+    /// Pruning failed because of low disk space
+    /// </summary>
+    NotEnoughDiskSpace,
 }

src/Nethermind/Nethermind.Blockchain/KnownChainSizes.cs

@@ -2,46 +2,95 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
-using System.Collections.Generic;
 using Nethermind.Core;
 using Nethermind.Core.Extensions;
 
 namespace Nethermind.Blockchain
 {
-    public static class Known
+    public interface IChainEstimations
     {
-        public readonly struct SizeInfo
+        long? StateSize { get; }
+        long? PruningSize { get; }
+    }
+
+    public static class ChainSizes
+    {
+        public class UnknownChain : IChainEstimations
+        {
+            public long? StateSize => null;
+            public long? PruningSize => null;
+
+            public static readonly IChainEstimations Instance = new UnknownChain();
+        }
+
+        private class ChainEstimations : IChainEstimations
         {
-            public SizeInfo(
-                long sizeAtUpdateDate,
-                long dailyGrowth,
-                DateTime updateDate)
+            private readonly LinearExtrapolation? _stateSizeEstimator;
+            private readonly LinearExtrapolation? _prunedStateEstimator;
+
+            public ChainEstimations(LinearExtrapolation? stateSizeEstimator = null, LinearExtrapolation? prunedStateEstimator = null)
             {
-                SizeAtUpdateDate = sizeAtUpdateDate;
-                DailyGrowth = dailyGrowth;
-                UpdateDate = updateDate;
+                _stateSizeEstimator = stateSizeEstimator;
+                _prunedStateEstimator = prunedStateEstimator;
             }
 
-            public long SizeAtUpdateDate { get; }
-            public long DailyGrowth { get; }
-            public DateTime UpdateDate { get; }
+            public long? StateSize => _stateSizeEstimator?.Estimate;
+            public long? PruningSize => _prunedStateEstimator?.Estimate;
+        }
 
-            public long Current => SizeAtUpdateDate + (DateTime.UtcNow - UpdateDate).Days * DailyGrowth;
+        private class LinearExtrapolation
+        {
+            private readonly long _atUpdate;
+            private readonly long _dailyGrowth;
+            private readonly DateTime _updateDate;
+
+            public LinearExtrapolation(long atUpdate, long dailyGrowth, DateTime updateDate)
+            {
+                _atUpdate = atUpdate;
+                _dailyGrowth = dailyGrowth;
+                _updateDate = updateDate;
+            }
+
+            public LinearExtrapolation(long firstValue, DateTime firstDate, long secondValue, DateTime secondDate)
+            {
+                _atUpdate = firstValue;
+                _dailyGrowth = (long)((secondValue - firstValue) / (secondDate - firstDate).TotalDays);
+                _updateDate = firstDate;
+            }
+
+            public long Estimate => _atUpdate + (DateTime.UtcNow - _updateDate).Days * _dailyGrowth;
         }
 
         /// <summary>
         /// Size in bytes, daily growth rate and the date of manual update
         /// </summary>
-        public static Dictionary<ulong, SizeInfo> ChainSize = new()
+        public static IChainEstimations CreateChainSizeInfo(ulong chainId)
         {
-            { BlockchainIds.Goerli, new SizeInfo(8490.MB(), 15.MB(), new DateTime(2021, 12, 7)) },
-            { BlockchainIds.Rinkeby, new SizeInfo(34700.MB(), 20.MB(), new DateTime(2021, 12, 7)) },
-            { BlockchainIds.Ropsten, new SizeInfo(35900.MB(), 25.MB(), new DateTime(2021, 12, 7)) },
-            { BlockchainIds.Mainnet, new SizeInfo(90000.MB(), 70.MB(), new DateTime(2022, 04, 7)) },
-            { BlockchainIds.Gnosis, new SizeInfo(18000.MB(), 48.MB(), new DateTime(2021, 12, 7)) },
-            { BlockchainIds.EnergyWeb, new SizeInfo(15300.MB(), 15.MB(), new DateTime(2021, 12, 7)) },
-            { BlockchainIds.Volta, new SizeInfo(17500.MB(), 10.MB(), new DateTime(2021, 11, 7)) },
-            { BlockchainIds.PoaCore, new SizeInfo(13900.MB(), 4.MB(), new DateTime(2021, 12, 7)) },
-        };
+            return chainId switch
+            {
+                BlockchainIds.Goerli => new ChainEstimations(
+                    new LinearExtrapolation(8490.MB(), 15.MB(), new DateTime(2021, 12, 7))),
+                BlockchainIds.Rinkeby => new ChainEstimations(
+                    new LinearExtrapolation(34700.MB(), 20.MB(), new DateTime(2021, 12, 7))),
+                BlockchainIds.Ropsten => new ChainEstimations(
+                    new LinearExtrapolation(35900.MB(), 25.MB(), new DateTime(2021, 12, 7))),
+                BlockchainIds.Mainnet => new ChainEstimations(
+                    new LinearExtrapolation(90000.MB(), 70.MB(), new DateTime(2022, 04, 7)),
+                    new LinearExtrapolation(163.GB(), new DateTime(2023, 4, 11, 19, 49, 0),
+                        177653453177, new DateTime(2023, 04, 29, 00, 50, 0))),
+                BlockchainIds.Gnosis => new ChainEstimations(
+                    new LinearExtrapolation(18000.MB(), 48.MB(), new DateTime(2021, 12, 7))),
+                BlockchainIds.EnergyWeb => new ChainEstimations(
+                    new LinearExtrapolation(15300.MB(), 15.MB(), new DateTime(2021, 12, 7))),
+                BlockchainIds.Volta => new ChainEstimations(
+                    new LinearExtrapolation(17500.MB(), 10.MB(), new DateTime(2021, 11, 7))),
+                BlockchainIds.PoaCore => new ChainEstimations(
+                    new LinearExtrapolation(13900.MB(), 4.MB(), new DateTime(2021, 12, 7))),
+                BlockchainIds.Sepolia => new ChainEstimations(null,
+                    new LinearExtrapolation(2457.MB(), new DateTime(2023, 04, 9, 21, 11, 0),
+                        3699505976, new DateTime(2023, 04, 28, 20, 18, 0))),
+                _ => UnknownChain.Instance
+            };
+        }
     }
 }

src/Nethermind/Nethermind.HealthChecks/DbDriveInfoProvider.cs → src/Nethermind/Nethermind.Core/Extensions/DbDriveInfoProvider.cs

@@ -20,7 +20,7 @@
 using System.IO.Abstractions;
 using System.Linq;
 
-namespace Nethermind.HealthChecks
+namespace Nethermind.Core.Extensions
 {
     public static class DbDriveInfoProvider
     {
@@ -30,7 +30,7 @@ static IDriveInfo FindDriveForDirectory(IDriveInfo[] drives, DirectoryInfo dir)
             {
                 string dPath = dir.LinkTarget ?? dir.FullName;
                 IEnumerable<IDriveInfo> candidateDrives = drives.Where(drive => dPath.StartsWith(drive.RootDirectory.FullName));
-                IDriveInfo result = null;
+                IDriveInfo? result = null;
                 foreach (IDriveInfo driveInfo in candidateDrives)
                 {
                     result ??= driveInfo;
@@ -40,7 +40,7 @@ static IDriveInfo FindDriveForDirectory(IDriveInfo[] drives, DirectoryInfo dir)
                     }
                 }
 
-                return result;
+                return result!;
             }
 
             DirectoryInfo topDir = new(dbPath);

src/Nethermind/Nethermind.Core/Nethermind.Core.csproj

@@ -13,6 +13,7 @@
     <PackageReference Include="Microsoft.Extensions.ObjectPool" />
     <PackageReference Include="Nethermind.Crypto.SecP256k1" />
     <PackageReference Include="Nethermind.Numerics.Int256" />
+    <PackageReference Include="TestableIO.System.IO.Abstractions.Wrappers" />
     <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="System.IdentityModel.Tokens.Jwt" />
   </ItemGroup>

src/Nethermind/Nethermind.Db/IPruningConfig.cs

@@ -58,5 +58,8 @@ public interface IPruningConfig : IConfig
                                   "'AlwaysShutdown': shuts Nethermind down once the prune completes, whether it succeeded or failed.",
             DefaultValue = "None")]
         FullPruningCompletionBehavior FullPruningCompletionBehavior { get; set; }
+
+        [ConfigItem(Description = "Disables available disk space check.", DefaultValue = "true", HiddenFromDocs = true)]
+        bool AvailableSpaceCheckEnabled { get; set; }
     }
 }

src/Nethermind/Nethermind.Db/PruningConfig.cs

@@ -30,5 +30,6 @@ public bool Enabled
         public int FullPruningMemoryBudgetMb { get; set; } = 0;
         public int FullPruningMinimumDelayHours { get; set; } = 240;
         public FullPruningCompletionBehavior FullPruningCompletionBehavior { get; set; } = FullPruningCompletionBehavior.None;
+        public bool AvailableSpaceCheckEnabled { get; set; } = true;
     }
 }

src/Nethermind/Nethermind.HealthChecks/HealthChecksPlugin.cs

@@ -14,7 +14,7 @@
 using Nethermind.Logging;
 using Nethermind.JsonRpc;
 using Nethermind.Monitoring.Config;
-using Nethermind.Core.Exceptions;
+using Nethermind.Core.Extensions;
 
 namespace Nethermind.HealthChecks
 {

src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO.Abstractions;
 using System.Threading;
 using System.Threading.Tasks;
 using Nethermind.Api;
@@ -314,7 +315,10 @@ private static void InitializeFullPruning(
                         api.PruningTrigger.Add(pruningTrigger);
                     }
 
-                    FullPruner pruner = new(fullPruningDb, api.PruningTrigger, pruningConfig, api.BlockTree!, stateReader, api.ProcessExit!, api.LogManager);
+                    IDriveInfo drive = api.FileSystem.GetDriveInfos(fullPruningDb.GetPath(initConfig.BaseDbPath))[0];
+                    FullPruner pruner = new(fullPruningDb, api.PruningTrigger, pruningConfig, api.BlockTree!,
+                        stateReader, api.ProcessExit!, ChainSizes.CreateChainSizeInfo(api.ChainSpec!.ChainId),
+                        drive, api.LogManager);
                     api.DisposeStack.Push(pruner);
                 }
             }