Feature DB File warmer

src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs

@@ -51,6 +51,7 @@ public class DbConfig : IDbConfig
     public int? UseRibbonFilterStartingFromLevel { get; set; }
     public ulong BytesPerSync { get; set; } = 0;
     public double? DataBlockIndexUtilRatio { get; set; }
+    public bool EnableFileWarmer { get; set; } = false;
 
     public ulong BlobTransactionsDbBlockCacheSize { get; set; } = (ulong)32.MiB();
 
@@ -200,6 +201,7 @@ public class DbConfig : IDbConfig
     public int? StateDbBloomFilterBitsPerKey { get; set; } = 15;
     public int? StateDbUseRibbonFilterStartingFromLevel { get; set; } = 2;
     public double? StateDbDataBlockIndexUtilRatio { get; set; } = 0.5;
+    public bool StateDbEnableFileWarmer { get; set; }
     public IDictionary<string, string>? StateDbAdditionalRocksDbOptions { get; set; }
 
     public uint RecycleLogFileNum { get; set; } = 0;

src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs

@@ -52,6 +52,7 @@ public interface IDbConfig : IConfig
     int? UseRibbonFilterStartingFromLevel { get; set; }
     ulong BytesPerSync { get; set; }
     double? DataBlockIndexUtilRatio { get; set; }
+    bool EnableFileWarmer { get; set; }
 
     ulong BlobTransactionsDbBlockCacheSize { get; set; }
 
@@ -201,6 +202,7 @@ public interface IDbConfig : IConfig
     int? StateDbBloomFilterBitsPerKey { get; set; }
     int? StateDbUseRibbonFilterStartingFromLevel { get; set; }
     double? StateDbDataBlockIndexUtilRatio { get; set; }
+    bool StateDbEnableFileWarmer { get; set; }
     IDictionary<string, string>? StateDbAdditionalRocksDbOptions { get; set; }
 
     /// <summary>

src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs

@@ -73,6 +73,7 @@ public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? colum
     public int? UseRibbonFilterStartingFromLevel => ReadConfig<int?>(nameof(UseRibbonFilterStartingFromLevel));
     public ulong BytesPerSync => ReadConfig<ulong>(nameof(BytesPerSync));
     public double? DataBlockIndexUtilRatio => ReadConfig<double?>(nameof(DataBlockIndexUtilRatio));
+    public bool EnableFileWarmer => ReadConfig<bool>(nameof(EnableFileWarmer));
 
     private T? ReadConfig<T>(string propertyName)
     {

src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs

@@ -7,8 +7,11 @@
 using System.Collections.Generic;
 using System.IO;
 using System.IO.Abstractions;
+using System.Linq;
 using System.Reflection;
 using System.Threading;
+using System.Threading.Channels;
+using System.Threading.Tasks;
 using ConcurrentCollections;
 using Nethermind.Config;
 using Nethermind.Core;
@@ -183,6 +186,11 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan
                 }
             }
 
+            if (_perTableDbConfig.EnableFileWarmer)
+            {
+                WarmupFile(_fullPath, db);
+            }
+
             return db;
         }
         catch (DllNotFoundException e) when (e.Message.Contains("libdl"))
@@ -202,6 +210,81 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan
 
     }
 
+    private void WarmupFile(string basePath, RocksDb db)
+    {
+        long availableMemory = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes;
+        _logger.Info($"Warming up database {Name} assuming {availableMemory} bytes of available memory");
+        List<(FileMetadata metadata, DateTime creationTime)> fileMetadatas = new();
+
+        foreach (LiveFileMetadata liveFileMetadata in db.GetLiveFilesMetadata())
+        {
+            string fullPath = Path.Join(basePath, liveFileMetadata.FileMetadata.FileName);
+            try
+            {
+                DateTime creationTime = File.GetCreationTimeUtc(fullPath);
+                fileMetadatas.Add((liveFileMetadata.FileMetadata, creationTime));
+            }
+            catch (IOException)
+            {
+                // Maybe the file is gone or something. We ignore it.
+            }
+        }
+
+        fileMetadatas.Sort((item1, item2) =>
+        {
+            // Sort them by level so that lower level get priority
+            int levelDiff = item1.metadata.FileLevel - item2.metadata.FileLevel;
+            if (levelDiff != 0) return levelDiff;
+
+            // Otherwise, we pick which file is newest.
+            return item2.creationTime.CompareTo(item1.creationTime);
+        });
+
+        long totalSize = 0;
+        fileMetadatas = fileMetadatas.TakeWhile(metadata =>
+        {
+            availableMemory -= (long)metadata.metadata.FileSize;
+            bool take = availableMemory > 0;
+            if (take)
+            {
+                totalSize += (long)metadata.metadata.FileSize;
+            }
+            return take;
+        })
+            // We reverse them again so that lower level goes last so that it is the freshest.
+            // Not all of the available memory is actually available so we are probably over reading things.
+            .Reverse()
+            .ToList();
+
+        long totalRead = 0;
+        Parallel.ForEach(fileMetadatas, (task) =>
+        {
+            string fullPath = Path.Join(basePath, task.metadata.FileName);
+            _logger.Info($"{(totalRead * 100 / (double)totalSize):00.00}% Warming up file {fullPath}");
+
+            try
+            {
+                byte[] buffer = new byte[512.KiB()];
+                using FileStream stream = File.OpenRead(fullPath);
+                int readCount = buffer.Length;
+                while (readCount == buffer.Length)
+                {
+                    readCount = stream.Read(buffer);
+                    Interlocked.Add(ref totalRead, readCount);
+                }
+            }
+            catch (FileNotFoundException)
+            {
+                // Happens sometimes. We do nothing here.
+            }
+            catch (IOException e)
+            {
+                // Something unusual, but nothing noteworthy.
+                _logger.Warn($"Exception warming up {fullPath} {e}");
+            }
+        });
+    }
+
     private void CreateMarkerIfCorrupt(RocksDbSharpException rocksDbException)
     {
         if (rocksDbException.Message.Contains("Corruption:"))

src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs

@@ -129,6 +129,24 @@ public void Dispose_wont_cause_ObjectDisposedException_when_batch_is_still_open(
             db.Dispose();
         }
 
+        [Test]
+        public void CanOpenWithFileWarmer()
+        {
+            IDbConfig config = new DbConfig();
+            config.EnableFileWarmer = true;
+            {
+                using DbOnTheRocks db = new("testFileWarmer", GetRocksDbSettings("testFileWarmer", "FileWarmerTest"), config, LimboLogs.Instance);
+                for (int i = 0; i < 1000; i++)
+                {
+                    db[i.ToBigEndianByteArray()] = i.ToBigEndianByteArray();
+                }
+            }
+
+            {
+                using DbOnTheRocks db = new("testFileWarmer", GetRocksDbSettings("testFileWarmer", "FileWarmerTest"), config, LimboLogs.Instance);
+            }
+        }
+
         [Test]
         public void Corrupted_exception_on_open_would_create_marker()
         {