Skip to content

Commit

Permalink
Rename WriteBytesPerSec to BytesPerSec and add direct io flag. (#5641)
Browse files Browse the repository at this point in the history 
* Rename WriteBytesPerSec to BytesPerSec

* Fix build and add flags to turn on direct io

* Whitespace
  • Loading branch information
@asdacap
asdacap authored May 3, 2023
1 parent c2bdcdd commit f6b1fa2
Show file tree
Hide file tree
Showing 4 changed files with 76 additions and 15 deletions.
34 changes: 34 additions & 0 deletions src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@ public class DbConfig : IDbConfig
public ulong BlockCacheSize { get; set; } = 0;
public bool CacheIndexAndFilterBlocks { get; set; } = false;
public int? MaxOpenFiles { get; set; }
public long? MaxBytesPerSec { get; set; }
public long? MaxWriteBytesPerSec { get; set; }
public int? BlockSize { get; set; } = 16 * 1024;
public bool? UseDirectReads { get; set; } = false;
public bool? UseDirectIoForFlushAndCompactions { get; set; } = false;
public IDictionary<string, string>? AdditionalRocksDbOptions { get; set; }

public ulong ReceiptsDbWriteBufferSize { get; set; } = (ulong)8.MiB();
Expand All @@ -28,7 +31,10 @@ public class DbConfig : IDbConfig
public bool ReceiptsDbCacheIndexAndFilterBlocks { get; set; } = false;
public int? ReceiptsDbMaxOpenFiles { get; set; }
public long? ReceiptsDbMaxWriteBytesPerSec { get; set; }
public long? ReceiptsDbMaxBytesPerSec { get; set; }
public int? ReceiptsBlockSize { get; set; }
public bool? ReceiptsUseDirectReads { get; set; } = false;
public bool? ReceiptsUseDirectIoForFlushAndCompactions { get; set; } = false;
public IDictionary<string, string>? ReceiptsDbAdditionalRocksDbOptions { get; set; }

public ulong BlocksDbWriteBufferSize { get; set; } = (ulong)8.MiB();
Expand All @@ -37,7 +43,10 @@ public class DbConfig : IDbConfig
public bool BlocksDbCacheIndexAndFilterBlocks { get; set; } = false;
public int? BlocksDbMaxOpenFiles { get; set; }
public long? BlocksDbMaxWriteBytesPerSec { get; set; }
public long? BlocksDbMaxBytesPerSec { get; set; }
public int? BlocksBlockSize { get; set; }
public bool? BlocksUseDirectReads { get; set; } = false;
public bool? BlocksUseDirectIoForFlushAndCompactions { get; set; } = false;
public IDictionary<string, string>? BlocksDbAdditionalRocksDbOptions { get; set; }

public ulong HeadersDbWriteBufferSize { get; set; } = (ulong)8.MiB();
Expand All @@ -46,7 +55,10 @@ public class DbConfig : IDbConfig
public bool HeadersDbCacheIndexAndFilterBlocks { get; set; } = false;
public int? HeadersDbMaxOpenFiles { get; set; }
public long? HeadersDbMaxWriteBytesPerSec { get; set; }
public long? HeadersDbMaxBytesPerSec { get; set; }
public int? HeadersBlockSize { get; set; } = 4 * 1024;
public bool? HeadersUseDirectReads { get; set; } = false;
public bool? HeadersUseDirectIoForFlushAndCompactions { get; set; } = false;
public IDictionary<string, string>? HeadersDbAdditionalRocksDbOptions { get; set; }

public ulong BlockInfosDbWriteBufferSize { get; set; } = (ulong)8.MiB();
Expand All @@ -55,7 +67,10 @@ public class DbConfig : IDbConfig
public bool BlockInfosDbCacheIndexAndFilterBlocks { get; set; } = false;
public int? BlockInfosDbMaxOpenFiles { get; set; }
public long? BlockInfosDbMaxWriteBytesPerSec { get; set; }
public long? BlockInfosDbMaxBytesPerSec { get; set; }
public int? BlockInfosBlockSize { get; set; }
public bool? BlockInfosUseDirectReads { get; set; } = false;
public bool? BlockInfosUseDirectIoForFlushAndCompactions { get; set; } = false;
public IDictionary<string, string>? BlockInfosDbAdditionalRocksDbOptions { get; set; }

public ulong PendingTxsDbWriteBufferSize { get; set; } = (ulong)4.MiB();
Expand All @@ -64,7 +79,10 @@ public class DbConfig : IDbConfig
public bool PendingTxsDbCacheIndexAndFilterBlocks { get; set; } = false;
public int? PendingTxsDbMaxOpenFiles { get; set; }
public long? PendingTxsDbMaxWriteBytesPerSec { get; set; }
public long? PendingTxsDbMaxBytesPerSec { get; set; }
public int? PendingTxsBlockSize { get; set; }
public bool? PendingTxsUseDirectReads { get; set; } = false;
public bool? PendingTxsUseDirectIoForFlushAndCompactions { get; set; } = false;
public IDictionary<string, string>? PendingTxsDbAdditionalRocksDbOptions { get; set; }

public ulong CodeDbWriteBufferSize { get; set; } = (ulong)2.MiB();
Expand All @@ -73,7 +91,10 @@ public class DbConfig : IDbConfig
public bool CodeDbCacheIndexAndFilterBlocks { get; set; } = false;
public int? CodeDbMaxOpenFiles { get; set; }
public long? CodeDbMaxWriteBytesPerSec { get; set; }
public long? CodeDbMaxBytesPerSec { get; set; }
public int? CodeBlockSize { get; set; }
public bool? CodeUseDirectReads { get; set; } = false;
public bool? CodeUseDirectIoForFlushAndCompactions { get; set; } = false;
public IDictionary<string, string>? CodeDbAdditionalRocksDbOptions { get; set; }

public ulong BloomDbWriteBufferSize { get; set; } = (ulong)1.KiB();
Expand All @@ -82,6 +103,7 @@ public class DbConfig : IDbConfig
public bool BloomDbCacheIndexAndFilterBlocks { get; set; } = false;
public int? BloomDbMaxOpenFiles { get; set; }
public long? BloomDbMaxWriteBytesPerSec { get; set; }
public long? BloomDbMaxBytesPerSec { get; set; }
public IDictionary<string, string>? BloomDbAdditionalRocksDbOptions { get; set; }

public ulong WitnessDbWriteBufferSize { get; set; } = (ulong)1.KiB();
Expand All @@ -90,7 +112,10 @@ public class DbConfig : IDbConfig
public bool WitnessDbCacheIndexAndFilterBlocks { get; set; } = false;
public int? WitnessDbMaxOpenFiles { get; set; }
public long? WitnessDbMaxWriteBytesPerSec { get; set; }
public long? WitnessDbMaxBytesPerSec { get; set; }
public int? WitnessBlockSize { get; set; }
public bool? WitnessUseDirectReads { get; set; } = false;
public bool? WitnessUseDirectIoForFlushAndCompactions { get; set; } = false;
public IDictionary<string, string>? WitnessDbAdditionalRocksDbOptions { get; set; }

// TODO - profile and customize
Expand All @@ -100,7 +125,10 @@ public class DbConfig : IDbConfig
public bool CanonicalHashTrieDbCacheIndexAndFilterBlocks { get; set; } = false;
public int? CanonicalHashTrieDbMaxOpenFiles { get; set; }
public long? CanonicalHashTrieDbMaxWriteBytesPerSec { get; set; }
public long? CanonicalHashTrieDbMaxBytesPerSec { get; set; }
public int? CanonicalHashTrieBlockSize { get; set; }
public bool? CanonicalHashTrieUseDirectReads { get; set; } = false;
public bool? CanonicalHashTrieUseDirectIoForFlushAndCompactions { get; set; } = false;
public IDictionary<string, string>? CanonicalHashTrieDbAdditionalRocksDbOptions { get; set; }

public ulong MetadataDbWriteBufferSize { get; set; } = (ulong)1.KiB();
Expand All @@ -109,7 +137,10 @@ public class DbConfig : IDbConfig
public bool MetadataDbCacheIndexAndFilterBlocks { get; set; } = false;
public int? MetadataDbMaxOpenFiles { get; set; }
public long? MetadataDbMaxWriteBytesPerSec { get; set; }
public long? MetadataDbMaxBytesPerSec { get; set; }
public int? MetadataBlockSize { get; set; }
public bool? MetadataUseDirectReads { get; set; } = false;
public bool? MetadataUseDirectIoForFlushAndCompactions { get; set; } = false;
public IDictionary<string, string>? MetadataDbAdditionalRocksDbOptions { get; set; }

public ulong StateDbWriteBufferSize { get; set; }
Expand All @@ -118,7 +149,10 @@ public class DbConfig : IDbConfig
public bool StateDbCacheIndexAndFilterBlocks { get; set; }
public int? StateDbMaxOpenFiles { get; set; }
public long? StateDbMaxWriteBytesPerSec { get; set; }
public long? StateDbMaxBytesPerSec { get; set; }
public int? StateDbBlockSize { get; set; } = 4 * 1024;
public bool? StateDbUseDirectReads { get; set; } = false;
public bool? StateDbUseDirectIoForFlushAndCompactions { get; set; } = false;
public IDictionary<string, string>? StateDbAdditionalRocksDbOptions { get; set; }

public uint RecycleLogFileNum { get; set; } = 0;
Expand Down
46 changes: 34 additions & 12 deletions src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,106 +19,128 @@ public interface IDbConfig : IConfig
int? MaxOpenFiles { get; set; }
uint RecycleLogFileNum { get; set; }
bool WriteAheadLogSync { get; set; }
long? MaxWriteBytesPerSec { get; set; }
long? MaxBytesPerSec { get; set; }
int? BlockSize { get; set; }
bool? UseDirectReads { get; set; }
bool? UseDirectIoForFlushAndCompactions { get; set; }
IDictionary<string, string>? AdditionalRocksDbOptions { get; set; }

ulong ReceiptsDbWriteBufferSize { get; set; }
uint ReceiptsDbWriteBufferNumber { get; set; }
ulong ReceiptsDbBlockCacheSize { get; set; }
bool ReceiptsDbCacheIndexAndFilterBlocks { get; set; }
int? ReceiptsDbMaxOpenFiles { get; set; }
long? ReceiptsDbMaxWriteBytesPerSec { get; set; }
long? ReceiptsDbMaxBytesPerSec { get; set; }
int? ReceiptsBlockSize { get; set; }
bool? ReceiptsUseDirectReads { get; set; }
bool? ReceiptsUseDirectIoForFlushAndCompactions { get; set; }
IDictionary<string, string>? ReceiptsDbAdditionalRocksDbOptions { get; set; }

ulong BlocksDbWriteBufferSize { get; set; }
uint BlocksDbWriteBufferNumber { get; set; }
ulong BlocksDbBlockCacheSize { get; set; }
bool BlocksDbCacheIndexAndFilterBlocks { get; set; }
int? BlocksDbMaxOpenFiles { get; set; }
long? BlocksDbMaxWriteBytesPerSec { get; set; }
long? BlocksDbMaxBytesPerSec { get; set; }
int? BlocksBlockSize { get; set; }
bool? BlocksUseDirectReads { get; set; }
bool? BlocksUseDirectIoForFlushAndCompactions { get; set; }
IDictionary<string, string>? BlocksDbAdditionalRocksDbOptions { get; set; }

ulong HeadersDbWriteBufferSize { get; set; }
uint HeadersDbWriteBufferNumber { get; set; }
ulong HeadersDbBlockCacheSize { get; set; }
bool HeadersDbCacheIndexAndFilterBlocks { get; set; }
int? HeadersDbMaxOpenFiles { get; set; }
long? HeadersDbMaxWriteBytesPerSec { get; set; }
long? HeadersDbMaxBytesPerSec { get; set; }
int? HeadersBlockSize { get; set; }
bool? HeadersUseDirectReads { get; set; }
bool? HeadersUseDirectIoForFlushAndCompactions { get; set; }
IDictionary<string, string>? HeadersDbAdditionalRocksDbOptions { get; set; }

ulong BlockInfosDbWriteBufferSize { get; set; }
uint BlockInfosDbWriteBufferNumber { get; set; }
ulong BlockInfosDbBlockCacheSize { get; set; }
bool BlockInfosDbCacheIndexAndFilterBlocks { get; set; }
int? BlockInfosDbMaxOpenFiles { get; set; }
long? BlockInfosDbMaxWriteBytesPerSec { get; set; }
long? BlockInfosDbMaxBytesPerSec { get; set; }
int? BlockInfosBlockSize { get; set; }
bool? BlockInfosUseDirectReads { get; set; }
bool? BlockInfosUseDirectIoForFlushAndCompactions { get; set; }
IDictionary<string, string>? BlockInfosDbAdditionalRocksDbOptions { get; set; }

ulong PendingTxsDbWriteBufferSize { get; set; }
uint PendingTxsDbWriteBufferNumber { get; set; }
ulong PendingTxsDbBlockCacheSize { get; set; }
bool PendingTxsDbCacheIndexAndFilterBlocks { get; set; }
int? PendingTxsDbMaxOpenFiles { get; set; }
long? PendingTxsDbMaxWriteBytesPerSec { get; set; }
long? PendingTxsDbMaxBytesPerSec { get; set; }
int? PendingTxsBlockSize { get; set; }
bool? PendingTxsUseDirectReads { get; set; }
bool? PendingTxsUseDirectIoForFlushAndCompactions { get; set; }
IDictionary<string, string>? PendingTxsDbAdditionalRocksDbOptions { get; set; }

ulong CodeDbWriteBufferSize { get; set; }
uint CodeDbWriteBufferNumber { get; set; }
ulong CodeDbBlockCacheSize { get; set; }
bool CodeDbCacheIndexAndFilterBlocks { get; set; }
int? CodeDbMaxOpenFiles { get; set; }
long? CodeDbMaxWriteBytesPerSec { get; set; }
long? CodeDbMaxBytesPerSec { get; set; }
int? CodeBlockSize { get; set; }
bool? CodeUseDirectReads { get; set; }
bool? CodeUseDirectIoForFlushAndCompactions { get; set; }
IDictionary<string, string>? CodeDbAdditionalRocksDbOptions { get; set; }

ulong BloomDbWriteBufferSize { get; set; }
uint BloomDbWriteBufferNumber { get; set; }
ulong BloomDbBlockCacheSize { get; set; }
bool BloomDbCacheIndexAndFilterBlocks { get; set; }
int? BloomDbMaxOpenFiles { get; set; }
long? BloomDbMaxWriteBytesPerSec { get; set; }
long? BloomDbMaxBytesPerSec { get; set; }
IDictionary<string, string>? BloomDbAdditionalRocksDbOptions { get; set; }

ulong WitnessDbWriteBufferSize { get; set; }
uint WitnessDbWriteBufferNumber { get; set; }
ulong WitnessDbBlockCacheSize { get; set; }
bool WitnessDbCacheIndexAndFilterBlocks { get; set; }
int? WitnessDbMaxOpenFiles { get; set; }
long? WitnessDbMaxWriteBytesPerSec { get; set; }
long? WitnessDbMaxBytesPerSec { get; set; }
int? WitnessBlockSize { get; set; }
bool? WitnessUseDirectReads { get; set; }
bool? WitnessUseDirectIoForFlushAndCompactions { get; set; }
IDictionary<string, string>? WitnessDbAdditionalRocksDbOptions { get; set; }

ulong CanonicalHashTrieDbWriteBufferSize { get; set; }
uint CanonicalHashTrieDbWriteBufferNumber { get; set; }
ulong CanonicalHashTrieDbBlockCacheSize { get; set; }
bool CanonicalHashTrieDbCacheIndexAndFilterBlocks { get; set; }
int? CanonicalHashTrieDbMaxOpenFiles { get; set; }
long? CanonicalHashTrieDbMaxWriteBytesPerSec { get; set; }
long? CanonicalHashTrieDbMaxBytesPerSec { get; set; }
int? CanonicalHashTrieBlockSize { get; set; }
bool? CanonicalHashTrieUseDirectReads { get; set; }
bool? CanonicalHashTrieUseDirectIoForFlushAndCompactions { get; set; }
IDictionary<string, string>? CanonicalHashTrieDbAdditionalRocksDbOptions { get; set; }

ulong MetadataDbWriteBufferSize { get; set; }
uint MetadataDbWriteBufferNumber { get; set; }
ulong MetadataDbBlockCacheSize { get; set; }
bool MetadataDbCacheIndexAndFilterBlocks { get; set; }
int? MetadataDbMaxOpenFiles { get; set; }
long? MetadataDbMaxWriteBytesPerSec { get; set; }
long? MetadataDbMaxBytesPerSec { get; set; }
int? MetadataBlockSize { get; set; }
bool? MetadataUseDirectReads { get; set; }
bool? MetadataUseDirectIoForFlushAndCompactions { get; set; }
IDictionary<string, string>? MetadataDbAdditionalRocksDbOptions { get; set; }

ulong StateDbWriteBufferSize { get; set; }
uint StateDbWriteBufferNumber { get; set; }
ulong StateDbBlockCacheSize { get; set; }
bool StateDbCacheIndexAndFilterBlocks { get; set; }
int? StateDbMaxOpenFiles { get; set; }
long? StateDbMaxWriteBytesPerSec { get; set; }
long? StateDbMaxBytesPerSec { get; set; }
int? StateDbBlockSize { get; set; }
bool? StateDbUseDirectReads { get; set; }
bool? StateDbUseDirectIoForFlushAndCompactions { get; set; }
IDictionary<string, string>? StateDbAdditionalRocksDbOptions { get; set; }

/// <summary>
Expand Down
4 changes: 3 additions & 1 deletion src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,11 @@ public PerTableDbConfig(IDbConfig dbConfig, RocksDbSettings rocksDbSettings, str
public IDictionary<string, string>? AdditionalRocksDbOptions => ReadConfig<IDictionary<string, string>?>(nameof(AdditionalRocksDbOptions));

public int? MaxOpenFiles => ReadConfig<int?>(nameof(MaxOpenFiles));
public long? MaxWriteBytesPerSec => ReadConfig<long?>(nameof(MaxWriteBytesPerSec));
public long? MaxBytesPerSec => ReadConfig<long?>(nameof(MaxBytesPerSec));
public uint RecycleLogFileNum => ReadConfig<uint>(nameof(RecycleLogFileNum));
public bool WriteAheadLogSync => ReadConfig<bool>(nameof(WriteAheadLogSync));
public bool UseDirectReads => ReadConfig<bool>(nameof(UseDirectReads));
public bool UseDirectIoForFlushAndCompactions => ReadConfig<bool>(nameof(UseDirectIoForFlushAndCompactions));
public int? BlockSize => ReadConfig<int?>(nameof(BlockSize));
public bool EnableDbStatistics => _dbConfig.EnableDbStatistics;
public uint StatsDumpPeriodSec => _dbConfig.StatsDumpPeriodSec;
Expand Down
7 changes: 5 additions & 2 deletions src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,10 +350,10 @@ protected virtual void BuildOptions<T>(PerTableDbConfig dbConfig, Options<T> opt
options.SetMaxOpenFiles(dbConfig.MaxOpenFiles.Value);
}

if (dbConfig.MaxWriteBytesPerSec.HasValue)
if (dbConfig.MaxBytesPerSec.HasValue)
{
_rateLimiter =
_rocksDbNative.rocksdb_ratelimiter_create(dbConfig.MaxWriteBytesPerSec.Value, 1000, 10);
_rocksDbNative.rocksdb_ratelimiter_create(dbConfig.MaxBytesPerSec.Value, 1000, 10);
_rocksDbNative.rocksdb_options_set_ratelimiter(options.Handle, _rateLimiter.Value);
}

Expand Down Expand Up @@ -381,6 +381,9 @@ protected virtual void BuildOptions<T>(PerTableDbConfig dbConfig, Options<T> opt
options.SetRecycleLogFileNum(dbConfig
.RecycleLogFileNum); // potential optimization for reusing allocated log files

options.SetUseDirectReads(dbConfig.UseDirectReads);
options.SetUseDirectIoForFlushAndCompaction(dbConfig.UseDirectIoForFlushAndCompactions);

// VERY important to reduce stalls. Allow L0->L1 compaction to happen with multiple thread.
_rocksDbNative.rocksdb_options_set_max_subcompactions(options.Handle, (uint)Environment.ProcessorCount);

Expand Down

0 comments on commit f6b1fa2

Please sign in to comment.