Eip1559 - Fee market change for ETH 1.0 chain (#2341)

* init eip1559 txpool

* first stage of removing NDM from Nethermind solution

* goerli synchronizing

* tests passing lol

* data protection

* silly

* try to resolve this ref conflict

* 3.1.8

* 3.1.8

* refs...

* check the rest

* next try

* back proxy

* remove ndm refs

* try to resolve on clean

* remove YAML

* more updates

* protection...

* 16.7.1

* try without git tag

* 16.7.1 ever

* bring back git describe

* 16.7.1

* try with api moved out

* no CLI test

* plugins

* update

* one more

* adding projs

* 16.7.1 one more

* bigger ref review

* remove CLI

* move IInitConfig

* IInitCOnfig fix

* allow CLI modules loading from other assemblies

* clean CLI

* type discovery and fixed the issue with assembly loader contexts

* a bit better info on initial timings

* resolve IPs in parallel

* fixed nethdev init

* discover / initialize and dynamics for test

* ipResolver tests fix

* type discovery non static for tests

* dynamic causing trouble

* move validator store to AuraAPi

* NDM api initializer improvements and some comments on the Nethermind API

* Baseline and Vault plugins

* some cleanup around plugins

* fix the build after IPlugin -> INethermindPlugin rename

* removed apprunner, apprunnerbase, cleared startup and bootstrap, cleared json rpc loaders and grpc and publishers and much more

* simpler CLI modules loading

* remove unused usings

* post review fixes

* NDM plugin fixes

* add runner to tests

* fix cc build file

* how do I stop this

* resolving frombasedir too

* ANalytics added as a full plugin example

* tx pool listener

* runner tests name

* removed unused classes

* plugin loader for NDM

* fixed test actions namings

* Revert "init eip1559 txpool"

This reverts commit cbf656d.

* new header encoding

* eip1559 gas targets

* limits for gas in txs

* configs

* eip1559

* various fixes

* fully synced eip1559

* gas limit vs gas target

* post merge build

* post merge fixes

* remove diag

* unnecessary change

* less changes

* update eip1559 duration

* set base fee in Clique

* fix no split on gas limit check

* a + b

* tx pool generating transactions

* large testnet

* one more fix

* various test fixes post merge

* post merge ethereum tests build

* fix to high limits on eip1559

* forgiving diag tracer

src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs

@@ -88,7 +88,7 @@ public static BlockHeader Convert(TestBlockHeaderJson headerJson)
             );
 
             header.Bloom = new Bloom(Bytes.FromHexString(headerJson.Bloom));
-            header.GasUsed = (long) Bytes.FromHexString(headerJson.GasUsed).ToUnsignedBigInteger();
+            header.GasUsedLegacy = (long) Bytes.FromHexString(headerJson.GasUsed).ToUnsignedBigInteger();
             header.Hash = new Keccak(headerJson.Hash);
             header.MixHash = new Keccak(headerJson.MixHash);
             header.Nonce = (ulong) Bytes.FromHexString(headerJson.Nonce).ToUnsignedBigInteger();
@@ -274,4 +274,4 @@ public static IEnumerable<BlockchainTest> ConvertToBlockchainTests(string json)
             return testsByName;
         }
     }
-}
+}

src/Nethermind/Nethermind.Api/BasicApiExtensions.cs

@@ -27,7 +27,7 @@ public static T Config<T>(this IBasicApi api) where T : IConfig
             return api.ConfigProvider.GetConfig<T>();
         }
 
-        public static T Db<T>(this IBasicApi api, string dbName) where T : IDb
+        public static T Db<T>(this IBasicApi api, string dbName) where T : class, IDb
         {
             return api.DbProvider!.GetDb<T>(dbName);
         }

src/Nethermind/Nethermind.Blockchain.Test/Validators/HeaderValidatorTests.cs

@@ -128,7 +128,7 @@ public void When_gas_limit_just_correct_low()
         [Test]
         public void When_gas_used_above_gas_limit()
         {
-            _block.Header.GasUsed = _parentBlock.Header.GasLimit + 1;
+            _block.Header.GasUsedLegacy = _parentBlock.Header.GasLimit + 1;
             _block.Header.SealEngineType = SealEngineType.None;
             _block.Header.Hash = _block.CalculateHash();
 

src/Nethermind/Nethermind.Blockchain/Processing/BlockProcessor.cs

@@ -311,7 +311,8 @@ private Block PrepareBlockForProcessing(Block suggestedBlock)
                 TotalDifficulty = bh.TotalDifficulty,
                 AuRaStep = bh.AuRaStep,
                 AuRaSignature = bh.AuRaSignature,
-                ReceiptsRoot = bh.ReceiptsRoot
+                ReceiptsRoot = bh.ReceiptsRoot,
+                BaseFee = bh.BaseFee
             };
 
             return new Block(header, suggestedBlock.Transactions, suggestedBlock.Ommers);

src/Nethermind/Nethermind.Blockchain/Validators/HeaderValidator.cs

@@ -21,6 +21,7 @@
 using Nethermind.Core.Extensions;
 using Nethermind.Core.Specs;
 using Nethermind.Crypto;
+using Nethermind.Int256;
 using Nethermind.Logging;
 
 namespace Nethermind.Blockchain.Validators
@@ -82,8 +83,7 @@ public bool Validate(BlockHeader header, BlockHeader parent, bool isOmmer = fals
             {
                 if (header.Number == 0)
                 {
-                    var isGenesisValid = ValidateGenesis(header);
-                    ;
+                    bool isGenesisValid = ValidateGenesis(header);
                     if (!isGenesisValid)
                     {
                         if (_logger.IsWarn) _logger.Warn($"Invalid genesis block header ({header.Hash})");
@@ -113,7 +113,7 @@ public bool Validate(BlockHeader header, BlockHeader parent, bool isOmmer = fals
                 if (_logger.IsWarn) _logger.Warn($"Invalid block header ({header.Hash}) - seal parameters incorrect");
             }
 
-            bool gasUsedBelowLimit = header.GasUsed <= header.GasLimit;
+            bool gasUsedBelowLimit = header.GasUsed <= header.GetGasTarget1559(spec) * 2 + header.GetGasTargetLegacy(spec);
             if (!gasUsedBelowLimit)
             {
                 if (_logger.IsWarn) _logger.Warn($"Invalid block header ({header.Hash}) - gas used above gas limit");
@@ -136,6 +136,14 @@ public bool Validate(BlockHeader header, BlockHeader parent, bool isOmmer = fals
 
             if (_logger.IsTrace) _logger.Trace($"Validating block {header.ToString(BlockHeader.Format.Short)}, extraData {header.ExtraData.ToHexString(true)}");
 
+            bool baseFeeIsCorrect = true;
+
+            UInt256 expectedBaseFee = BlockHeader.CalculateBaseFee(parent, spec);
+            if (spec.IsEip1559Enabled)
+            {
+                baseFeeIsCorrect = expectedBaseFee == header.BaseFee;
+            }
+
             return
                 totalDifficultyCorrect &&
                 gasUsedBelowLimit &&
@@ -145,13 +153,14 @@ public bool Validate(BlockHeader header, BlockHeader parent, bool isOmmer = fals
                 timestampMoreThanAtParent &&
                 numberIsParentPlusOne &&
                 hashAsExpected &&
-                extraDataValid;
+                extraDataValid &&
+                baseFeeIsCorrect;
         }
 
         protected virtual bool ValidateGasLimitRange(BlockHeader header, BlockHeader parent, IReleaseSpec spec)
         {
             long maxGasLimitDifference = parent.GasLimit / spec.GasLimitBoundDivisor;
-            
+
             bool gasLimitNotTooHigh = header.GasLimit <= parent.GasLimit + maxGasLimitDifference;
             if (!gasLimitNotTooHigh)
             {

src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs

@@ -182,9 +182,9 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f
                     new CryptoRandom(),
                     snapshotManager,
                     cliqueSealer,
-                    new TargetAdjustedGasLimitCalculator(GoerliSpecProvider.Instance, new MiningConfig()), 
-                    _cliqueConfig,
+                    new TargetAdjustedGasLimitCalculator(GoerliSpecProvider.Instance, new MiningConfig()),
                     MainnetSpecProvider.Instance, 
+                    _cliqueConfig,
                     nodeLogManager);
                 blockProducer.Start();
 

src/Nethermind/Nethermind.Clique.Test/CliqueRpcModuleTests.cs

@@ -20,6 +20,8 @@
 using Nethermind.Consensus.Clique;
 using Nethermind.Consensus.Transactions;
 using Nethermind.Core;
+using Nethermind.Core.Crypto;
+using Nethermind.Core.Specs;
 using Nethermind.Core.Test.Builders;
 using Nethermind.Crypto;
 using Nethermind.Db;
@@ -50,9 +52,9 @@ public void Sets_clique_block_producer_properly()
                 Substitute.For<ICryptoRandom>(),
                 Substitute.For<ISnapshotManager>(),
                 new CliqueSealer(signer, cliqueConfig, Substitute.For<ISnapshotManager>(), LimboLogs.Instance),
-                new TargetAdjustedGasLimitCalculator(GoerliSpecProvider.Instance, new MiningConfig()), 
-                cliqueConfig,
+                new TargetAdjustedGasLimitCalculator(GoerliSpecProvider.Instance, new MiningConfig()),
                 MainnetSpecProvider.Instance, 
+                cliqueConfig,
                 LimboLogs.Instance);
 
             SnapshotManager snapshotManager = new SnapshotManager(CliqueConfig.Default, new MemDb(), Substitute.For<IBlockTree>(), NullEthereumEcdsa.Instance, LimboLogs.Instance);

src/Nethermind/Nethermind.Consensus.Clique/CliqueBlockProducer.cs

@@ -51,9 +51,10 @@ public class CliqueBlockProducer : ICliqueBlockProducer, IDisposable
         private readonly IBlockchainProcessor _processor;
         private readonly ISealer _sealer;
         private readonly IGasLimitCalculator _gasLimitCalculator;
+        private readonly ISpecProvider _specProvider;
         private readonly ISnapshotManager _snapshotManager;
         private readonly ICliqueConfig _config;
-        private readonly ISpecProvider _spec;
+
         private readonly ConcurrentDictionary<Address, bool> _proposals = new ConcurrentDictionary<Address, bool>();
 
         private readonly CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
@@ -69,8 +70,8 @@ public CliqueBlockProducer(
             ISnapshotManager snapshotManager,
             ISealer cliqueSealer,
             IGasLimitCalculator gasLimitCalculator,
+            ISpecProvider? specProvider,
             ICliqueConfig config,
-            ISpecProvider? spec,
             ILogManager logManager)
         {
             _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager));
@@ -82,9 +83,9 @@ public CliqueBlockProducer(
             _cryptoRandom = cryptoRandom ?? throw new ArgumentNullException(nameof(cryptoRandom));
             _sealer = cliqueSealer ?? throw new ArgumentNullException(nameof(cliqueSealer));
             _gasLimitCalculator = gasLimitCalculator ?? throw new ArgumentNullException(nameof(gasLimitCalculator));
+            _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider));
             _snapshotManager = snapshotManager ?? throw new ArgumentNullException(nameof(snapshotManager));
             _config = config ?? throw new ArgumentNullException(nameof(config));
-            _spec = spec ?? throw new ArgumentNullException(nameof(spec));
             _wiggle = new WiggleRandomizer(_cryptoRandom, _snapshotManager);
 
             _timer.AutoReset = false;
@@ -93,7 +94,8 @@ public CliqueBlockProducer(
             _timer.Start();
         }
 
-        private readonly BlockingCollection<Block> _signalsQueue = new BlockingCollection<Block>(new ConcurrentQueue<Block>());
+        private readonly BlockingCollection<Block> _signalsQueue =
+            new BlockingCollection<Block>(new ConcurrentQueue<Block>());
 
         private Block? _scheduledBlock;
 
@@ -133,7 +135,7 @@ private void TimerOnElapsed(object sender, ElapsedEventArgs e)
                     _timer.Enabled = true;
                     return;
                 }
-                
+
                 Block? scheduledBlock = _scheduledBlock;
                 if (scheduledBlock == null)
                 {
@@ -147,27 +149,33 @@ private void TimerOnElapsed(object sender, ElapsedEventArgs e)
                 }
 
                 string turnDescription = scheduledBlock.IsInTurn() ? "IN TURN" : "OUT OF TURN";
-                
+
                 int wiggle = _wiggle.WiggleFor(scheduledBlock.Header);
                 if (scheduledBlock.Timestamp * 1000 + (UInt256)wiggle < _timestamper.UnixTime.Milliseconds)
                 {
                     if (scheduledBlock.TotalDifficulty > _blockTree.Head.TotalDifficulty)
                     {
-                        if(ReferenceEquals(scheduledBlock, _scheduledBlock))
+                        if (ReferenceEquals(scheduledBlock, _scheduledBlock))
                         {
-                            BlockHeader parent = _blockTree.FindParentHeader(scheduledBlock.Header, BlockTreeLookupOptions.TotalDifficultyNotNeeded);
+                            BlockHeader parent = _blockTree.FindParentHeader(scheduledBlock.Header,
+                                BlockTreeLookupOptions.TotalDifficultyNotNeeded);
                             Address parentSigner = _snapshotManager.GetBlockSealer(parent);
-                            
+
                             string parentTurnDescription = parent.IsInTurn() ? "IN TURN" : "OUT OF TURN";
-                            string parentDetails = $"{parentTurnDescription} {parent.TimestampDate:HH:mm:ss} {parent.ToString(BlockHeader.Format.Short)} sealed by {KnownAddresses.GetDescription(parentSigner)}";
-
-                            if (_logger.IsInfo) _logger.Info($"Suggesting own {turnDescription} {_scheduledBlock.TimestampDate:HH:mm:ss} {scheduledBlock.ToString(Block.Format.HashNumberDiffAndTx)} based on {parentDetails} after the delay of {wiggle}");
+                            string parentDetails =
+                                $"{parentTurnDescription} {parent.TimestampDate:HH:mm:ss} {parent.ToString(BlockHeader.Format.Short)} sealed by {KnownAddresses.GetDescription(parentSigner)}";
+
+                            if (_logger.IsInfo)
+                                _logger.Info(
+                                    $"Suggesting own {turnDescription} {_scheduledBlock.TimestampDate:HH:mm:ss} {scheduledBlock.ToString(Block.Format.HashNumberDiffAndTx)} based on {parentDetails} after the delay of {wiggle}");
                             _blockTree.SuggestBlock(scheduledBlock);
                         }
                     }
                     else
                     {
-                        if (_logger.IsInfo) _logger.Info($"Dropping a losing block {scheduledBlock.ToString(Block.Format.HashNumberDiffAndTx)}");
+                        if (_logger.IsInfo)
+                            _logger.Info(
+                                $"Dropping a losing block {scheduledBlock.ToString(Block.Format.HashNumberDiffAndTx)}");
                     }
 
                     if (ReferenceEquals(scheduledBlock, _scheduledBlock))
@@ -177,7 +185,8 @@ private void TimerOnElapsed(object sender, ElapsedEventArgs e)
                 }
                 else
                 {
-                    if (_logger.IsTrace) _logger.Trace($"Not yet {scheduledBlock.ToString(Block.Format.HashNumberDiffAndTx)}");
+                    if (_logger.IsTrace)
+                        _logger.Trace($"Not yet {scheduledBlock.ToString(Block.Format.HashNumberDiffAndTx)}");
                 }
 
                 _timer.Enabled = true;
@@ -243,7 +252,8 @@ private void ConsumeSignal()
                     }
 
                     if (_logger.IsInfo) _logger.Info($"Processing prepared block {block.Number}");
-                    Block processedBlock = _processor.Process(block, ProcessingOptions.ProducingBlock, NullBlockTracer.Instance);
+                    Block processedBlock = _processor.Process(block, ProcessingOptions.ProducingBlock,
+                        NullBlockTracer.Instance);
                     if (processedBlock == null)
                     {
                         if (_logger.IsInfo) _logger.Info($"Prepared block has lost the race");
@@ -259,13 +269,16 @@ private void ConsumeSignal()
                         {
                             if (t.Result != null)
                             {
-                                if (_logger.IsInfo) _logger.Info($"Sealed block {t.Result.ToString(Block.Format.HashNumberDiffAndTx)}");
+                                if (_logger.IsInfo)
+                                    _logger.Info($"Sealed block {t.Result.ToString(Block.Format.HashNumberDiffAndTx)}");
                                 _scheduledBlock = t.Result;
                                 Metrics.BlocksSealed++;
                             }
                             else
                             {
-                                if (_logger.IsInfo) _logger.Info($"Failed to seal block {processedBlock.ToString(Block.Format.HashNumberDiffAndTx)} (null seal)");
+                                if (_logger.IsInfo)
+                                    _logger.Info(
+                                        $"Failed to seal block {processedBlock.ToString(Block.Format.HashNumberDiffAndTx)} (null seal)");
                                 Metrics.FailedBlockSeals++;
                             }
                         }
@@ -283,7 +296,10 @@ private void ConsumeSignal()
                 }
                 catch (Exception e)
                 {
-                    if (_logger.IsError) _logger.Error($"Block producer could not produce block on top of {parentBlock.ToString(Block.Format.Short)}", e);
+                    if (_logger.IsError)
+                        _logger.Error(
+                            $"Block producer could not produce block on top of {parentBlock.ToString(Block.Format.Short)}",
+                            e);
                     Metrics.FailedBlockSeals++;
                 }
             }
@@ -303,7 +319,9 @@ public async Task StopAsync()
             BlockHeader parentHeader = parentBlock.Header;
             if (parentHeader == null)
             {
-                if (_logger.IsError) _logger.Error($"Preparing new block on top of {parentBlock.ToString(Block.Format.Short)} - parent header is null");
+                if (_logger.IsError)
+                    _logger.Error(
+                        $"Preparing new block on top of {parentBlock.ToString(Block.Format.Short)} - parent header is null");
                 return null;
             }
 
@@ -319,7 +337,8 @@ public async Task StopAsync()
                 return null;
             }
 
-            if (_logger.IsInfo) _logger.Info($"Preparing new block on top of {parentBlock.ToString(Block.Format.Short)}");
+            if (_logger.IsInfo)
+                _logger.Info($"Preparing new block on top of {parentBlock.ToString(Block.Format.Short)}");
 
             UInt256 timestamp = _timestamper.UnixTime.Seconds;
 
@@ -337,7 +356,7 @@ public async Task StopAsync()
             long number = header.Number;
             // Assemble the voting snapshot to check which votes make sense
             Snapshot snapshot = _snapshotManager.GetOrCreateSnapshot(number - 1, header.ParentHash);
-            bool isEpochBlock = (ulong) number % 30000 == 0;
+            bool isEpochBlock = (ulong)number % 30000 == 0;
             if (!isEpochBlock && _proposals.Any())
             {
                 // Gather all the proposals that make sense voting on
@@ -356,14 +375,19 @@ public async Task StopAsync()
                 if (addresses.Count > 0)
                 {
                     header.Beneficiary = addresses[_cryptoRandom.NextInt(addresses.Count)];
-                    header.Nonce = _proposals[header.Beneficiary] ? Clique.NonceAuthVote : Clique.NonceDropVote;
+                    if (_proposals.TryGetValue(header.Beneficiary!, out bool proposal))
+                    {
+                        header.Nonce = proposal ? Clique.NonceAuthVote : Clique.NonceDropVote;
+                    }
                 }
             }
 
             // Set the correct difficulty
+            header.BaseFee = BlockHeader.CalculateBaseFee(parentHeader, _specProvider.GetSpec(header.Number));
             header.Difficulty = CalculateDifficulty(snapshot, _sealer.Address);
             header.TotalDifficulty = parentBlock.TotalDifficulty + header.Difficulty;
-            if (_logger.IsDebug) _logger.Debug($"Setting total difficulty to {parentBlock.TotalDifficulty} + {header.Difficulty}.");
+            if (_logger.IsDebug)
+                _logger.Debug($"Setting total difficulty to {parentBlock.TotalDifficulty} + {header.Difficulty}.");
 
             // Set extra data
             int mainBytesLength = Clique.ExtraVanityLength + Clique.ExtraSealLength;
@@ -397,7 +421,7 @@ public async Task StopAsync()
 
             var selectedTxs = _txSource.GetTransactions(parentBlock.Header, header.GasLimit);
             Block block = new Block(header, selectedTxs, Array.Empty<BlockHeader>());
-            header.TxRoot = new TxTrie(block.Transactions, _spec.GetSpec(block.Number)).RootHash;
+            header.TxRoot = new TxTrie(block.Transactions, _specProvider.GetSpec(block.Number)).RootHash;
             block.Header.Author = _sealer.Address;
             return block;
         }