Merge branch 'master' into test/missingnodetest

.github/workflows/build-tools.yml

@@ -18,6 +18,7 @@ jobs:
         config: [release]
         project:
           - DocGen/DocGen.sln
+          - Evm/Evm.sln
           - HiveCompare/HiveCompare.sln
           - HiveConsensusWorkflowGenerator/HiveConsensusWorkflowGenerator.csproj
           - Nethermind.Tools.Kute/Nethermind.Tools.Kute.csproj

.github/workflows/dependency-review.yml

@@ -0,0 +1,18 @@
+name: Dependency review
+
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    name: Dependency review
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Dependency review
+        uses: actions/dependency-review-action@v4
+        with:
+          fail-on-severity: high

src/Nethermind/Nethermind.Api/NethermindApi.cs

@@ -77,12 +77,13 @@ public NethermindApi(IConfigProvider configProvider, IJsonSerializer jsonSeriali
         public IBlockchainBridge CreateBlockchainBridge()
         {
             ReadOnlyBlockTree readOnlyTree = BlockTree!.AsReadOnly();
+            OverridableWorldStateManager overridableWorldStateManager = new(DbProvider!, WorldStateManager!.TrieStore, LogManager);
 
             // TODO: reuse the same trie cache here
-            ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv = new(
-                WorldStateManager!,
+            OverridableTxProcessingEnv txProcessingEnv = new(
+                overridableWorldStateManager,
                 readOnlyTree,
-                SpecProvider,
+                SpecProvider!,
                 LogManager);
 
             SimulateReadOnlyBlocksProcessingEnvFactory simulateReadOnlyBlocksProcessingEnvFactory =
@@ -97,7 +98,7 @@ public IBlockchainBridge CreateBlockchainBridge()
             IBlocksConfig blocksConfig = ConfigProvider.GetConfig<IBlocksConfig>();
 
             return new BlockchainBridge(
-                readOnlyTxProcessingEnv,
+                txProcessingEnv,
                 simulateReadOnlyBlocksProcessingEnvFactory,
                 TxPool,
                 ReceiptFinder,

src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs

@@ -630,8 +630,10 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin
                 break;
             }
 
-            branchingPoint = _blockTree.FindParentHeader(toBeProcessed.Header,
-                BlockTreeLookupOptions.TotalDifficultyNotNeeded);
+            branchingPoint = options.ContainsFlag(ProcessingOptions.ForceSameBlock)
+                ? toBeProcessed.Header
+                : _blockTree.FindParentHeader(toBeProcessed.Header, BlockTreeLookupOptions.TotalDifficultyNotNeeded);
+
             if (branchingPoint is null)
             {
                 // genesis block

src/Nethermind/Nethermind.Consensus/Processing/OverridableTxProcessingEnv.cs

@@ -0,0 +1,64 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Collections.Generic;
+using Nethermind.Blockchain;
+using Nethermind.Core;
+using Nethermind.Core.Crypto;
+using Nethermind.Core.Specs;
+using Nethermind.Evm;
+using Nethermind.Evm.TransactionProcessing;
+using Nethermind.Logging;
+using Nethermind.State;
+
+namespace Nethermind.Consensus.Processing;
+
+public class OverridableTxProcessingEnv : ReadOnlyTxProcessingEnvBase, IOverridableTxProcessorSource
+{
+    private readonly Lazy<ITransactionProcessor> _transactionProcessorLazy;
+
+    protected new OverridableWorldState StateProvider { get; }
+    protected OverridableWorldStateManager WorldStateManager { get; }
+    protected OverridableCodeInfoRepository CodeInfoRepository { get; }
+    protected IVirtualMachine Machine { get; }
+    protected ITransactionProcessor TransactionProcessor => _transactionProcessorLazy.Value;
+
+    public OverridableTxProcessingEnv(
+        OverridableWorldStateManager worldStateManager,
+        IReadOnlyBlockTree readOnlyBlockTree,
+        ISpecProvider specProvider,
+        ILogManager? logManager,
+        IWorldState? worldStateToWarmUp = null
+    ) : base(worldStateManager, readOnlyBlockTree, specProvider, logManager, worldStateToWarmUp)
+    {
+        WorldStateManager = worldStateManager;
+        StateProvider = (OverridableWorldState)base.StateProvider;
+        CodeInfoRepository = new(new CodeInfoRepository((worldStateToWarmUp as IPreBlockCaches)?.Caches.PrecompileCache));
+        Machine = new VirtualMachine(BlockhashProvider, specProvider, CodeInfoRepository, logManager);
+        _transactionProcessorLazy = new(CreateTransactionProcessor);
+    }
+
+    protected virtual ITransactionProcessor CreateTransactionProcessor() =>
+        new TransactionProcessor(SpecProvider, StateProvider, Machine, CodeInfoRepository, LogManager);
+
+    IOverridableTxProcessingScope IOverridableTxProcessorSource.Build(Hash256 stateRoot) => Build(stateRoot);
+
+    public OverridableTxProcessingScope Build(Hash256 stateRoot)
+    {
+        Hash256 originalStateRoot = StateProvider.StateRoot;
+        StateProvider.StateRoot = stateRoot;
+        return new(CodeInfoRepository, TransactionProcessor, StateProvider, originalStateRoot);
+    }
+
+    IOverridableTxProcessingScope IOverridableTxProcessorSource.BuildAndOverride(BlockHeader header, Dictionary<Address, AccountOverride>? stateOverride)
+    {
+        OverridableTxProcessingScope scope = Build(header.StateRoot ?? throw new ArgumentException($"Block {header.Hash} state root is null", nameof(header)));
+        if (stateOverride != null)
+        {
+            scope.WorldState.ApplyStateOverrides(scope.CodeInfoRepository, stateOverride, SpecProvider.GetSpec(header), header.Number);
+            header.StateRoot = scope.WorldState.StateRoot;
+        }
+        return scope;
+    }
+}

src/Nethermind/Nethermind.Consensus/Processing/OverridableTxProcessingScope.cs

@@ -0,0 +1,29 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using Nethermind.Core.Crypto;
+using Nethermind.Evm;
+using Nethermind.Evm.TransactionProcessing;
+using Nethermind.State;
+
+namespace Nethermind.Consensus.Processing;
+
+public class OverridableTxProcessingScope(
+    IOverridableCodeInfoRepository codeInfoRepository,
+    ITransactionProcessor transactionProcessor,
+    OverridableWorldState worldState,
+    Hash256 originalStateRoot
+) : IOverridableTxProcessingScope
+{
+    public IOverridableCodeInfoRepository CodeInfoRepository => codeInfoRepository;
+    public ITransactionProcessor TransactionProcessor => transactionProcessor;
+    public IWorldState WorldState => worldState;
+
+    public void Dispose()
+    {
+        worldState.StateRoot = originalStateRoot;
+        worldState.Reset();
+        worldState.ResetOverrides();
+        codeInfoRepository.ResetOverrides();
+    }
+}

src/Nethermind/Nethermind.Consensus/Processing/ProcessingOptions.cs

@@ -51,7 +51,10 @@ public enum ProcessingOptions
         /// </summary>
         MarkAsProcessed = 128,
 
-        All = 255,
+        /// <summary>
+        /// Forces to run on top of the specified block state, instead of reverting to the previous one.
+        /// </summary>
+        ForceSameBlock = 1 << 8,
 
         /// <summary>
         /// Combination of switches for block producers when they preprocess block for state root calculation.
@@ -63,6 +66,12 @@ public enum ProcessingOptions
         /// </summary>
         Trace = ForceProcessing | ReadOnlyChain | DoNotVerifyNonce | NoValidation,
 
+        /// <summary>
+        /// EVM tracing needs to process one or more transactions on top of the specified block (instead of the previous one)
+        /// without storing the data on chain.
+        /// </summary>
+        TraceTransactions = Trace | ForceSameBlock,
+
         /// <summary>
         /// Processing options for engine_NewPayload
         /// </summary>

src/Nethermind/Nethermind.Consensus/Tracing/GethStyleTracer.cs

@@ -16,6 +16,7 @@
 using Nethermind.Core.Extensions;
 using Nethermind.Core.Specs;
 using Nethermind.Crypto;
+using Nethermind.Evm;
 using Nethermind.Evm.Tracing;
 using Nethermind.Evm.Tracing.GethStyle;
 using Nethermind.Evm.Tracing.GethStyle.Custom.JavaScript;
@@ -36,6 +37,7 @@ public class GethStyleTracer : IGethStyleTracer
     private readonly IWorldState _worldState;
     private readonly IReceiptStorage _receiptStorage;
     private readonly IFileSystem _fileSystem;
+    private readonly IOverridableTxProcessorSource _env;
 
     public GethStyleTracer(IBlockchainProcessor processor,
         IWorldState worldState,
@@ -44,7 +46,8 @@ public GethStyleTracer(IBlockchainProcessor processor,
         IBadBlockStore badBlockStore,
         ISpecProvider specProvider,
         ChangeableTransactionProcessorAdapter transactionProcessorAdapter,
-        IFileSystem fileSystem)
+        IFileSystem fileSystem,
+        IOverridableTxProcessorSource env)
     {
         _processor = processor ?? throw new ArgumentNullException(nameof(processor));
         _worldState = worldState;
@@ -54,6 +57,7 @@ public GethStyleTracer(IBlockchainProcessor processor,
         _specProvider = specProvider;
         _transactionProcessorAdapter = transactionProcessorAdapter;
         _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
+        _env = env ?? throw new ArgumentNullException(nameof(env));
     }
 
     public GethLikeTxTrace Trace(Hash256 blockHash, int txIndex, GethTraceOptions options, CancellationToken cancellationToken)
@@ -82,7 +86,10 @@ public GethLikeTxTrace Trace(Hash256 blockHash, int txIndex, GethTraceOptions op
 
         try
         {
-            return Trace(block, tx.Hash, cancellationToken, options);
+            Dictionary<Address, AccountOverride>? stateOverride = options.StateOverrides;
+            using IOverridableTxProcessingScope? scope = stateOverride != null ? _env.BuildAndOverride(block.Header, stateOverride) : null;
+
+            return Trace(block, tx.Hash, cancellationToken, options, ProcessingOptions.TraceTransactions);
         }
         finally
         {
@@ -189,15 +196,16 @@ public IEnumerable<string> TraceBadBlockToFile(Hash256 blockHash, GethTraceOptio
         return tracer.FileNames;
     }
 
-    private GethLikeTxTrace? Trace(Block block, Hash256? txHash, CancellationToken cancellationToken, GethTraceOptions options)
+    private GethLikeTxTrace? Trace(Block block, Hash256? txHash, CancellationToken cancellationToken, GethTraceOptions options,
+        ProcessingOptions processingOptions = ProcessingOptions.Trace)
     {
         ArgumentNullException.ThrowIfNull(txHash);
 
         IBlockTracer<GethLikeTxTrace> tracer = CreateOptionsTracer(block.Header, options with { TxHash = txHash });
 
         try
         {
-            _processor.Process(block, ProcessingOptions.Trace, tracer.WithCancellation(cancellationToken));
+            _processor.Process(block, processingOptions, tracer.WithCancellation(cancellationToken));
             return tracer.BuildResult().SingleOrDefault();
         }
         catch

src/Nethermind/Nethermind.Consensus/Tracing/Tracer.cs

@@ -16,18 +16,20 @@ public class Tracer : ITracer
         private readonly IWorldState _stateProvider;
         private readonly IBlockchainProcessor _traceProcessor;
         private readonly IBlockchainProcessor _executeProcessor;
-        private readonly ProcessingOptions _processingOptions;
+        private readonly ProcessingOptions _executeOptions;
+        private readonly ProcessingOptions _traceOptions;
 
         public Tracer(IWorldState stateProvider, IBlockchainProcessor traceProcessor, IBlockchainProcessor executeProcessor,
-            ProcessingOptions processingOptions = ProcessingOptions.Trace)
+            ProcessingOptions executeOptions = ProcessingOptions.Trace, ProcessingOptions traceOptions = ProcessingOptions.Trace)
         {
             _traceProcessor = traceProcessor;
             _executeProcessor = executeProcessor;
             _stateProvider = stateProvider;
-            _processingOptions = processingOptions;
+            _executeOptions = executeOptions;
+            _traceOptions = traceOptions;
         }
 
-        private void Process(Block block, IBlockTracer blockTracer, IBlockchainProcessor processor)
+        private void Process(Block block, IBlockTracer blockTracer, IBlockchainProcessor processor, ProcessingOptions options)
         {
             /* We force process since we want to process a block that has already been processed in the past and normally it would be ignored.
                We also want to make it read only so the state is not modified persistently in any way. */
@@ -36,7 +38,7 @@ We also want to make it read only so the state is not modified persistently in a
 
             try
             {
-                processor.Process(block, _processingOptions, blockTracer);
+                processor.Process(block, options, blockTracer);
             }
             catch (Exception)
             {
@@ -47,9 +49,9 @@ We also want to make it read only so the state is not modified persistently in a
             blockTracer.EndBlockTrace();
         }
 
-        public void Trace(Block block, IBlockTracer tracer) => Process(block, tracer, _traceProcessor);
+        public void Trace(Block block, IBlockTracer tracer) => Process(block, tracer, _traceProcessor, _traceOptions);
 
-        public void Execute(Block block, IBlockTracer tracer) => Process(block, tracer, _executeProcessor);
+        public void Execute(Block block, IBlockTracer tracer) => Process(block, tracer, _executeProcessor, _executeOptions);
 
         public void Accept(ITreeVisitor visitor, Hash256 stateRoot)
         {

src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs

@@ -77,6 +77,7 @@ public IBlockFinder BlockFinder
     public IWorldState State { get; set; } = null!;
     public IReadOnlyStateProvider ReadOnlyState { get; private set; } = null!;
     public IDb StateDb => DbProvider.StateDb;
+    public IDb BlocksDb => DbProvider.BlocksDb;
     public TrieStore TrieStore { get; set; } = null!;
     public IBlockProducer BlockProducer { get; private set; } = null!;
     public IBlockProducerRunner BlockProducerRunner { get; protected set; } = null!;

src/Nethermind/Nethermind.Crypto/BlsSigner.cs

@@ -40,6 +40,7 @@ public static Signature Sign(Span<long> buf, Bls.SecretKey sk, ReadOnlySpan<byte
     public static Signature Sign(Bls.SecretKey sk, ReadOnlySpan<byte> message)
         => Sign(new long[G2.Sz], sk, message);
 
+    [SkipLocalsInit]
     public static bool Verify(G1Affine publicKey, Signature signature, ReadOnlySpan<byte> message)
     {
         int len = 2 * GT.Sz;
@@ -126,6 +127,9 @@ public AggregatedPublicKey(Span<long> buf)
         public void FromSk(Bls.SecretKey sk)
             => _point.FromSk(sk);
 
+        public void Reset()
+            => _point.Zero();
+
         public bool TryDecode(ReadOnlySpan<byte> publicKeyBytes, out Bls.ERROR err)
             => _point.TryDecode(publicKeyBytes, out err);
 
@@ -138,6 +142,7 @@ public void Aggregate(G1Affine publicKey)
         public void Aggregate(AggregatedPublicKey aggregatedPublicKey)
             => _point.Aggregate(aggregatedPublicKey.PublicKey);
 
+        [SkipLocalsInit]
         public bool TryAggregate(ReadOnlySpan<byte> publicKeyBytes, out Bls.ERROR err)
         {
             G1Affine pk = new(stackalloc long[G1Affine.Sz]);

src/Nethermind/Nethermind.Facade/Proxy/Models/AccountOverride.cs → src/Nethermind/Nethermind.Evm/AccountOverride.cs

@@ -6,7 +6,7 @@
 using Nethermind.Core.Crypto;
 using Nethermind.Int256;
 
-namespace Nethermind.Facade.Proxy.Models;
+namespace Nethermind.Evm;
 
 public class AccountOverride
 {

src/Nethermind/Nethermind.Evm/IOverridableCodeInfoRepository.cs

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using Nethermind.Core;
+using Nethermind.Core.Specs;
+using Nethermind.Evm.CodeAnalysis;
+using Nethermind.State;
+
+namespace Nethermind.Evm;
+
+public interface IOverridableCodeInfoRepository : ICodeInfoRepository
+{
+    void SetCodeOverwrite(IWorldState worldState, IReleaseSpec vmSpec, Address key, CodeInfo value, Address? redirectAddress = null);
+    public void ResetOverrides();
+}

src/Nethermind/Nethermind.Facade/OverridableCodeInfoRepository.cs → src/Nethermind/Nethermind.Evm/OverridableCodeInfoRepository.cs

@@ -7,13 +7,12 @@
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Specs;
-using Nethermind.Evm;
 using Nethermind.Evm.CodeAnalysis;
 using Nethermind.State;
 
-namespace Nethermind.Facade;
+namespace Nethermind.Evm;
 
-public class OverridableCodeInfoRepository(ICodeInfoRepository codeInfoRepository) : ICodeInfoRepository
+public class OverridableCodeInfoRepository(ICodeInfoRepository codeInfoRepository) : IOverridableCodeInfoRepository
 {
     private readonly Dictionary<Address, CodeInfo> _codeOverwrites = new();
 
@@ -51,4 +50,6 @@ public bool TryGetDelegation(IReadOnlyStateProvider worldState, Address address,
 
     public ValueHash256 GetExecutableCodeHash(IWorldState worldState, Address address) =>
         codeInfoRepository.GetExecutableCodeHash(worldState, address);
+
+    public void ResetOverrides() => _codeOverwrites.Clear();
 }