Refactoring (NethermindEth#6635)

* Resolved conflicts * max body size for http json rpc (NethermindEth#36) * Format * Self-recovery of TxPool max capacity * fix ProcessedTransactionsDbCleanerTests * Update txPool worst value if removal unsuccessful (NethermindEth#6702) (cherry picked from commit f262cb2) * Clearer error message for missing starting block state (NethermindEth#6672) * Update txPool worst value if removal unsuccessful (NethermindEth#6702) * Add Cancun hard-fork settings for Gnosis (NethermindEth#6709) * Cancun spec and config for mainnet (NethermindEth#6679) * Add Paris release spec class for completeness (NethermindEth#6633) * 0.0.0.0 to + --------- Co-authored-by: smartprogrammer <smartprogrammer@windowslive.com> Co-authored-by: Marcin Sobczak <marcindsobczak@gmail.com> Co-authored-by: Ben Adams <thundercat@illyriad.co.uk> Co-authored-by: Ruben Buniatyan <rubo@users.noreply.github.com> Co-authored-by: Ahmad Bitar <33181301+smartprogrammer93@users.noreply.github.com> Co-authored-by: Kamil Chodoła <kamil@nethermind.io> Co-authored-by: Lukasz Rozmej <lukasz.rozmej@gmail.com>
fuseio · Jun 25, 2024 · f983610 · f983610
1 parent 59eb1a5
commit f983610
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 174 deletions.
diff --git a/.github/workflows/hive-consensus-tests.yml b/.github/workflows/hive-consensus-tests.yml
@@ -23,6 +23,9 @@ jobs:
     outputs:
       cleanRef: ${{ steps.prepare_ref.outputs.cleanRef }}
     runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    needs: [create_docker_image]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -5901,7 +5904,7 @@ jobs:
         run: |
           echo "PARALLELISM=${{ github.event.inputs.parallelism || '3' }}" >> $GITHUB_ENV
       - name: Check out Nethermind repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: nethermind
       - name: Set up QEMU
@@ -6074,150 +6077,45 @@ jobs:
         run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcEIP1559 --sim.parallelism $PARALLELISM
       - name: Print results
         run: |
-          chmod +x nethermind/scripts/hive-results.sh
-          nethermind/scripts/hive-results.sh "hive/workspace/logs/*.json"
-  test_65:
-    name: 65. Combined tests (e.g. bcEIP3675)
+          cd nethermind/tools/HiveConsensusWorkflowGenerator
+          dotnet run
+          cat matrix.json
+          echo "matrix=$(jq -c . matrix.json)" >> $GITHUB_OUTPUT
+
+  run_hive_tests:
     runs-on: ubuntu-latest
+    needs: [generate_hive_consensus_tests, create_docker_image]
+    strategy:
+      fail-fast: false
+      matrix:
+        hiveTests: ${{fromJson(needs.generate_hive_consensus_tests.outputs.matrix)}}
     steps:
       - name: Set up parameters
         run: |
-          echo "PARALLELISM=${{ github.event.inputs.parallelism || '3' }}" >> $GITHUB_ENV
+          echo "PARALLELISM=${{ github.event.inputs.parallelism || '8' }}" >> $GITHUB_ENV
+
       - name: Check out Nethermind repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: nethermind
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Build Docker image
-        uses: docker/build-push-action@v3
-        with:
-          context: nethermind
-          file: nethermind/Dockerfile
-          tags: nethermind:test-${{ github.sha }}
-          outputs: type=docker,dest=/tmp/image.tar
+
       - name: Install Linux packages
         run: |
           sudo apt-get update
           sudo apt-get install libsnappy-dev libc6-dev libc6 build-essential
+
       - name: Set up Go environment
         uses: actions/setup-go@v3.0.0
         with:
           go-version: ">=1.17.0"
       - name: Check out Hive repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: ethereum/hive
           ref: master
           path: hive
+
       - name: Patch Hive Dockerfile
-        run: sed -i 's#FROM $baseimage:$tag#FROM nethermind:test-${{ github.sha }}#g' hive/clients/nethermind/Dockerfile
-      - name: Build Hive
-        working-directory: hive
-        run: go build .
-      - name: Load Docker image
-        run: docker load --input /tmp/image.tar
-      - name: Run bcEIP3675
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcEIP3675 --sim.parallelism $PARALLELISM
-      - name: Run bcForgedTest
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcForgedTest --sim.parallelism $PARALLELISM
-      - name: Run bcInvalidHeaderTest
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcInvalidHeaderTest --sim.parallelism $PARALLELISM
-      - name: Run bcMultiChainTest
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcMultiChainTest --sim.parallelism $PARALLELISM
-      - name: Run bcUncleHeaderValidity
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcUncleHeaderValidity --sim.parallelism $PARALLELISM
-      - name: Run bcUncleSpecialTests
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcUncleSpecialTests --sim.parallelism $PARALLELISM
-      - name: Run bcUncleTest
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcUncleTest --sim.parallelism $PARALLELISM
-      - name: Run bcArrowGlacierToMerge
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcArrowGlacierToMerge --sim.parallelism $PARALLELISM
-      - name: Run bcBerlinToLondon
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcBerlinToLondon --sim.parallelism $PARALLELISM
-      - name: Run bcByzantiumToConstantinopleFix
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcByzantiumToConstantinopleFix --sim.parallelism $PARALLELISM
-      - name: Run bcEIP158ToByzantium
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcEIP158ToByzantium --sim.parallelism $PARALLELISM
-      - name: Run bcFrontierToHomestead
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcFrontierToHomestead --sim.parallelism $PARALLELISM
-      - name: Run bcHomesteadToDao
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcHomesteadToDao --sim.parallelism $PARALLELISM
-      - name: Run bcHomesteadToEIP150
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcHomesteadToEIP150 --sim.parallelism $PARALLELISM
-      - name: Run bcMergeToShanghai
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcMergeToShanghai --sim.parallelism $PARALLELISM
-      - name: Run bcBlockGasLimitTest
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcBlockGasLimitTest --sim.parallelism $PARALLELISM
-      - name: Run bcEIP1559
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcEIP1559 --sim.parallelism $PARALLELISM
-      - name: Run bcEIP3675
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcEIP3675 --sim.parallelism $PARALLELISM
-      - name: Print results
-        run: |
-          chmod +x nethermind/scripts/hive-results.sh
-          nethermind/scripts/hive-results.sh "hive/workspace/logs/*.json"
-  test_66:
-    name: 66. bcExploitTest
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up parameters
-        run: |
-          echo "PARALLELISM=${{ github.event.inputs.parallelism || '3' }}" >> $GITHUB_ENV
-      - name: Check out Nethermind repository
-        uses: actions/checkout@v3
-        with:
-          path: nethermind
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Build Docker image
-        uses: docker/build-push-action@v3
-        with:
-          context: nethermind
-          file: nethermind/Dockerfile
-          tags: nethermind:test-${{ github.sha }}
-          outputs: type=docker,dest=/tmp/image.tar
-      - name: Install Linux packages
         run: |
           sudo apt-get update
           sudo apt-get install libsnappy-dev libc6-dev libc6 build-essential
@@ -6236,39 +6134,10 @@ jobs:
       - name: Build Hive
         working-directory: hive
         run: go build .
-      - name: Load Docker image
-        run: docker load --input /tmp/image.tar
-      - name: Run bcExploitTest
+
+      - name: Run hive for tests - ${{ join(matrix.hiveTests.testNames, ', ') }}
         continue-on-error: true
         working-directory: hive
-        run: ./hive --client nethermind --sim ethereum/consensus --sim.limit /bcExploitTest --sim.parallelism $PARALLELISM
-      - name: Print results
-        run: |
-          chmod +x nethermind/scripts/hive-results.sh
-          nethermind/scripts/hive-results.sh "hive/workspace/logs/*.json"
-  test_67:
-    name: 67. Combined tests (e.g. bcExample)
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up parameters
-        run: |
-          echo "PARALLELISM=${{ github.event.inputs.parallelism || '3' }}" >> $GITHUB_ENV
-      - name: Check out Nethermind repository
-        uses: actions/checkout@v3
-        with:
-          path: nethermind
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Build Docker image
-        uses: docker/build-push-action@v3
-        with:
-          context: nethermind
-          file: nethermind/Dockerfile
-          tags: nethermind:test-${{ github.sha }}
-          outputs: type=docker,dest=/tmp/image.tar
-      - name: Install Linux packages
         run: |
           sudo apt-get update
           sudo apt-get install libsnappy-dev libc6-dev libc6 build-essential

diff --git a/src/Nethermind/Nethermind.Consensus/Processing/TransactionProcessorAdapterExtensions.cs b/src/Nethermind/Nethermind.Consensus/Processing/TransactionProcessorAdapterExtensions.cs
@@ -11,21 +11,21 @@ namespace Nethermind.Consensus.Processing;
 
 internal static class TransactionProcessorAdapterExtensions
 {
-    public static TransactionResult ProcessTransaction(this ITransactionProcessorAdapter transactionProcessor,
-        in BlockExecutionContext blkCtx,
-        Transaction currentTx,
-        BlockReceiptsTracer receiptsTracer,
-        ProcessingOptions processingOptions,
-        IWorldState stateProvider)
-    {
-        if (processingOptions.ContainsFlag(ProcessingOptions.DoNotVerifyNonce))
+        public static TransactionResult ProcessTransaction(this ITransactionProcessorAdapter transactionProcessor,
+            in BlockExecutionContext blkCtx,
+            Transaction currentTx,
+            BlockReceiptsTracer receiptsTracer,
+            ProcessingOptions processingOptions,
+            IWorldState stateProvider)
         {
-            currentTx.Nonce = stateProvider.GetNonce(currentTx.SenderAddress!);
-        }
+                if (processingOptions.ContainsFlag(ProcessingOptions.DoNotVerifyNonce))
+                {
+                        currentTx.Nonce = stateProvider.GetNonce(currentTx.SenderAddress!);
+                }
 
-        using ITxTracer tracer = receiptsTracer.StartNewTxTrace(currentTx);
-        TransactionResult result = transactionProcessor.Execute(currentTx, in blkCtx, receiptsTracer);
-        receiptsTracer.EndTxTrace();
-        return result;
-    }
+                using ITxTracer tracer = receiptsTracer.StartNewTxTrace(currentTx);
+                TransactionResult result = transactionProcessor.Execute(currentTx, in blkCtx, receiptsTracer);
+                receiptsTracer.EndTxTrace();
+                return result;
+        }
 }
diff --git a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs
@@ -148,6 +148,13 @@ private static string ToHexStringWithEip55Checksum(ReadOnlySpan<byte> bytes, boo
             return result;
         }
 
+        public static ReadOnlySpan<byte> TakeAndMove(this ref ReadOnlySpan<byte> span, int length)
+        {
+            ReadOnlySpan<byte> s = span[..length];
+            span = span[length..];
+            return s;
+        }
+
         public static bool IsNullOrEmpty<T>(this in Span<T> span) => span.Length == 0;
         public static bool IsNull<T>(this in Span<T> span) => Unsafe.IsNullRef(ref MemoryMarshal.GetReference(span));
         public static bool IsNullOrEmpty<T>(this in ReadOnlySpan<T> span) => span.Length == 0;

diff --git a/src/Nethermind/Nethermind.Evm.Test/TransactionSubstateTests.cs b/src/Nethermind/Nethermind.Evm.Test/TransactionSubstateTests.cs
@@ -16,12 +16,15 @@ public class TransactionSubstateTests
         [Test]
         public void should_return_proper_revert_error_when_there_is_no_exception()
         {
-            byte[] data = {0, 0, 0, 0,
+            byte[] data =
+            {
+                0, 0, 0, 0,
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x20,
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5,
-                0x05, 0x06, 0x07, 0x08, 0x09};
+                0x05, 0x06, 0x07, 0x08, 0x09
+            };
             ReadOnlyMemory<byte> readOnlyMemory = new(data);
             TransactionSubstate transactionSubstate = new(readOnlyMemory,
                 0,
@@ -158,12 +161,14 @@ public void should_return_proper_revert_error_when_using_special_functions((byte
         [Ignore("Badly implemented")]
         public void should_return_proper_revert_error_when_revert_custom_error()
         {
-            byte[] data = {
+            byte[] data =
+            {
                 0x22, 0x02, 0x66, 0xb6, // Keccak of `FailedOp(uint256,string)` == 0x220266b6
                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
                 0x17,
                 0x41, 0x41, 0x32, 0x31, 0x20, 0x64, 0x69, 0x64, 0x6e, 0x27, 0x74, 0x20, 0x70, 0x61, 0x79, 0x20, 0x70, 0x72, 0x65, 0x66, 0x75, 0x6e, 0x64, // "AA21 didn't pay prefund"
-                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+            };
             ReadOnlyMemory<byte> readOnlyMemory = new(data);
             TransactionSubstate transactionSubstate = new(
                 readOnlyMemory,

diff --git a/src/Nethermind/Nethermind.Evm/TransactionSubstate.cs b/src/Nethermind/Nethermind.Evm/TransactionSubstate.cs
@@ -7,6 +7,7 @@
 using System.Text;
 using System.Text.Unicode;
 using Nethermind.Core;
+using Nethermind.Core.Crypto;
 using Nethermind.Core.Extensions;
 using Nethermind.Int256;
 using Nethermind.Logging;
@@ -23,6 +24,7 @@ public class TransactionSubstate
     private const string Revert = "revert";
 
     private const int RevertPrefix = 4;
+    private const int WordSize = EvmPooledMemory.WordSize;
 
     public const string RevertedErrorMessagePrefix = "Reverted ";
     public static readonly byte[] ErrorFunctionSelector = Keccak.Compute("Error(string)").BytesToArray()[..RevertPrefix];