From 73bfa1e1e2a9b7ebe9a7df61730eb873fe3f5279 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Fri, 26 Apr 2024 12:22:44 +0100 Subject: [PATCH 1/4] Vectorize DataCost --- .../Nethermind.Evm/IntrinsicGasCalculator.cs | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs index f3e54899438..18586134075 100644 --- a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs +++ b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs @@ -2,12 +2,15 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; using System.IO; +using System.Numerics; using Nethermind.Core; using Nethermind.Core.Eip2930; using Nethermind.Core.Specs; using Nethermind.Int256; +using System.Runtime.Intrinsics; +using System.Runtime.InteropServices; +using System.Runtime.CompilerServices; namespace Nethermind.Evm; @@ -37,16 +40,44 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec) { long txDataNonZeroGasCost = releaseSpec.IsEip2028Enabled ? GasCostOf.TxDataNonZeroEip2028 : GasCostOf.TxDataNonZero; - long dataCost = 0; Span data = transaction.Data.GetValueOrDefault().Span; - for (int i = 0; i < data.Length; i++) + + var dataCost = transaction.IsContractCreation && releaseSpec.IsEip3860Enabled + ? EvmPooledMemory.Div32Ceiling((UInt256)data.Length) * GasCostOf.InitCodeWord + : 0; + + if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) { - dataCost += data[i] == 0 ? GasCostOf.TxDataZero : txDataNonZeroGasCost; + ref byte bytes = ref MemoryMarshal.GetReference(data); + int i = 0; + for (; i < data.Length - Vector256.Count; i += Vector256.Count) + { + Vector256 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref bytes, i)); + uint flags = Vector256.Equals(dataVector, default).ExtractMostSignificantBits(); + var zeros = BitOperations.PopCount(flags); + dataCost += zeros * GasCostOf.TxDataZero + (Vector256.Count - zeros) * txDataNonZeroGasCost; + } + + data = data[i..]; } + if (Vector128.IsHardwareAccelerated && data.Length >= Vector128.Count) + { + ref byte bytes = ref MemoryMarshal.GetReference(data); + int i = 0; + for (; i < data.Length - Vector128.Count; i += Vector128.Count) + { + Vector128 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref MemoryMarshal.GetReference(data), i)); + uint flags = Vector128.Equals(dataVector, default).ExtractMostSignificantBits(); + var zeros = BitOperations.PopCount(flags); + dataCost += zeros * GasCostOf.TxDataZero + (Vector128.Count - zeros) * txDataNonZeroGasCost; + } - if (transaction.IsContractCreation && releaseSpec.IsEip3860Enabled) + data = data[i..]; + } + + for (int i = 0; i < data.Length; i++) { - dataCost += EvmPooledMemory.Div32Ceiling((UInt256)data.Length) * GasCostOf.InitCodeWord; + dataCost += data[i] == 0 ? GasCostOf.TxDataZero : txDataNonZeroGasCost; } return dataCost; From 07aa9e8c713704a919e6b67f8575d48a320e2e09 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Fri, 26 Apr 2024 12:37:20 +0100 Subject: [PATCH 2/4] Add 512 --- .../Nethermind.Evm/IntrinsicGasCalculator.cs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs index 18586134075..080b3205c0d 100644 --- a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs +++ b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs @@ -46,6 +46,20 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec) ? EvmPooledMemory.Div32Ceiling((UInt256)data.Length) * GasCostOf.InitCodeWord : 0; + if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) + { + ref byte bytes = ref MemoryMarshal.GetReference(data); + int i = 0; + for (; i < data.Length - Vector512.Count; i += Vector512.Count) + { + Vector512 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref bytes, i)); + ulong flags = Vector512.Equals(dataVector, default).ExtractMostSignificantBits(); + int zeros = BitOperations.PopCount(flags); + dataCost += zeros * GasCostOf.TxDataZero + (Vector512.Count - zeros) * txDataNonZeroGasCost; + } + + data = data[i..]; + } if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) { ref byte bytes = ref MemoryMarshal.GetReference(data); @@ -54,7 +68,7 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec) { Vector256 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref bytes, i)); uint flags = Vector256.Equals(dataVector, default).ExtractMostSignificantBits(); - var zeros = BitOperations.PopCount(flags); + int zeros = BitOperations.PopCount(flags); dataCost += zeros * GasCostOf.TxDataZero + (Vector256.Count - zeros) * txDataNonZeroGasCost; } @@ -68,7 +82,7 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec) { Vector128 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref MemoryMarshal.GetReference(data), i)); uint flags = Vector128.Equals(dataVector, default).ExtractMostSignificantBits(); - var zeros = BitOperations.PopCount(flags); + int zeros = BitOperations.PopCount(flags); dataCost += zeros * GasCostOf.TxDataZero + (Vector128.Count - zeros) * txDataNonZeroGasCost; } From 1bdf25051589dd855aa6f9126ca898217297f1d5 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Fri, 26 Apr 2024 12:45:09 +0100 Subject: [PATCH 3/4] Do zero calc at end --- .../Nethermind.Evm/IntrinsicGasCalculator.cs | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs index 080b3205c0d..659a28aafde 100644 --- a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs +++ b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs @@ -41,11 +41,8 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec) long txDataNonZeroGasCost = releaseSpec.IsEip2028Enabled ? GasCostOf.TxDataNonZeroEip2028 : GasCostOf.TxDataNonZero; Span data = transaction.Data.GetValueOrDefault().Span; - - var dataCost = transaction.IsContractCreation && releaseSpec.IsEip3860Enabled - ? EvmPooledMemory.Div32Ceiling((UInt256)data.Length) * GasCostOf.InitCodeWord - : 0; - + int dataLength = data.Length; + int totalZeros = 0; if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) { ref byte bytes = ref MemoryMarshal.GetReference(data); @@ -54,8 +51,7 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec) { Vector512 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref bytes, i)); ulong flags = Vector512.Equals(dataVector, default).ExtractMostSignificantBits(); - int zeros = BitOperations.PopCount(flags); - dataCost += zeros * GasCostOf.TxDataZero + (Vector512.Count - zeros) * txDataNonZeroGasCost; + totalZeros += BitOperations.PopCount(flags); } data = data[i..]; @@ -68,8 +64,7 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec) { Vector256 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref bytes, i)); uint flags = Vector256.Equals(dataVector, default).ExtractMostSignificantBits(); - int zeros = BitOperations.PopCount(flags); - dataCost += zeros * GasCostOf.TxDataZero + (Vector256.Count - zeros) * txDataNonZeroGasCost; + totalZeros += BitOperations.PopCount(flags); } data = data[i..]; @@ -82,8 +77,7 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec) { Vector128 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref MemoryMarshal.GetReference(data), i)); uint flags = Vector128.Equals(dataVector, default).ExtractMostSignificantBits(); - int zeros = BitOperations.PopCount(flags); - dataCost += zeros * GasCostOf.TxDataZero + (Vector128.Count - zeros) * txDataNonZeroGasCost; + totalZeros += BitOperations.PopCount(flags); } data = data[i..]; @@ -91,10 +85,19 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec) for (int i = 0; i < data.Length; i++) { - dataCost += data[i] == 0 ? GasCostOf.TxDataZero : txDataNonZeroGasCost; + if (data[i] == 0) + { + totalZeros++; + } } - return dataCost; + var baseDataCost = (transaction.IsContractCreation && releaseSpec.IsEip3860Enabled + ? EvmPooledMemory.Div32Ceiling((UInt256)dataLength) * GasCostOf.InitCodeWord + : 0); + + return baseDataCost + + totalZeros * GasCostOf.TxDataZero + + (dataLength - totalZeros) * txDataNonZeroGasCost; } private static long AccessListCost(Transaction transaction, IReleaseSpec releaseSpec) From 965d168d8a6b7ff4d464bb97f51abd4e7a663f52 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Fri, 26 Apr 2024 12:59:43 +0100 Subject: [PATCH 4/4] Add CountZeros extension --- .../Nethermind.Core/Extensions/Bytes.cs | 57 +++++++++++++++++++ .../Nethermind.Evm/IntrinsicGasCalculator.cs | 54 ++---------------- 2 files changed, 61 insertions(+), 50 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs index 85b1bd8c7cb..72a0267ce9c 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs @@ -969,6 +969,63 @@ public static int CountLeadingZeros(this ReadOnlySpan bytes) return leadingZeros; } + public static int CountZeros(this Span data) + => CountZeros((ReadOnlySpan)data); + + public static int CountZeros(this ReadOnlySpan data) + { + int totalZeros = 0; + if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) + { + ref byte bytes = ref MemoryMarshal.GetReference(data); + int i = 0; + for (; i < data.Length - Vector512.Count; i += Vector512.Count) + { + Vector512 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref bytes, i)); + ulong flags = Vector512.Equals(dataVector, default).ExtractMostSignificantBits(); + totalZeros += BitOperations.PopCount(flags); + } + + data = data[i..]; + } + if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) + { + ref byte bytes = ref MemoryMarshal.GetReference(data); + int i = 0; + for (; i < data.Length - Vector256.Count; i += Vector256.Count) + { + Vector256 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref bytes, i)); + uint flags = Vector256.Equals(dataVector, default).ExtractMostSignificantBits(); + totalZeros += BitOperations.PopCount(flags); + } + + data = data[i..]; + } + if (Vector128.IsHardwareAccelerated && data.Length >= Vector128.Count) + { + ref byte bytes = ref MemoryMarshal.GetReference(data); + int i = 0; + for (; i < data.Length - Vector128.Count; i += Vector128.Count) + { + Vector128 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref MemoryMarshal.GetReference(data), i)); + uint flags = Vector128.Equals(dataVector, default).ExtractMostSignificantBits(); + totalZeros += BitOperations.PopCount(flags); + } + + data = data[i..]; + } + + for (int i = 0; i < data.Length; i++) + { + if (data[i] == 0) + { + totalZeros++; + } + } + + return totalZeros; + } + [DebuggerStepThrough] public static byte[] FromUtf8HexString(scoped ReadOnlySpan hexString) { diff --git a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs index 659a28aafde..7742bcfc5ba 100644 --- a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs +++ b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs @@ -11,6 +11,7 @@ using System.Runtime.Intrinsics; using System.Runtime.InteropServices; using System.Runtime.CompilerServices; +using Nethermind.Core.Extensions; namespace Nethermind.Evm; @@ -41,63 +42,16 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec) long txDataNonZeroGasCost = releaseSpec.IsEip2028Enabled ? GasCostOf.TxDataNonZeroEip2028 : GasCostOf.TxDataNonZero; Span data = transaction.Data.GetValueOrDefault().Span; - int dataLength = data.Length; - int totalZeros = 0; - if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) - { - ref byte bytes = ref MemoryMarshal.GetReference(data); - int i = 0; - for (; i < data.Length - Vector512.Count; i += Vector512.Count) - { - Vector512 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref bytes, i)); - ulong flags = Vector512.Equals(dataVector, default).ExtractMostSignificantBits(); - totalZeros += BitOperations.PopCount(flags); - } - - data = data[i..]; - } - if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) - { - ref byte bytes = ref MemoryMarshal.GetReference(data); - int i = 0; - for (; i < data.Length - Vector256.Count; i += Vector256.Count) - { - Vector256 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref bytes, i)); - uint flags = Vector256.Equals(dataVector, default).ExtractMostSignificantBits(); - totalZeros += BitOperations.PopCount(flags); - } - data = data[i..]; - } - if (Vector128.IsHardwareAccelerated && data.Length >= Vector128.Count) - { - ref byte bytes = ref MemoryMarshal.GetReference(data); - int i = 0; - for (; i < data.Length - Vector128.Count; i += Vector128.Count) - { - Vector128 dataVector = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref MemoryMarshal.GetReference(data), i)); - uint flags = Vector128.Equals(dataVector, default).ExtractMostSignificantBits(); - totalZeros += BitOperations.PopCount(flags); - } - - data = data[i..]; - } - - for (int i = 0; i < data.Length; i++) - { - if (data[i] == 0) - { - totalZeros++; - } - } + int totalZeros = data.CountZeros(); var baseDataCost = (transaction.IsContractCreation && releaseSpec.IsEip3860Enabled - ? EvmPooledMemory.Div32Ceiling((UInt256)dataLength) * GasCostOf.InitCodeWord + ? EvmPooledMemory.Div32Ceiling((UInt256)data.Length) * GasCostOf.InitCodeWord : 0); return baseDataCost + totalZeros * GasCostOf.TxDataZero + - (dataLength - totalZeros) * txDataNonZeroGasCost; + (data.Length - totalZeros) * txDataNonZeroGasCost; } private static long AccessListCost(Transaction transaction, IReleaseSpec releaseSpec)