From e4725a48e75ef42f1211340f5f004b975607e4a9 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 17 May 2023 20:23:52 +0800 Subject: [PATCH 01/23] Integrate with ReceiptRecoveryBlock --- .../Receipts/ReceiptsIteratorTests.cs | 2 +- .../Receipts/IReceiptsRecovery.cs | 11 +++- .../Receipts/PersistentReceiptStorage.cs | 3 +- .../Receipts/ReceiptRecoveryBlock.cs | 62 +++++++++++++++++++ .../Receipts/ReceiptsIterator.cs | 1 + .../Receipts/ReceiptsRecovery.cs | 27 ++++---- 6 files changed, 90 insertions(+), 16 deletions(-) create mode 100644 src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptRecoveryBlock.cs diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/ReceiptsIteratorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/ReceiptsIteratorTests.cs index 4c2e9033965..8734c5115f4 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/ReceiptsIteratorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/ReceiptsIteratorTests.cs @@ -112,7 +112,7 @@ private ReceiptsIterator CreateIterator(TxReceipt[] receipts, Block block) false ); - ReceiptsIterator iterator = new ReceiptsIterator(span, blockDb, () => recovery.CreateRecoveryContext(block), _decoder.GetRefDecoder(span)); + ReceiptsIterator iterator = new ReceiptsIterator(span, blockDb, () => recovery.CreateRecoveryContext(new ReceiptRecoveryBlock(block)), _decoder.GetRefDecoder(span)); return iterator; } } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs index 14c902ffff4..13baa72c8c9 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs @@ -1,18 +1,23 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using Nethermind.Core; namespace Nethermind.Blockchain.Receipts { public interface IReceiptsRecovery { - ReceiptsRecoveryResult TryRecover(Block block, TxReceipt[] receipts, bool forceRecoverSender = true); + ReceiptsRecoveryResult TryRecover(ReceiptRecoveryBlock block, TxReceipt[] receipts, bool forceRecoverSender = true); + + ReceiptsRecoveryResult TryRecover(Block block, TxReceipt[] receipts, bool forceRecoverSender = true) => + TryRecover(new ReceiptRecoveryBlock(block), receipts, forceRecoverSender); + bool NeedRecover(TxReceipt[] receipts, bool forceRecoverSender = true, bool recoverSenderOnly = false); - IRecoveryContext CreateRecoveryContext(Block block, bool forceRecoverSender = false); + IRecoveryContext CreateRecoveryContext(ReceiptRecoveryBlock block, bool forceRecoverSender = false); - public interface IRecoveryContext + public interface IRecoveryContext: IDisposable { void RecoverReceiptData(TxReceipt receipt); void RecoverReceiptData(ref TxReceiptStructRef receipt); diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs index 6f0b0636893..2467b92d9d4 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs @@ -238,7 +238,8 @@ public bool TryGetReceiptsIterator(long blockNumber, Keccak blockHash, out Recei recoveryContextFactory = () => { Block block = _blockTree.FindBlock(blockHash); - return _receiptsRecovery.CreateRecoveryContext(block!); + // TODO: Send lazy here + return _receiptsRecovery.CreateRecoveryContext(new ReceiptRecoveryBlock(block!)); }; } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptRecoveryBlock.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptRecoveryBlock.cs new file mode 100644 index 00000000000..304a45434b0 --- /dev/null +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptRecoveryBlock.cs @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using System.Text; +using Nethermind.Core.Crypto; +using Nethermind.Int256; +using Nethermind.Serialization.Rlp; + +namespace Nethermind.Core; + +/// +/// A Block specifically for receipt recovery. Does not contain any of the fields that are not needed for that. +/// Retain span from DB as memory and must be explicitly disposed. +/// +[DebuggerDisplay("{Hash} ({Number})")] +public class ReceiptRecoveryBlock +{ + public ReceiptRecoveryBlock(Block block) + { + Header = block.Header; + _transactions = block.Transactions; + } + + public ReceiptRecoveryBlock(BlockHeader header, Memory data) + { + Header = header ?? throw new ArgumentNullException(nameof(header)); + Data = data; + } + + private Memory Data { get; set; } + private Memory[] TransactionData { get; set; } + + + private Transaction[]? _transactions = null; + + public BlockHeader Header { get; } + public int TransactionCount => _transactions?.Length ?? TransactionData.Length; + + public Transaction GetTransaction(int idx) + { + if (_transactions != null) + { + return _transactions[idx]; + } + + Rlp.ValueDecoderContext decoderContext = new(TransactionData[idx].Span); + + return TxDecoder.Instance.Decode(ref decoderContext); + } + + public Keccak? Hash => Header.Hash; // do not add setter here + public long Number => Header.Number; // do not add setter here + + public void Dispose() + { + } +} diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsIterator.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsIterator.cs index 0aceea457c0..b95f2b1af5a 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsIterator.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsIterator.cs @@ -111,6 +111,7 @@ public void Dispose() { _blocksDb?.DangerousReleaseMemory(_decoderContext.Data); } + _recoveryContext?.Dispose(); } public LogEntriesIterator IterateLogs(TxReceiptStructRef receipt) diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs index 779e42672eb..1f638f349fe 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs @@ -23,16 +23,16 @@ public ReceiptsRecovery(IEthereumEcdsa? ecdsa, ISpecProvider? specProvider, bool _reinsertReceiptOnRecover = reinsertReceiptOnRecover; } - public ReceiptsRecoveryResult TryRecover(Block block, TxReceipt[] receipts, bool forceRecoverSender = true) + public ReceiptsRecoveryResult TryRecover(ReceiptRecoveryBlock block, TxReceipt[] receipts, bool forceRecoverSender = true) { - var canRecover = block.Transactions.Length == receipts?.Length; + var canRecover = block.TransactionCount == receipts?.Length; if (canRecover) { var needRecover = NeedRecover(receipts, forceRecoverSender); if (needRecover) { - var ctx = CreateRecoveryContext(block, forceRecoverSender); - for (int receiptIndex = 0; receiptIndex < block.Transactions.Length; receiptIndex++) + using var ctx = CreateRecoveryContext(block, forceRecoverSender); + for (int receiptIndex = 0; receiptIndex < block.TransactionCount; receiptIndex++) { if (receipts.Length > receiptIndex) { @@ -55,7 +55,7 @@ public ReceiptsRecoveryResult TryRecover(Block block, TxReceipt[] receipts, bool return ReceiptsRecoveryResult.Fail; } - public IReceiptsRecovery.IRecoveryContext CreateRecoveryContext(Block block, bool forceRecoverSender = false) + public IReceiptsRecovery.IRecoveryContext CreateRecoveryContext(ReceiptRecoveryBlock block, bool forceRecoverSender = false) { var releaseSpec = _specProvider.GetSpec(block.Header); return new RecoveryContext(releaseSpec, block, forceRecoverSender, _ecdsa); @@ -72,14 +72,14 @@ public bool NeedRecover(TxReceipt[] receipts, bool forceRecoverSender = true, bo private class RecoveryContext : IReceiptsRecovery.IRecoveryContext { private readonly IReleaseSpec _releaseSpec; - private readonly Block _block; + private readonly ReceiptRecoveryBlock _block; private readonly bool _forceRecoverSender; private readonly IEthereumEcdsa _ecdsa; private long _gasUsedBefore = 0; private int _transactionIndex = 0; - public RecoveryContext(IReleaseSpec releaseSpec, Block block, bool forceRecoverSender, IEthereumEcdsa ecdsa) + public RecoveryContext(IReleaseSpec releaseSpec, ReceiptRecoveryBlock block, bool forceRecoverSender, IEthereumEcdsa ecdsa) { _releaseSpec = releaseSpec; _block = block; @@ -89,12 +89,12 @@ public RecoveryContext(IReleaseSpec releaseSpec, Block block, bool forceRecoverS public void RecoverReceiptData(TxReceipt receipt) { - if (_transactionIndex >= _block.Transactions.Length) + if (_transactionIndex >= _block.TransactionCount) { throw new InvalidOperationException("Trying to recover more receipt that transaction"); } - Transaction transaction = _block.Transactions[_transactionIndex]; + Transaction transaction = _block.GetTransaction(_transactionIndex); receipt.TxType = transaction.Type; receipt.BlockHash = _block.Hash; @@ -117,12 +117,12 @@ public void RecoverReceiptData(TxReceipt receipt) public void RecoverReceiptData(ref TxReceiptStructRef receipt) { - if (_transactionIndex >= _block.Transactions.Length) + if (_transactionIndex >= _block.TransactionCount) { throw new InvalidOperationException("Trying to recover more receipt that transaction"); } - Transaction transaction = _block.Transactions[_transactionIndex]; + Transaction transaction = _block.GetTransaction(_transactionIndex); receipt.TxType = transaction.Type; receipt.BlockHash = _block.Hash!.ToStructRef(); @@ -151,6 +151,11 @@ private void IncrementContext(long gasUsedTotal) _transactionIndex++; _gasUsedBefore = gasUsedTotal; } + + public void Dispose() + { + _block.Dispose(); + } } } } From d2b369d06e46f366a7af2e78427e297151ad786b Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 17 May 2023 21:54:54 +0800 Subject: [PATCH 02/23] Can run --- .../Receipts/PersistentReceiptStorageTests.cs | 2 +- .../Receipts/ReceiptRecoveryBlock.cs | 34 ++++++++---- .../Receipts/ReceiptsRecovery.cs | 4 +- .../Nethermind.Db.Test/DbOnTheRocksTests.cs | 17 ++++++ .../Nethermind.Db/DbSpanMemoryManager.cs | 52 +++++++++++++++++++ .../Nethermind.Db/Nethermind.Db.csproj | 1 + .../Nethermind.Serialization.Rlp/TxDecoder.cs | 1 + 7 files changed, 97 insertions(+), 14 deletions(-) create mode 100644 src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs index 3186373e6ac..65192b39fd7 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs @@ -393,7 +393,7 @@ public void When_NewHeadBlock_ClearOldTxIndex() { (block, TxReceipt[] receipts) = PrepareBlock(block, isFinalized, headNumber); _storage.Insert(block, receipts); - _receiptsRecovery.TryRecover(block, receipts); + _receiptsRecovery.TryRecover(new ReceiptRecoveryBlock(block), receipts); return (block, receipts); } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptRecoveryBlock.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptRecoveryBlock.cs index 304a45434b0..c33962f98ea 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptRecoveryBlock.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptRecoveryBlock.cs @@ -2,12 +2,16 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Reflection; +using System.Runtime.InteropServices; using System.Text; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; +using Nethermind.Db; using Nethermind.Int256; using Nethermind.Serialization.Rlp; @@ -24,33 +28,40 @@ public ReceiptRecoveryBlock(Block block) { Header = block.Header; _transactions = block.Transactions; + TransactionCount = _transactions.Length; } - public ReceiptRecoveryBlock(BlockHeader header, Memory data) + public ReceiptRecoveryBlock(IMemoryOwner? memoryOwner, BlockHeader header, Memory transactionData, int transactionCount) { - Header = header ?? throw new ArgumentNullException(nameof(header)); - Data = data; + Header = header; + _memoryOwner = memoryOwner; + _transactionData = transactionData; + TransactionCount = transactionCount; } - private Memory Data { get; set; } - private Memory[] TransactionData { get; set; } - + private IMemoryOwner? _memoryOwner; + private Memory _transactionData { get; set; } + private int _currentTransactionPosition = 0; private Transaction[]? _transactions = null; + private int _currentTransactionIndex = 0; public BlockHeader Header { get; } - public int TransactionCount => _transactions?.Length ?? TransactionData.Length; + public int TransactionCount { get; } - public Transaction GetTransaction(int idx) + public Transaction GetNextTransaction() { if (_transactions != null) { - return _transactions[idx]; + return _transactions[_currentTransactionIndex++]; } - Rlp.ValueDecoderContext decoderContext = new(TransactionData[idx].Span); + Rlp.ValueDecoderContext decoderContext = new(_transactionData.Span); + decoderContext.Position = _currentTransactionPosition; + Transaction tx = TxDecoder.Instance.Decode(ref decoderContext); + _currentTransactionPosition = decoderContext.Position; - return TxDecoder.Instance.Decode(ref decoderContext); + return tx; } public Keccak? Hash => Header.Hash; // do not add setter here @@ -58,5 +69,6 @@ public Transaction GetTransaction(int idx) public void Dispose() { + _memoryOwner?.Dispose(); } } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs index 1f638f349fe..f53135fd21c 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs @@ -94,7 +94,7 @@ public void RecoverReceiptData(TxReceipt receipt) throw new InvalidOperationException("Trying to recover more receipt that transaction"); } - Transaction transaction = _block.GetTransaction(_transactionIndex); + Transaction transaction = _block.GetNextTransaction(); receipt.TxType = transaction.Type; receipt.BlockHash = _block.Hash; @@ -122,7 +122,7 @@ public void RecoverReceiptData(ref TxReceiptStructRef receipt) throw new InvalidOperationException("Trying to recover more receipt that transaction"); } - Transaction transaction = _block.GetTransaction(_transactionIndex); + Transaction transaction = _block.GetNextTransaction(); receipt.TxType = transaction.Type; receipt.BlockHash = _block.Hash!.ToStructRef(); diff --git a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs index 6b7a9587138..cd8a773d5c3 100644 --- a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers; using System.Collections.Generic; using System.IO; using System.IO.Abstractions; @@ -10,6 +11,7 @@ using System.Threading.Tasks; using FluentAssertions; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Db.Rocks; @@ -249,6 +251,21 @@ public void Smoke_test_span() _db.DangerousReleaseMemory(readSpan); } + [Test] + public void Smoke_test_span_with_memory_manager() + { + byte[] key = new byte[] { 1, 2, 3 }; + byte[] value = new byte[] { 4, 5, 6 }; + _db.PutSpan(key, value); + Span readSpan = _db.GetSpan(key); + Assert.That(readSpan.ToArray(), Is.EqualTo(new byte[] { 4, 5, 6 })); + + IMemoryOwner manager = new DbSpanMemoryManager(_db, readSpan); + Memory theMemory = manager.Memory; + Assert.That(theMemory.ToArray(), Is.EqualTo(new byte[] { 4, 5, 6 })); + manager.Dispose(); + } + private static RocksDbSettings GetRocksDbSettings(string dbPath, string dbName) { return new(dbName, dbPath) diff --git a/src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs b/src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs new file mode 100644 index 00000000000..c3c074d5aec --- /dev/null +++ b/src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Nethermind.Db; + +namespace Nethermind.Core.Buffers; + +public class DbSpanMemoryManager: MemoryManager +{ + private readonly IDbWithSpan _db; + private readonly unsafe void* _ptr; + private readonly int _length; + + public DbSpanMemoryManager(IDbWithSpan db, Span span) + { + unsafe + { + _db = db; + _ptr = Unsafe.AsPointer(ref span.GetPinnableReference()); + _length = span.Length; + } + } + + protected override void Dispose(bool disposing) + { + _db.DangerousReleaseMemory(GetSpan()); + } + + public override Span GetSpan() + { + unsafe + { + return new Span(_ptr, _length); + } + } + + public override MemoryHandle Pin(int elementIndex = 0) + { + unsafe + { + return new MemoryHandle(_ptr); + } + } + + public override void Unpin() + { + } +} diff --git a/src/Nethermind/Nethermind.Db/Nethermind.Db.csproj b/src/Nethermind/Nethermind.Db/Nethermind.Db.csproj index 3eb9ee73a73..10676c58f85 100644 --- a/src/Nethermind/Nethermind.Db/Nethermind.Db.csproj +++ b/src/Nethermind/Nethermind.Db/Nethermind.Db.csproj @@ -4,6 +4,7 @@ latest annotations true + true diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs index 8d1402c6047..7449e61521b 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs @@ -13,6 +13,7 @@ namespace Nethermind.Serialization.Rlp public class TxDecoder : TxDecoder, ITransactionSizeCalculator { public const int MaxDelayedHashTxnSize = 32768; + public static TxDecoder Instance = new TxDecoder(); public int GetLength(Transaction tx) { From d3e2018b353dbc9ed3ae9857071d69ef795dd97c Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 17 May 2023 22:13:46 +0800 Subject: [PATCH 03/23] Injected block store --- .../Receipts/PersistentReceiptStorageTests.cs | 4 ++++ .../Nethermind.Blockchain/BlockTree.cs | 23 +++++++++++++++---- .../Receipts/PersistentReceiptStorage.cs | 6 ++++- .../Steps/InitializeBlockTree.cs | 9 +++++++- .../Modules/Trace/ParityStyleTracerTests.cs | 1 + .../ReceiptStorageTests.cs | 4 ++++ 6 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs index 65192b39fd7..23c2a5f85f2 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Threading; using FluentAssertions; +using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Receipts; using Nethermind.Core; using Nethermind.Core.Crypto; @@ -29,6 +30,7 @@ public class PersistentReceiptStorageTests private TestMemColumnsDb _receiptsDb = null!; private ReceiptsRecovery _receiptsRecovery; private IBlockTree _blockTree; + private IBlockStore _blockStore; private readonly bool _useCompactReceipts; private ReceiptConfig _receiptConfig; private PersistentReceiptStorage _storage; @@ -49,6 +51,7 @@ public void SetUp() _receiptsDb = new TestMemColumnsDb(); _receiptsDb.GetColumnDb(ReceiptsColumns.Blocks).Set(Keccak.Zero, Array.Empty()); _blockTree = Substitute.For(); + _blockStore = Substitute.For(); CreateStorage(); } @@ -60,6 +63,7 @@ private void CreateStorage() MainnetSpecProvider.Instance, _receiptsRecovery, _blockTree, + _blockStore, _receiptConfig, _decoder ) diff --git a/src/Nethermind/Nethermind.Blockchain/BlockTree.cs b/src/Nethermind/Nethermind.Blockchain/BlockTree.cs index 4220e7acda1..6cf184341f0 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockTree.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockTree.cs @@ -120,7 +120,7 @@ public BlockTree( ISpecProvider? specProvider, IBloomStorage? bloomStorage, ILogManager? logManager) - : this(dbProvider?.BlocksDb, dbProvider?.HeadersDb, dbProvider?.BlockInfosDb, dbProvider?.MetadataDb, + : this(new BlockStore(dbProvider?.BlocksDb), dbProvider?.HeadersDb, dbProvider?.BlockInfosDb, dbProvider?.MetadataDb, chainLevelInfoRepository, specProvider, bloomStorage, new SyncConfig(), logManager) { } @@ -132,7 +132,7 @@ public BlockTree( IBloomStorage? bloomStorage, ISyncConfig? syncConfig, ILogManager? logManager) - : this(dbProvider?.BlocksDb, dbProvider?.HeadersDb, dbProvider?.BlockInfosDb, dbProvider?.MetadataDb, + : this(new BlockStore(dbProvider?.BlocksDb), dbProvider?.HeadersDb, dbProvider?.BlockInfosDb, dbProvider?.MetadataDb, chainLevelInfoRepository, specProvider, bloomStorage, syncConfig, logManager) { } @@ -145,7 +145,7 @@ public BlockTree( ISpecProvider? specProvider, IBloomStorage? bloomStorage, ILogManager? logManager) - : this(blockDb, headerDb, blockInfoDb, new MemDb(), chainLevelInfoRepository, specProvider, bloomStorage, + : this(new BlockStore(blockDb), headerDb, blockInfoDb, new MemDb(), chainLevelInfoRepository, specProvider, bloomStorage, new SyncConfig(), logManager) { } @@ -160,9 +160,24 @@ public BlockTree( IBloomStorage? bloomStorage, ISyncConfig? syncConfig, ILogManager? logManager) + : this(new BlockStore(blockDb), headerDb, blockInfoDb, metadataDb, chainLevelInfoRepository, specProvider, bloomStorage, + syncConfig, logManager) + { + } + + public BlockTree( + IBlockStore? blockStore, + IDb? headerDb, + IDb? blockInfoDb, + IDb? metadataDb, + IChainLevelInfoRepository? chainLevelInfoRepository, + ISpecProvider? specProvider, + IBloomStorage? bloomStorage, + ISyncConfig? syncConfig, + ILogManager? logManager) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _blockStore = new BlockStore(blockDb ?? throw new ArgumentNullException(nameof(blockDb))); + _blockStore = blockStore ?? throw new ArgumentNullException(nameof(blockStore)); _headerDb = headerDb ?? throw new ArgumentNullException(nameof(headerDb)); _blockInfoDb = blockInfoDb ?? throw new ArgumentNullException(nameof(blockInfoDb)); _metadataDb = metadataDb ?? throw new ArgumentNullException(nameof(metadataDb)); diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs index 2467b92d9d4..194dbfda09a 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Threading.Tasks; +using Nethermind.Blockchain.Blocks; using Nethermind.Core; using Nethermind.Core.Caching; using Nethermind.Core.Crypto; @@ -29,6 +30,7 @@ public class PersistentReceiptStorage : IReceiptStorage private long _migratedBlockNumber; private readonly ReceiptArrayStorageDecoder _storageDecoder = ReceiptArrayStorageDecoder.Instance; private readonly IBlockTree _blockTree; + private readonly IBlockStore _blockStore; private readonly IReceiptConfig _receiptConfig; private readonly bool _legacyHashKey; @@ -40,6 +42,7 @@ public PersistentReceiptStorage( ISpecProvider specProvider, IReceiptsRecovery receiptsRecovery, IBlockTree blockTree, + IBlockStore blockStore, IReceiptConfig receiptConfig, ReceiptArrayStorageDecoder? storageDecoder = null ) @@ -52,6 +55,7 @@ public PersistentReceiptStorage( _blocksDb = _database.GetColumnDb(ReceiptsColumns.Blocks); _transactionDb = _database.GetColumnDb(ReceiptsColumns.Transactions); _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); + _blockStore = blockStore ?? throw new ArgumentNullException(nameof(blockStore)); _storageDecoder = storageDecoder ?? ReceiptArrayStorageDecoder.Instance; _receiptConfig = receiptConfig ?? throw new ArgumentNullException(nameof(receiptConfig)); @@ -237,7 +241,7 @@ public bool TryGetReceiptsIterator(long blockNumber, Keccak blockHash, out Recei { recoveryContextFactory = () => { - Block block = _blockTree.FindBlock(blockHash); + Block block = _blockStore.Get(blockHash); // TODO: Send lazy here return _receiptsRecovery.CreateRecoveryContext(new ReceiptRecoveryBlock(block!)); }; diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockTree.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockTree.cs index 9c6946c953f..277413e51d4 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockTree.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockTree.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Nethermind.Api; using Nethermind.Blockchain; +using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; @@ -48,8 +49,13 @@ public Task Execute(CancellationToken cancellationToken) IChainLevelInfoRepository chainLevelInfoRepository = _set.ChainLevelInfoRepository = new ChainLevelInfoRepository(_get.DbProvider!.BlockInfosDb); + IBlockStore blockStore = new BlockStore(_get.DbProvider.BlocksDb); + IBlockTree blockTree = _set.BlockTree = new BlockTree( - _get.DbProvider, + blockStore, + _get.DbProvider.HeadersDb, + _get.DbProvider.BlockInfosDb, + _get.DbProvider.MetadataDb, chainLevelInfoRepository, _get.SpecProvider, bloomStorage, @@ -76,6 +82,7 @@ public Task Execute(CancellationToken cancellationToken) _get.SpecProvider!, receiptsRecovery, blockTree, + blockStore, receiptConfig, new ReceiptArrayStorageDecoder(receiptConfig.CompactReceiptStore)) : NullReceiptStorage.Instance; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs index 9d7d196d877..0303160e4f6 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs @@ -4,6 +4,7 @@ using System.Linq; using FluentAssertions; using Nethermind.Blockchain; +using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; diff --git a/src/Nethermind/Nethermind.TxPool.Test/ReceiptStorageTests.cs b/src/Nethermind/Nethermind.TxPool.Test/ReceiptStorageTests.cs index 63604449679..7729c797877 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/ReceiptStorageTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/ReceiptStorageTests.cs @@ -3,6 +3,7 @@ using FluentAssertions; using Nethermind.Blockchain; +using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Core; @@ -28,6 +29,7 @@ public class ReceiptStorageTests private IReceiptFinder _receiptFinder; private IReceiptStorage _inMemoryStorage; private IBlockTree _blockTree; + private IBlockStore _blockStore; public ReceiptStorageTests(bool useEip2718) { @@ -42,12 +44,14 @@ public void Setup() _blockTree = Build.A.BlockTree() .WithBlocks(Build.A.Block.TestObject) .TestObject; + _blockStore = Substitute.For(); ReceiptsRecovery receiptsRecovery = new(_ethereumEcdsa, _specProvider); _persistentStorage = new PersistentReceiptStorage( new MemColumnsDb(), _specProvider, receiptsRecovery, _blockTree, + _blockStore, new ReceiptConfig() ); _receiptFinder = new FullInfoReceiptFinder(_persistentStorage, receiptsRecovery, Substitute.For()); From d2261a8f8e144c61a98dfa3645d17c0d68f47d68 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 17 May 2023 22:20:01 +0800 Subject: [PATCH 04/23] Setup integration up to block store --- .../Blocks/BlockStoreTests.cs | 21 +++++++++++++++++++ .../Blocks/BlockStore.cs | 6 ++++++ .../Blocks/IBlockStore.cs | 1 + .../Receipts/PersistentReceiptStorage.cs | 5 ++--- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs index d213605f506..04a1e9768ab 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs @@ -60,4 +60,25 @@ public void Test_when_cached_does_not_touch_db_on_next_get() retrieved = store.Get(block.Hash, true); retrieved.Should().BeEquivalentTo(block); } + + [Test] + public void Test_getReceiptRecoveryBlock_produce_same_transaction_as_normal_get() + { + TestMemDb db = new TestMemDb(); + BlockStore store = new BlockStore(db); + + Block block = Build.A.Block.WithNumber(1).TestObject; + store.Insert(block); + + ReceiptRecoveryBlock? retrieved = store.GetReceiptRecoveryBlock(block.Hash); + retrieved.Should().NotBeNull(); + + retrieved.Header.Should().BeEquivalentTo(block.Header); + retrieved.TransactionCount.Should().Be(block.Transactions.Length); + + for (int i = 0; i < retrieved.TransactionCount; i++) + { + retrieved.GetNextTransaction().Should().Be(block.Transactions[i]); + } + } } diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index 0b4df139b66..ed0b364df38 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -58,6 +58,12 @@ public void Delete(Keccak blockHash) return _blockDb.Get(blockHash, _blockDecoder, _blockCache, shouldCache); } + public ReceiptRecoveryBlock GetReceiptRecoveryBlock(Keccak blockHash) + { + // TODO: Hard logic here + return new ReceiptRecoveryBlock(Get(blockHash, false)); + } + public void Cache(Block block) { _blockCache.Set(block.Hash, block); diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs index 0c03587569d..5765a5332d2 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs @@ -15,6 +15,7 @@ public interface IBlockStore void Insert(Block block); void Delete(Keccak blockHash); Block Get(Keccak blockHash, bool shouldCache = true); + ReceiptRecoveryBlock GetReceiptRecoveryBlock(Keccak blockHash); void Cache(Block block); diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs index 194dbfda09a..dbdb506b3c7 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs @@ -241,9 +241,8 @@ public bool TryGetReceiptsIterator(long blockNumber, Keccak blockHash, out Recei { recoveryContextFactory = () => { - Block block = _blockStore.Get(blockHash); - // TODO: Send lazy here - return _receiptsRecovery.CreateRecoveryContext(new ReceiptRecoveryBlock(block!)); + ReceiptRecoveryBlock block = _blockStore.GetReceiptRecoveryBlock(blockHash); + return _receiptsRecovery.CreateRecoveryContext(block); }; } From 9a5fcf5481da263ae7ccbe13d700b8635ae0f5f0 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 18 May 2023 12:42:27 +0800 Subject: [PATCH 05/23] Shortcut deserialization --- .../Blocks/BlockStoreTests.cs | 12 +++++-- .../Blocks/BlockStore.cs | 9 +++-- .../Blocks/IBlockStore.cs | 1 + .../Receipts/IReceiptsRecovery.cs | 1 + .../Receipts/ReceiptsRecovery.cs | 1 + src/Nethermind/Nethermind.Db/IDbWithSpan.cs | 7 ++++ .../BlockDecoder.cs | 36 +++++++++++++++++++ .../ReceiptRecoveryBlock.cs | 16 +++------ 8 files changed, 67 insertions(+), 16 deletions(-) rename src/Nethermind/{Nethermind.Blockchain/Receipts => Nethermind.Serialization.Rlp}/ReceiptRecoveryBlock.cs (81%) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs index 04a1e9768ab..501ba334dfa 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs @@ -1,11 +1,15 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; +using System.Runtime.InteropServices.JavaScript; using FluentAssertions; using Nethermind.Blockchain.Blocks; using Nethermind.Core; using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; +using Nethermind.Serialization.Rlp; +using Nethermind.Specs; using NUnit.Framework; namespace Nethermind.Blockchain.Test.Blocks; @@ -67,7 +71,10 @@ public void Test_getReceiptRecoveryBlock_produce_same_transaction_as_normal_get( TestMemDb db = new TestMemDb(); BlockStore store = new BlockStore(db); - Block block = Build.A.Block.WithNumber(1).TestObject; + Block block = Build.A.Block.WithNumber(1) + .WithTransactions(3, MainnetSpecProvider.Instance) + .TestObject; + store.Insert(block); ReceiptRecoveryBlock? retrieved = store.GetReceiptRecoveryBlock(block.Hash); @@ -78,7 +85,8 @@ public void Test_getReceiptRecoveryBlock_produce_same_transaction_as_normal_get( for (int i = 0; i < retrieved.TransactionCount; i++) { - retrieved.GetNextTransaction().Should().Be(block.Transactions[i]); + block.Transactions[i].Data = Array.Empty(); + retrieved.GetNextTransaction().Should().BeEquivalentTo(block.Transactions[i]); } } } diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index ed0b364df38..43346aeeeab 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers; using Nethermind.Core; using Nethermind.Core.Caching; using Nethermind.Core.Crypto; @@ -13,6 +14,7 @@ namespace Nethermind.Blockchain.Blocks; public class BlockStore : IBlockStore { private readonly IDb _blockDb; + private readonly IDbWithSpan _blockDbAsSpan; private readonly BlockDecoder _blockDecoder = new(); private const int CacheSize = 64; @@ -22,6 +24,7 @@ private readonly LruCache public BlockStore(IDb blockDb) { _blockDb = blockDb; + _blockDbAsSpan = (IDbWithSpan)_blockDb; } public void SetMetadata(byte[] key, byte[] value) @@ -55,13 +58,15 @@ public void Delete(Keccak blockHash) public Block? Get(Keccak blockHash, bool shouldCache) { + return _blockDb.Get(blockHash, _blockDecoder, _blockCache, shouldCache); } public ReceiptRecoveryBlock GetReceiptRecoveryBlock(Keccak blockHash) { - // TODO: Hard logic here - return new ReceiptRecoveryBlock(Get(blockHash, false)); + IMemoryOwner memory = _blockDbAsSpan.GetOwnedMemory(blockHash.Bytes); + + return _blockDecoder.DecodeToReceiptRecoveryBlock(memory, RlpBehaviors.None); } public void Cache(Block block) diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs index 5765a5332d2..219c0adcbf8 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs @@ -3,6 +3,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Serialization.Rlp; namespace Nethermind.Blockchain.Blocks; diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs index 13baa72c8c9..80041df1172 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs @@ -3,6 +3,7 @@ using System; using Nethermind.Core; +using Nethermind.Serialization.Rlp; namespace Nethermind.Blockchain.Receipts { diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs index f53135fd21c..9119de757b9 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs @@ -7,6 +7,7 @@ using Nethermind.Core.Specs; using Nethermind.Crypto; using Nethermind.Evm; +using Nethermind.Serialization.Rlp; namespace Nethermind.Blockchain.Receipts { diff --git a/src/Nethermind/Nethermind.Db/IDbWithSpan.cs b/src/Nethermind/Nethermind.Db/IDbWithSpan.cs index 461b702ab15..716e01ba2b5 100644 --- a/src/Nethermind/Nethermind.Db/IDbWithSpan.cs +++ b/src/Nethermind/Nethermind.Db/IDbWithSpan.cs @@ -2,6 +2,8 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers; +using Nethermind.Core.Buffers; namespace Nethermind.Db { @@ -15,5 +17,10 @@ public interface IDbWithSpan : IDb Span GetSpan(ReadOnlySpan key); void PutSpan(ReadOnlySpan key, ReadOnlySpan value); void DangerousReleaseMemory(in Span span); + IMemoryOwner GetOwnedMemory(ReadOnlySpan key) + { + Span span = GetSpan(key); + return new DbSpanMemoryManager(this, span); + } } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs index 425f84c664b..1301ea3b6b7 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; +using System.Buffers; using System.Collections.Generic; using System.Linq; using Nethermind.Core; @@ -246,5 +248,39 @@ public void Encode(RlpStream stream, Block? item, RlpBehaviors rlpBehaviors = Rl } } } + + public ReceiptRecoveryBlock DecodeToReceiptRecoveryBlock(IMemoryOwner memory, RlpBehaviors rlpBehaviors) + { + Rlp.ValueDecoderContext decoderContext = new Rlp.ValueDecoderContext(memory.Memory.Span); + + if (decoderContext.IsNextItemNull()) + { + decoderContext.ReadByte(); + return null; + } + + int sequenceLength = decoderContext.ReadSequenceLength(); + int blockCheck = decoderContext.Position + sequenceLength; + + BlockHeader header = Rlp.Decode(ref decoderContext); + + int contentLength = decoderContext.ReadSequenceLength(); + int transactionCount = decoderContext.PeekNumberOfItemsRemaining(decoderContext.Position + contentLength); + Memory transactionMemory = memory.Memory.Slice(decoderContext.Position, contentLength); + decoderContext.Position += contentLength; // Skip the whole tx + + decoderContext.SkipItem(); // Skip uncles + + if (decoderContext.Position != blockCheck) + { + decoderContext.SkipItem(); // Skip withdrawals + } + if ((rlpBehaviors & RlpBehaviors.AllowExtraBytes) != RlpBehaviors.AllowExtraBytes) + { + decoderContext.Check(blockCheck); + } + + return new ReceiptRecoveryBlock(memory, header, transactionMemory, transactionCount); + } } } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptRecoveryBlock.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs similarity index 81% rename from src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptRecoveryBlock.cs rename to src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs index c33962f98ea..c0ab95ede78 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptRecoveryBlock.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs @@ -3,19 +3,11 @@ using System; using System.Buffers; -using System.Collections.Generic; using System.Diagnostics; -using System.Linq; -using System.Reflection; -using System.Runtime.InteropServices; -using System.Text; -using Nethermind.Core.Buffers; +using Nethermind.Core; using Nethermind.Core.Crypto; -using Nethermind.Db; -using Nethermind.Int256; -using Nethermind.Serialization.Rlp; -namespace Nethermind.Core; +namespace Nethermind.Serialization.Rlp; /// /// A Block specifically for receipt recovery. Does not contain any of the fields that are not needed for that. @@ -31,7 +23,7 @@ public ReceiptRecoveryBlock(Block block) TransactionCount = _transactions.Length; } - public ReceiptRecoveryBlock(IMemoryOwner? memoryOwner, BlockHeader header, Memory transactionData, int transactionCount) + public ReceiptRecoveryBlock(IMemoryOwner memoryOwner, BlockHeader header, Memory transactionData, int transactionCount) { Header = header; _memoryOwner = memoryOwner; @@ -58,7 +50,7 @@ public Transaction GetNextTransaction() Rlp.ValueDecoderContext decoderContext = new(_transactionData.Span); decoderContext.Position = _currentTransactionPosition; - Transaction tx = TxDecoder.Instance.Decode(ref decoderContext); + Transaction tx = TxDecoder.Instance.Decode(ref decoderContext, RlpBehaviors.AllowUnsigned); _currentTransactionPosition = decoderContext.Position; return tx; From 3d5dbebcc46b42f35ed873fefd0a55558065a772 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 18 May 2023 15:46:03 +0800 Subject: [PATCH 06/23] Disable lazy hash --- src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs | 2 +- src/Nethermind/Nethermind.Core/Transaction.cs | 4 ++-- src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs | 1 + src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index 43346aeeeab..056d84f176f 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -66,7 +66,7 @@ public ReceiptRecoveryBlock GetReceiptRecoveryBlock(Keccak blockHash) { IMemoryOwner memory = _blockDbAsSpan.GetOwnedMemory(blockHash.Bytes); - return _blockDecoder.DecodeToReceiptRecoveryBlock(memory, RlpBehaviors.None); + return _blockDecoder.DecodeToReceiptRecoveryBlock(memory, RlpBehaviors.None | RlpBehaviors.DisableLazyHash); } public void Cache(Block block) diff --git a/src/Nethermind/Nethermind.Core/Transaction.cs b/src/Nethermind/Nethermind.Core/Transaction.cs index fecb85a5229..dabd6e78542 100644 --- a/src/Nethermind/Nethermind.Core/Transaction.cs +++ b/src/Nethermind/Nethermind.Core/Transaction.cs @@ -66,11 +66,11 @@ public Keccak? Hash } set { - lock (this) + if (_preHash.Count != 0) { ClearPreHashInternal(); - _hash = value; } + _hash = value; } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs index fb0947e543a..30afd4c82f5 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs @@ -18,6 +18,7 @@ public enum RlpBehaviors // when we're calculating tx hash or sending raw transaction we should skip this wrapping // with additional wrapping for typed transactions we're decoding Uint8Array([TransactionType, TransactionPayload] // without wrapping we're decoding (TransactionType || TransactionPayload) + DisableLazyHash = 64, All = AllowExtraBytes | ForSealing | Storage | Eip658Receipts | AllowUnsigned | SkipTypedWrapping } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs index 7449e61521b..9585681681d 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs @@ -89,7 +89,7 @@ public class TxDecoder : rlpStream.Check(lastCheck); } - if (transactionSequence.Length <= TxDecoder.MaxDelayedHashTxnSize) + if (transactionSequence.Length <= TxDecoder.MaxDelayedHashTxnSize && (rlpBehaviors & RlpBehaviors.DisableLazyHash) == 0) { // Delay hash generation, as may be filtered as having too low gas etc transaction.SetPreHash(transactionSequence); @@ -271,7 +271,7 @@ private void EncodeEip1559PayloadWithoutPayload(T item, RlpStream stream, RlpBeh decoderContext.Check(lastCheck); } - if (transactionSequence.Length <= TxDecoder.MaxDelayedHashTxnSize) + if (transactionSequence.Length <= TxDecoder.MaxDelayedHashTxnSize && (rlpBehaviors & RlpBehaviors.DisableLazyHash) == 0) { // Delay hash generation, as may be filtered as having too low gas etc transaction.SetPreHash(transactionSequence); From 1b608446c7a5f60e8f05ec570640f091ade21074 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 18 May 2023 15:46:49 +0800 Subject: [PATCH 07/23] Fix memory leak --- src/Nethermind/Nethermind.Facade/Filters/LogFinder.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Facade/Filters/LogFinder.cs b/src/Nethermind/Nethermind.Facade/Filters/LogFinder.cs index 91968829bff..3d744691df2 100644 --- a/src/Nethermind/Nethermind.Facade/Filters/LogFinder.cs +++ b/src/Nethermind/Nethermind.Facade/Filters/LogFinder.cs @@ -214,7 +214,7 @@ private IEnumerable FindLogsInBlock(LogFilter filter, Keccak blockHas private static IEnumerable FilterLogsInBlockLowMemoryAllocation(LogFilter filter, ref ReceiptsIterator iterator, CancellationToken cancellationToken) { List logList = null; - using (iterator) + try { long logIndexInBlock = 0; while (iterator.TryGetNext(out var receipt)) @@ -262,6 +262,11 @@ private static IEnumerable FilterLogsInBlockLowMemoryAllocation(LogFi } } } + finally + { + // Confusingly, the `using` statement causes the recovery context to be null during the dispose. + iterator.Dispose(); + } return logList ?? (IEnumerable)Array.Empty(); } From 2f2f8a72ef0da2dbe1d5d7b009b42956a0067be8 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 18 May 2023 16:44:53 +0800 Subject: [PATCH 08/23] Disable lazy hash --- .../Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs index c0ab95ede78..6967637311f 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs @@ -50,7 +50,7 @@ public Transaction GetNextTransaction() Rlp.ValueDecoderContext decoderContext = new(_transactionData.Span); decoderContext.Position = _currentTransactionPosition; - Transaction tx = TxDecoder.Instance.Decode(ref decoderContext, RlpBehaviors.AllowUnsigned); + Transaction tx = TxDecoder.Instance.Decode(ref decoderContext, RlpBehaviors.AllowUnsigned | RlpBehaviors.DisableLazyHash); _currentTransactionPosition = decoderContext.Position; return tx; From 6f95b139ffd3ecb66270b82495e3906010a5404d Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 18 May 2023 20:50:35 +0800 Subject: [PATCH 09/23] Make tx use Memory --- .../ReportingValidatorContractTests.cs | 4 +-- .../Contract/ValidatorContractTests.cs | 5 ++-- .../Reward/AuRaRewardCalculatorTests.cs | 3 +- .../Validators/ContractBasedValidatorTests.cs | 2 +- .../TransactionPermissionContractV3.cs | 2 +- .../TransactionPermissionContractV4.cs | 3 +- .../TxPriorityContract.Destination.cs | 2 +- .../Encoding/TxDecoderTests.cs | 4 +-- .../Nethermind.Core.Test/TransactionTests.cs | 13 +++++++++ .../Extensions/MemoryExtensions.cs | 29 +++++++++++++++++++ src/Nethermind/Nethermind.Core/Transaction.cs | 6 ++-- .../Tracing/ParityLikeTxTracerTests.cs | 2 +- .../Nethermind.Evm/IntrinsicGasCalculator.cs | 4 ++- .../Tracing/ParityStyle/ParityLikeTxTracer.cs | 3 +- .../TransactionProcessor.cs | 5 ++-- .../Proxy/Models/CallTransactionModel.cs | 3 +- .../Data/Eip2930Tests.cs | 4 ++- .../Data/TransactionForRpc.cs | 11 +++++-- .../Modules/Parity/ParityTransaction.cs | 3 +- .../EngineModuleTests.V1.cs | 1 + .../P2P/SerializerTester.cs | 8 ++++- .../V62/TransactionsMessageSerializerTests.cs | 3 ++ ...ooledTransactionsMessageSerializerTests.cs | 3 ++ .../V66/BlockBodiesMessageSerializerTests.cs | 1 + .../Nethermind.Serialization.Rlp/TxDecoder.cs | 13 ++++----- 25 files changed, 105 insertions(+), 32 deletions(-) create mode 100644 src/Nethermind/Nethermind.Core/Extensions/MemoryExtensions.cs diff --git a/src/Nethermind/Nethermind.AuRa.Test/Contract/ReportingValidatorContractTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Contract/ReportingValidatorContractTests.cs index 2d6a8226378..af3dd07c56c 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Contract/ReportingValidatorContractTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Contract/ReportingValidatorContractTests.cs @@ -19,7 +19,7 @@ public void Should_generate_malicious_transaction() { ReportingValidatorContract contract = new(AbiEncoder.Instance, new Address("0x1000000000000000000000000000000000000001"), Substitute.For()); Transaction transaction = contract.ReportMalicious(new Address("0x75df42383afe6bf5194aa8fa0e9b3d5f9e869441"), 10, new byte[0]); - transaction.Data.ToHexString().Should().Be("c476dd4000000000000000000000000075df42383afe6bf5194aa8fa0e9b3d5f9e869441000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000000"); + transaction.Data.FasterToArray().ToHexString().Should().Be("c476dd4000000000000000000000000075df42383afe6bf5194aa8fa0e9b3d5f9e869441000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000000"); } [Test] @@ -27,7 +27,7 @@ public void Should_generate_benign_transaction() { ReportingValidatorContract contract = new(AbiEncoder.Instance, new Address("0x1000000000000000000000000000000000000001"), Substitute.For()); Transaction transaction = contract.ReportBenign(new Address("0x75df42383afe6bf5194aa8fa0e9b3d5f9e869441"), 10); - transaction.Data.ToHexString().Should().Be("d69f13bb00000000000000000000000075df42383afe6bf5194aa8fa0e9b3d5f9e869441000000000000000000000000000000000000000000000000000000000000000a"); + transaction.Data.FasterToArray().ToHexString().Should().Be("d69f13bb00000000000000000000000075df42383afe6bf5194aa8fa0e9b3d5f9e869441000000000000000000000000000000000000000000000000000000000000000a"); } } } diff --git a/src/Nethermind/Nethermind.AuRa.Test/Contract/ValidatorContractTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Contract/ValidatorContractTests.cs index 9b287c3735f..d6df3491e65 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Contract/ValidatorContractTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Contract/ValidatorContractTests.cs @@ -8,6 +8,7 @@ using Nethermind.Consensus.AuRa.Contracts; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; using Nethermind.Crypto; using Nethermind.Evm; @@ -84,11 +85,11 @@ public void finalize_change_should_call_correct_transaction() Arg.Is(t => IsEquivalentTo(expectation, t)), _block.Header, Arg.Any()); } - private static bool IsEquivalentTo(object expected, object item) + private static bool IsEquivalentTo(Transaction expected, Transaction item) { try { - item.Should().BeEquivalentTo(expected); + item.EqualToTransaction(expected); return true; } catch diff --git a/src/Nethermind/Nethermind.AuRa.Test/Reward/AuRaRewardCalculatorTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Reward/AuRaRewardCalculatorTests.cs index da5b91dd0ff..8ab51b1fe3f 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Reward/AuRaRewardCalculatorTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Reward/AuRaRewardCalculatorTests.cs @@ -10,6 +10,7 @@ using Nethermind.Consensus.AuRa.Rewards; using Nethermind.Consensus.Rewards; using Nethermind.Core; +using Nethermind.Core.Extensions; using Nethermind.Core.Test; using Nethermind.Specs.ChainSpecStyle; using Nethermind.Core.Test.Builders; @@ -213,7 +214,7 @@ private void SetupBlockRewards(IDictionary rewards) private bool CheckTransaction(Transaction t, ICollection
addresses, byte[] transactionData) => t.SenderAddress == Address.SystemUser && (t.To == _auraParameters.BlockRewardContractAddress || addresses.Contains(t.To)) - && t.Data == transactionData; + && t.Data.FasterToArray() == transactionData; private byte[] SetupAbiAddresses(params BlockReward[] rewards) { diff --git a/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs index 30d7123eeea..227d9ab749d 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs @@ -652,7 +652,7 @@ private byte[] SetupAbiAddresses(Address[] addresses) private bool CheckTransaction(Transaction t, (Address Sender, byte[] TransactionData) transactionInfo) { - return t.SenderAddress == transactionInfo.Sender && t.To == _contractAddress && t.Data == transactionInfo.TransactionData; + return t.SenderAddress == transactionInfo.Sender && t.To == _contractAddress && t.Data.FasterToArray() == transactionInfo.TransactionData; } public class ConsecutiveInitiateChangeTestParameters diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV3.cs b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV3.cs index c7ebc7634be..824ddc8de26 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV3.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV3.cs @@ -43,7 +43,7 @@ protected override object[] GetAllowedTxTypesParameters(Transaction tx, BlockHea return new object[] { - tx.SenderAddress, tx.To ?? Address.Zero, tx.Value, gasPrice, tx.Data ?? Array.Empty() + tx.SenderAddress, tx.To ?? Address.Zero, tx.Value, gasPrice, tx.Data.FasterToArray() ?? Array.Empty() }; } diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV4.cs b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV4.cs index 2cf64a2c396..44a9655cf38 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV4.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV4.cs @@ -4,6 +4,7 @@ using System; using Nethermind.Abi; using Nethermind.Core; +using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Int256; using Nethermind.Evm; @@ -40,7 +41,7 @@ protected override object[] GetAllowedTxTypesParameters(Transaction tx, BlockHea return new object[] { - tx.SenderAddress, tx.To ?? Address.Zero, tx.Value, tx.MaxFeePerGas, tx.MaxPriorityFeePerGas, tx.GasLimit, tx.Data ?? Array.Empty() + tx.SenderAddress, tx.To ?? Address.Zero, tx.Value, tx.MaxFeePerGas, tx.MaxPriorityFeePerGas, tx.GasLimit, tx.Data.FasterToArray() ?? Array.Empty() }; } diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TxPriorityContract.Destination.cs b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TxPriorityContract.Destination.cs index f8621384591..cec625b75cd 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TxPriorityContract.Destination.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TxPriorityContract.Destination.cs @@ -54,7 +54,7 @@ public static implicit operator DestinationTuple(Destination destination) => public static Destination GetTransactionKey(Transaction tx) { - byte[] fnSignature = tx.Data?.Length >= 4 ? AbiSignature.GetAddress(tx.Data) : FnSignatureEmpty; + byte[] fnSignature = tx.Data?.Length >= 4 ? AbiSignature.GetAddress(tx.Data.FasterToArray()) : FnSignatureEmpty; return new Destination(tx.To, fnSignature, UInt256.Zero); } diff --git a/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs b/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs index 5ca5be25127..b4720c62e4a 100644 --- a/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs @@ -99,7 +99,7 @@ public void Roundtrip((Transaction Tx, string Description) testCase) Transaction? decoded = _txDecoder.Decode(rlpStream); decoded!.SenderAddress = new EthereumEcdsa(TestBlockchainIds.ChainId, LimboLogs.Instance).RecoverAddress(decoded); decoded.Hash = decoded.CalculateHash(); - decoded.Should().BeEquivalentTo(testCase.Tx, testCase.Description); + decoded.EqualToTransaction(testCase.Tx); } [TestCaseSource(nameof(TestCaseSource))] @@ -114,7 +114,7 @@ public void Roundtrip_ValueDecoderContext((Transaction Tx, string Description) t Transaction? decoded = _txDecoder.Decode(ref decoderContext); decoded!.SenderAddress = new EthereumEcdsa(TestBlockchainIds.ChainId, LimboLogs.Instance).RecoverAddress(decoded); decoded.Hash = decoded.CalculateHash(); - decoded.Should().BeEquivalentTo(testCase.Tx, testCase.Description); + decoded.EqualToTransaction(testCase.Tx); } [TestCaseSource(nameof(YoloV3TestCases))] diff --git a/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs b/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs index 3b2d04f9e54..40d1115a970 100644 --- a/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs @@ -4,6 +4,7 @@ using System; using FluentAssertions; using System.Collections.Generic; +using Nethermind.Core.Extensions; using Nethermind.Int256; using Nethermind.Specs.ChainSpecStyle; using NUnit.Framework; @@ -42,4 +43,16 @@ public void Supports1559_returns_expected_results(int decodedFeeCap, bool expect Assert.That(transaction.Supports1559, Is.EqualTo(expectedSupports1559)); } } + + public static class TransactionTestExtensions + { + public static void EqualToTransaction(this Transaction subject, Transaction expectation) + { + subject.Should().BeEquivalentTo( + expectation, + o => o.ComparingByMembers().Excluding((tx) => tx.Data)); + + subject.Data.FasterToArray().Should().BeEquivalentTo(expectation.Data.FasterToArray()); + } + } } diff --git a/src/Nethermind/Nethermind.Core/Extensions/MemoryExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/MemoryExtensions.cs new file mode 100644 index 00000000000..643e60d8802 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Extensions/MemoryExtensions.cs @@ -0,0 +1,29 @@ +using System; +using System.Runtime.InteropServices; + +namespace Nethermind.Core.Extensions; + +public static class MemoryExtensions +{ + /// + /// Try to get the underlying array if possible to prevent copying. + /// + /// + /// + public static byte[]? FasterToArray(this in Memory? memory) + { + if (memory == null) return null; + + return memory.Value.FasterToArray(); + } + + public static byte[] FasterToArray(this in Memory memory) + { + if ( + MemoryMarshal.TryGetArray(memory, out ArraySegment segment) && + segment.Offset == 0 && segment.Count == segment.Array!.Length + ) return segment.Array; + + return memory.Span.ToArray(); + } +} diff --git a/src/Nethermind/Nethermind.Core/Transaction.cs b/src/Nethermind/Nethermind.Core/Transaction.cs index dabd6e78542..3c05bd139e3 100644 --- a/src/Nethermind/Nethermind.Core/Transaction.cs +++ b/src/Nethermind/Nethermind.Core/Transaction.cs @@ -37,7 +37,7 @@ public class Transaction public long GasLimit { get; set; } public Address? To { get; set; } public UInt256 Value { get; set; } - public byte[]? Data { get; set; } + public Memory? Data { get; set; } public Address? SenderAddress { get; set; } public Signature? Signature { get; set; } public bool IsSigned => Signature is not null; @@ -147,7 +147,7 @@ public string ToShortString() { string gasPriceString = Supports1559 ? $"maxPriorityFeePerGas: {MaxPriorityFeePerGas}, MaxFeePerGas: {MaxFeePerGas}" : $"gas price {GasPrice}"; - return $"[TX: hash {Hash} from {SenderAddress} to {To} with data {Data?.ToHexString()}, {gasPriceString} and limit {GasLimit}, nonce {Nonce}]"; + return $"[TX: hash {Hash} from {SenderAddress} to {To} with data {Data.FasterToArray()?.ToHexString()}, {gasPriceString} and limit {GasLimit}, nonce {Nonce}]"; } public string ToString(string indent) @@ -169,7 +169,7 @@ public string ToString(string indent) builder.AppendLine($"{indent}Gas Limit: {GasLimit}"); builder.AppendLine($"{indent}Nonce: {Nonce}"); builder.AppendLine($"{indent}Value: {Value}"); - builder.AppendLine($"{indent}Data: {(Data ?? Array.Empty()).ToHexString()}"); + builder.AppendLine($"{indent}Data: {(Data.FasterToArray() ?? Array.Empty()).ToHexString()}"); builder.AppendLine($"{indent}Signature: {(Signature?.Bytes ?? Array.Empty()).ToHexString()}"); builder.AppendLine($"{indent}V: {Signature?.V}"); builder.AppendLine($"{indent}ChainId: {Signature?.ChainId}"); diff --git a/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs b/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs index 28120bc5be1..4fd3fe4beca 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs @@ -46,7 +46,7 @@ public void On_failure_block_and_tx_fields_are_set() Assert.That(trace.TransactionHash, Is.EqualTo(tx.Hash), "tx hash"); Assert.That(trace.Action.Gas, Is.EqualTo((long)tx.GasLimit - 21000), "gas"); Assert.That(trace.Action.Value, Is.EqualTo(tx.Value), "value"); - Assert.That(trace.Action.Input, Is.EqualTo(tx.Data), "input"); + Assert.That(trace.Action.Input, Is.EqualTo(tx.Data.FasterToArray()), "input"); Assert.That(trace.Action.TraceAddress, Is.EqualTo(Array.Empty()), "trace address"); } diff --git a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs index 248f0855796..fc7459c9c56 100644 --- a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs +++ b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.IO; using System.Linq; using Nethermind.Core; @@ -39,9 +40,10 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec) long dataCost = 0; if (transaction.Data is not null) { + Span data = transaction.Data.Value.Span; for (int i = 0; i < transaction.DataLength; i++) { - dataCost += transaction.Data[i] == 0 ? GasCostOf.TxDataZero : txDataNonZeroGasCost; + dataCost += data[i] == 0 ? GasCostOf.TxDataZero : txDataNonZeroGasCost; } } diff --git a/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs index b5300031b50..66b3a3e5898 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs @@ -7,6 +7,7 @@ using FastEnumUtility; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Int256; namespace Nethermind.Evm.Tracing.ParityStyle @@ -255,7 +256,7 @@ public void MarkAsFailed(Address recipient, long gasSpent, byte[] output, string _trace.Action.From = _tx.SenderAddress; _trace.Action.To = _tx.To; _trace.Action.Value = _tx.Value; - _trace.Action.Input = _tx.Data; + _trace.Action.Input = _tx.Data.FasterToArray(); _trace.Action.Gas = _tx.GasLimit; _trace.Action.CallType = _tx.IsMessageCall ? "call" : "init"; _trace.Action.Error = error; diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs index 4610042986e..6a355c3c004 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs @@ -7,6 +7,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; +using static Nethermind.Core.Extensions.MemoryExtensions; using Nethermind.Crypto; using Nethermind.Evm.CodeAnalysis; using Nethermind.Evm.Tracing; @@ -148,8 +149,8 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra transaction.CalculateEffectiveGasPrice(spec.IsEip1559Enabled, block.BaseFeePerGas); long gasLimit = transaction.GasLimit; - byte[] machineCode = transaction.IsContractCreation ? transaction.Data : null; - byte[] data = transaction.IsMessageCall ? transaction.Data : Array.Empty(); + byte[] machineCode = transaction.IsContractCreation ? transaction.Data.FasterToArray() : null; + byte[] data = transaction.IsMessageCall ? transaction.Data.FasterToArray() : Array.Empty(); Address? caller = transaction.SenderAddress; if (_logger.IsTrace) _logger.Trace($"Executing tx {transaction.Hash}"); diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/CallTransactionModel.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/CallTransactionModel.cs index 4b993785ac3..18e6ae03e01 100644 --- a/src/Nethermind/Nethermind.Facade/Proxy/Models/CallTransactionModel.cs +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/CallTransactionModel.cs @@ -3,6 +3,7 @@ using Nethermind.Core; using Nethermind.Int256; +using static Nethermind.Core.Extensions.MemoryExtensions; namespace Nethermind.Facade.Proxy.Models { @@ -20,7 +21,7 @@ public static CallTransactionModel FromTransaction(Transaction transaction) { From = transaction.SenderAddress, To = transaction.To, - Data = transaction.Data, + Data = transaction.Data.FasterToArray(), Value = transaction.Value, Gas = (UInt256)transaction.GasLimit, GasPrice = transaction.GasPrice diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs index f18b7b4e9c8..39d5b9d19eb 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs @@ -5,6 +5,7 @@ using FluentAssertions; using Nethermind.Core; using Nethermind.Core.Eip2930; +using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; using Nethermind.Int256; using Nethermind.JsonRpc.Data; @@ -202,7 +203,8 @@ public void can_convert_from_Transaction_to_TransactionForRpc_and_back(TxType tx _transactionForRpc = new TransactionForRpc(_transaction); Transaction afterConversion = _transactionForRpc.ToTransaction(); - afterConversion.Should().BeEquivalentTo(_transaction); + afterConversion.Should().BeEquivalentTo(_transaction, option => option.ComparingByMembers().Excluding(tx => tx.Data)); + afterConversion.Data.FasterToArray().Should().BeEquivalentTo(_transaction.Data.FasterToArray()); } } } diff --git a/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs b/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs index 8d4ffbaa063..c59e0ffa632 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs @@ -27,7 +27,7 @@ public TransactionForRpc(Keccak? blockHash, long? blockNumber, int? txIndex, Tra Value = transaction.Value; GasPrice = transaction.GasPrice; Gas = transaction.GasLimit; - Input = Data = transaction.Data; + Input = Data = transaction.Data.FasterToArray(); if (transaction.Supports1559) { GasPrice = baseFee is not null @@ -139,6 +139,8 @@ public TransactionForRpc() public T ToTransaction(ulong? chainId = null) where T : Transaction, new() { + byte[]? data = Data ?? Input; + T tx = new() { GasLimit = Gas ?? 0, @@ -147,13 +149,18 @@ public TransactionForRpc() To = To, SenderAddress = From, Value = Value ?? 0, - Data = Data ?? Input, + Data = (Memory?)data, Type = Type, AccessList = TryGetAccessList(), ChainId = chainId, MaxFeePerDataGas = MaxFeePerDataGas, }; + if (data is null) + { + tx.Data = null; // Yes this is needed... really. Try a debugger. + } + if (tx.Supports1559) { tx.GasPrice = MaxPriorityFeePerGas ?? 0; diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Parity/ParityTransaction.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Parity/ParityTransaction.cs index c63c1b811c8..bd522f85eb4 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Parity/ParityTransaction.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Parity/ParityTransaction.cs @@ -3,6 +3,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Int256; using Nethermind.Evm; using Newtonsoft.Json; @@ -56,7 +57,7 @@ public ParityTransaction(Transaction transaction, byte[] raw, PublicKey publicKe GasPrice = transaction.GasPrice; Gas = transaction.GasLimit; Raw = raw; - Input = transaction.Data; + Input = transaction.Data.FasterToArray(); PublicKey = publicKey; ChainId = transaction.Signature.ChainId; R = transaction.Signature.R; diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs index ae0a6c32812..7aa66f29ddf 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs @@ -1168,6 +1168,7 @@ public async Task ExecutionPayloadV1_set_and_get_transactions_roundtrip() txsReceived.Should().BeEquivalentTo(txsSource, options => options .Excluding(t => t.ChainId) + .Excluding(t => t.Data) .Excluding(t => t.SenderAddress) .Excluding(t => t.Timestamp) ); diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/SerializerTester.cs b/src/Nethermind/Nethermind.Network.Test/P2P/SerializerTester.cs index 5973f1d84f7..1ff687963ac 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/SerializerTester.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/SerializerTester.cs @@ -2,8 +2,10 @@ // SPDX-License-Identifier: LGPL-3.0-only +using System; using DotNetty.Buffers; using FluentAssertions; +using Nethermind.Core.Extensions; using Nethermind.Network.P2P.Messages; using Nethermind.Serialization.Rlp; using NUnit.Framework; @@ -22,7 +24,11 @@ public static void TestZero(IZeroMessageSerializer serializer, T message, T deserialized = serializer.Deserialize(buffer); // RlpLength is calculated explicitly when serializing an object by Calculate method. It's null after deserialization. - deserialized.Should().BeEquivalentTo(message, options => options.Excluding(c => c.Name == "RlpLength")); + deserialized.Should().BeEquivalentTo(message, options => options + .Excluding(c => c.Name == "RlpLength") + .Using>((context => context.Subject.FasterToArray().Should().BeEquivalentTo(context.Expectation.FasterToArray()))) + .WhenTypeIs>() + ); Assert.That(buffer.ReadableBytes, Is.EqualTo(0), "readable bytes"); diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V62/TransactionsMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V62/TransactionsMessageSerializerTests.cs index 472fd6ec3b9..4dc46638f42 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V62/TransactionsMessageSerializerTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V62/TransactionsMessageSerializerTests.cs @@ -1,8 +1,11 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Linq; +using FluentAssertions; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; using Nethermind.Crypto; using Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages; diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V65/PooledTransactionsMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V65/PooledTransactionsMessageSerializerTests.cs index 52fa9a9ad4e..1030ad41b34 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V65/PooledTransactionsMessageSerializerTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V65/PooledTransactionsMessageSerializerTests.cs @@ -1,8 +1,11 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Linq; +using FluentAssertions; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; using Nethermind.Crypto; using Nethermind.Network.P2P.Subprotocols.Eth.V65.Messages; diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V66/BlockBodiesMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V66/BlockBodiesMessageSerializerTests.cs index 3f09fc3a476..6d1a002ef08 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V66/BlockBodiesMessageSerializerTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V66/BlockBodiesMessageSerializerTests.cs @@ -3,6 +3,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Test; using Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages; using Nethermind.Network.Test.P2P.Subprotocols.Eth.V62; using NUnit.Framework; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs index 9585681681d..56976928a57 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.IO; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -180,7 +179,7 @@ private void EncodeLegacyWithoutPayload(T item, RlpStream stream) stream.Encode(item.GasLimit); stream.Encode(item.To); stream.Encode(item.Value); - stream.Encode(item.Data); + stream.Encode(item.Data.FasterToArray()); } private void EncodeAccessListPayloadWithoutPayload(T item, RlpStream stream, RlpBehaviors rlpBehaviors) @@ -191,7 +190,7 @@ private void EncodeAccessListPayloadWithoutPayload(T item, RlpStream stream, Rlp stream.Encode(item.GasLimit); stream.Encode(item.To); stream.Encode(item.Value); - stream.Encode(item.Data); + stream.Encode(item.Data.FasterToArray()); _accessListDecoder.Encode(stream, item.AccessList, rlpBehaviors); } @@ -204,7 +203,7 @@ private void EncodeEip1559PayloadWithoutPayload(T item, RlpStream stream, RlpBeh stream.Encode(item.GasLimit); stream.Encode(item.To); stream.Encode(item.Value); - stream.Encode(item.Data); + stream.Encode(item.Data.FasterToArray()); _accessListDecoder.Encode(stream, item.AccessList, rlpBehaviors); } @@ -454,7 +453,7 @@ private int GetLegacyContentLength(T item) + Rlp.LengthOf(item.GasLimit) + Rlp.LengthOf(item.To) + Rlp.LengthOf(item.Value) - + Rlp.LengthOf(item.Data); + + Rlp.LengthOf(item.Data.FasterToArray()); } private int GetAccessListContentLength(T item) @@ -464,7 +463,7 @@ private int GetAccessListContentLength(T item) + Rlp.LengthOf(item.GasLimit) + Rlp.LengthOf(item.To) + Rlp.LengthOf(item.Value) - + Rlp.LengthOf(item.Data) + + Rlp.LengthOf(item.Data.FasterToArray()) + Rlp.LengthOf(item.ChainId ?? 0) + _accessListDecoder.GetLength(item.AccessList, RlpBehaviors.None); } @@ -477,7 +476,7 @@ private int GetEip1559ContentLength(T item) + Rlp.LengthOf(item.GasLimit) + Rlp.LengthOf(item.To) + Rlp.LengthOf(item.Value) - + Rlp.LengthOf(item.Data) + + Rlp.LengthOf(item.Data.FasterToArray()) + Rlp.LengthOf(item.ChainId ?? 0) + _accessListDecoder.GetLength(item.AccessList, RlpBehaviors.None); } From 7cd5ea636aa67fe54efcab064426ccea8c89930c Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 18 May 2023 21:11:01 +0800 Subject: [PATCH 10/23] Decode with memory --- .../Blocks/BlockStore.cs | 2 +- .../Encoding/TxDecoderTests.cs | 24 ++++++ .../Nethermind.Serialization.Rlp/Rlp.cs | 73 +++++++++++++++++++ .../RlpBehaviors.cs | 3 +- .../Nethermind.Serialization.Rlp/TxDecoder.cs | 31 ++++++-- 5 files changed, 126 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index 056d84f176f..cdcb2eaf61c 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -66,7 +66,7 @@ public ReceiptRecoveryBlock GetReceiptRecoveryBlock(Keccak blockHash) { IMemoryOwner memory = _blockDbAsSpan.GetOwnedMemory(blockHash.Bytes); - return _blockDecoder.DecodeToReceiptRecoveryBlock(memory, RlpBehaviors.None | RlpBehaviors.DisableLazyHash); + return _blockDecoder.DecodeToReceiptRecoveryBlock(memory, RlpBehaviors.DisableLazyHash | RlpBehaviors.SliceMemory); } public void Cache(Block block) diff --git a/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs b/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs index b4720c62e4a..8ee03a498d1 100644 --- a/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.InteropServices; using System.Threading.Tasks; using FluentAssertions; using FluentAssertions.Numeric; @@ -117,6 +118,29 @@ public void Roundtrip_ValueDecoderContext((Transaction Tx, string Description) t decoded.EqualToTransaction(testCase.Tx); } + + [TestCaseSource(nameof(TestCaseSource))] + public void Roundtrip_ValueDecoderContext_WithMemory((Transaction Tx, string Description) testCase) + { + RlpStream rlpStream = new(10000); + _txDecoder.Encode(rlpStream, testCase.Tx); + + Memory memory = rlpStream.Data; + Rlp.ValueDecoderContext decoderContext = new(memory); + rlpStream.Position = 0; + Transaction? decoded = _txDecoder.Decode(ref decoderContext, RlpBehaviors.SliceMemory); + decoded!.SenderAddress = new EthereumEcdsa(TestBlockchainIds.ChainId, LimboLogs.Instance).RecoverAddress(decoded); + decoded.Hash = decoded.CalculateHash(); + decoded.EqualToTransaction(testCase.Tx); + + if (decoded.Data.FasterToArray().Length > 0) + { + // Well RlpStream is backed by array. + MemoryMarshal.TryGetArray(decoded.Data.Value, out ArraySegment segment).Should().BeTrue(); + segment.Offset.Should().NotBe(0); // But its not the whole array. + } + } + [TestCaseSource(nameof(YoloV3TestCases))] public void Roundtrip_yolo_v3((string IncomingRlpHex, Keccak Hash) testCase) { diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs index 2de58271df0..a9224214dcf 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs @@ -544,6 +544,15 @@ public ValueDecoderContext(scoped in Span data) Position = 0; } + public ValueDecoderContext(Memory memory) + { + Memory = memory; + Data = memory.Span; + Position = 0; + } + + public Memory? Memory { get; } + public Span Data { get; } public bool IsEmpty => Data.IsEmpty; @@ -748,6 +757,13 @@ public Span Read(int length) return data; } + public Memory ReadMemory(int length) + { + Memory data = Memory.Value.Slice(Position, length); + Position += length; + return data; + } + public void Check(int nextCheck) { if (Position != nextCheck) @@ -1124,6 +1140,63 @@ public Span DecodeByteArraySpan(bool allowLeadingZeroBytes = true) throw new RlpException($"Unexpected prefix value of {prefix} when decoding a byte array."); } + public Memory? DecodeByteArrayMemory(bool allowLeadingZeroBytes = true) + { + if (Memory == null) + { + throw new RlpException("Rlp not backed by a Memory"); + } + + int prefix = ReadByte(); + if (!allowLeadingZeroBytes && prefix == 0) + { + throw new RlpException($"Non-canonical ulong (leading zero bytes) at position {Position}"); + } + + if (prefix < 128) + { + return Memory.Value.Slice(Position - 1, 1); + } + + if (prefix == 128) + { + return Array.Empty(); + } + + if (prefix <= 183) + { + int length = prefix - 128; + Memory buffer = ReadMemory(length); + Span asSpan = buffer.Span; + if (length == 1 && asSpan[0] < 128) + { + throw new RlpException($"Unexpected byte value {asSpan[0]}"); + } + + return buffer; + } + + if (prefix < 192) + { + int lengthOfLength = prefix - 183; + if (lengthOfLength > 4) + { + // strange but needed to pass tests - seems that spec gives int64 length and tests int32 length + throw new RlpException("Expected length of lenth less or equal 4"); + } + + int length = DeserializeLength(lengthOfLength); + if (length < 56) + { + throw new RlpException("Expected length greater or equal 56 and was {length}"); + } + + return ReadMemory(length); + } + + throw new RlpException($"Unexpected prefix value of {prefix} when decoding a byte array."); + } + public void SkipItem() { (int prefix, int content) = PeekPrefixAndContentLength(); diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs index 30afd4c82f5..962d3f0b745 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs @@ -18,7 +18,8 @@ public enum RlpBehaviors // when we're calculating tx hash or sending raw transaction we should skip this wrapping // with additional wrapping for typed transactions we're decoding Uint8Array([TransactionType, TransactionPayload] // without wrapping we're decoding (TransactionType || TransactionPayload) - DisableLazyHash = 64, + DisableLazyHash = 64, // Disable lazy hash calculation in transaction + SliceMemory = 128, // Slice Memory if available instead of copying span to an array. All = AllowExtraBytes | ForSealing | Storage | Eip658Receipts | AllowUnsigned | SkipTypedWrapping } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs index 56976928a57..b94436a722e 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs @@ -137,14 +137,21 @@ private void DecodeEip1559PayloadWithoutSig(T transaction, RlpStream rlpStream, transaction.AccessList = _accessListDecoder.Decode(rlpStream, rlpBehaviors); } - private void DecodeLegacyPayloadWithoutSig(T transaction, ref Rlp.ValueDecoderContext decoderContext) + private void DecodeLegacyPayloadWithoutSig(T transaction, ref Rlp.ValueDecoderContext decoderContext, RlpBehaviors rlpBehaviors) { transaction.Nonce = decoderContext.DecodeUInt256(allowLeadingZeroBytes: false); transaction.GasPrice = decoderContext.DecodeUInt256(allowLeadingZeroBytes: false); transaction.GasLimit = decoderContext.DecodeLong(allowLeadingZeroBytes: false); transaction.To = decoderContext.DecodeAddress(); transaction.Value = decoderContext.DecodeUInt256(allowLeadingZeroBytes: false); - transaction.Data = decoderContext.DecodeByteArray(); + if ((rlpBehaviors & RlpBehaviors.SliceMemory) != 0) + { + transaction.Data = decoderContext.DecodeByteArrayMemory(); + } + else + { + transaction.Data = decoderContext.DecodeByteArray(); + } } private void DecodeAccessListPayloadWithoutSig(T transaction, ref Rlp.ValueDecoderContext decoderContext, RlpBehaviors rlpBehaviors) @@ -155,7 +162,14 @@ private void DecodeAccessListPayloadWithoutSig(T transaction, ref Rlp.ValueDecod transaction.GasLimit = decoderContext.DecodeLong(allowLeadingZeroBytes: false); transaction.To = decoderContext.DecodeAddress(); transaction.Value = decoderContext.DecodeUInt256(allowLeadingZeroBytes: false); - transaction.Data = decoderContext.DecodeByteArray(); + if ((rlpBehaviors & RlpBehaviors.SliceMemory) != 0) + { + transaction.Data = decoderContext.DecodeByteArrayMemory(); + } + else + { + transaction.Data = decoderContext.DecodeByteArray(); + } transaction.AccessList = _accessListDecoder.Decode(ref decoderContext, rlpBehaviors); } @@ -168,7 +182,14 @@ private void DecodeEip1559PayloadWithoutSig(T transaction, ref Rlp.ValueDecoderC transaction.GasLimit = decoderContext.DecodeLong(allowLeadingZeroBytes: false); transaction.To = decoderContext.DecodeAddress(); transaction.Value = decoderContext.DecodeUInt256(allowLeadingZeroBytes: false); - transaction.Data = decoderContext.DecodeByteArray(); + if ((rlpBehaviors & RlpBehaviors.SliceMemory) != 0) + { + transaction.Data = decoderContext.DecodeByteArrayMemory(); + } + else + { + transaction.Data = decoderContext.DecodeByteArray(); + } transaction.AccessList = _accessListDecoder.Decode(ref decoderContext, rlpBehaviors); } @@ -248,7 +269,7 @@ private void EncodeEip1559PayloadWithoutPayload(T item, RlpStream stream, RlpBeh switch (transaction.Type) { case TxType.Legacy: - DecodeLegacyPayloadWithoutSig(transaction, ref decoderContext); + DecodeLegacyPayloadWithoutSig(transaction, ref decoderContext, rlpBehaviors); break; case TxType.AccessList: DecodeAccessListPayloadWithoutSig(transaction, ref decoderContext, rlpBehaviors); From 87cca06c3afa6efed629a88ec4d30261616b6126 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 19 May 2023 08:05:58 +0800 Subject: [PATCH 11/23] Simplifying code --- .../Blocks/BlockStore.cs | 2 +- .../Encoding/TxDecoderTests.cs | 23 ----------- .../BlockDecoder.cs | 2 +- .../ReceiptRecoveryBlock.cs | 2 +- .../Nethermind.Serialization.Rlp/Rlp.cs | 24 ++++++++++-- .../RlpBehaviors.cs | 1 - .../Nethermind.Serialization.Rlp/RlpStream.cs | 11 ++++++ .../Nethermind.Serialization.Rlp/TxDecoder.cs | 39 +++++-------------- 8 files changed, 43 insertions(+), 61 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index cdcb2eaf61c..7fbf65e0167 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -66,7 +66,7 @@ public ReceiptRecoveryBlock GetReceiptRecoveryBlock(Keccak blockHash) { IMemoryOwner memory = _blockDbAsSpan.GetOwnedMemory(blockHash.Bytes); - return _blockDecoder.DecodeToReceiptRecoveryBlock(memory, RlpBehaviors.DisableLazyHash | RlpBehaviors.SliceMemory); + return _blockDecoder.DecodeToReceiptRecoveryBlock(memory, RlpBehaviors.DisableLazyHash); } public void Cache(Block block) diff --git a/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs b/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs index 8ee03a498d1..ce5de9fd18f 100644 --- a/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs @@ -118,29 +118,6 @@ public void Roundtrip_ValueDecoderContext((Transaction Tx, string Description) t decoded.EqualToTransaction(testCase.Tx); } - - [TestCaseSource(nameof(TestCaseSource))] - public void Roundtrip_ValueDecoderContext_WithMemory((Transaction Tx, string Description) testCase) - { - RlpStream rlpStream = new(10000); - _txDecoder.Encode(rlpStream, testCase.Tx); - - Memory memory = rlpStream.Data; - Rlp.ValueDecoderContext decoderContext = new(memory); - rlpStream.Position = 0; - Transaction? decoded = _txDecoder.Decode(ref decoderContext, RlpBehaviors.SliceMemory); - decoded!.SenderAddress = new EthereumEcdsa(TestBlockchainIds.ChainId, LimboLogs.Instance).RecoverAddress(decoded); - decoded.Hash = decoded.CalculateHash(); - decoded.EqualToTransaction(testCase.Tx); - - if (decoded.Data.FasterToArray().Length > 0) - { - // Well RlpStream is backed by array. - MemoryMarshal.TryGetArray(decoded.Data.Value, out ArraySegment segment).Should().BeTrue(); - segment.Offset.Should().NotBe(0); // But its not the whole array. - } - } - [TestCaseSource(nameof(YoloV3TestCases))] public void Roundtrip_yolo_v3((string IncomingRlpHex, Keccak Hash) testCase) { diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs index 1301ea3b6b7..310e7b17c75 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs @@ -251,7 +251,7 @@ public void Encode(RlpStream stream, Block? item, RlpBehaviors rlpBehaviors = Rl public ReceiptRecoveryBlock DecodeToReceiptRecoveryBlock(IMemoryOwner memory, RlpBehaviors rlpBehaviors) { - Rlp.ValueDecoderContext decoderContext = new Rlp.ValueDecoderContext(memory.Memory.Span); + Rlp.ValueDecoderContext decoderContext = new Rlp.ValueDecoderContext(memory.Memory); if (decoderContext.IsNextItemNull()) { diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs index 6967637311f..36c19976824 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs @@ -48,7 +48,7 @@ public Transaction GetNextTransaction() return _transactions[_currentTransactionIndex++]; } - Rlp.ValueDecoderContext decoderContext = new(_transactionData.Span); + Rlp.ValueDecoderContext decoderContext = new(_transactionData, true); decoderContext.Position = _currentTransactionPosition; Transaction tx = TxDecoder.Instance.Decode(ref decoderContext, RlpBehaviors.AllowUnsigned | RlpBehaviors.DisableLazyHash); _currentTransactionPosition = decoderContext.Position; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs index a9224214dcf..6e7497b26c0 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs @@ -544,15 +544,21 @@ public ValueDecoderContext(scoped in Span data) Position = 0; } - public ValueDecoderContext(Memory memory) + public ValueDecoderContext(Memory memory, bool sliceMemory = false) { Memory = memory; Data = memory.Span; Position = 0; + + // Slice memory is turned off by default. Because if you are not careful and being explicit about it, + // you can end up with a memory leak. + _sliceMemory = sliceMemory; } public Memory? Memory { get; } + private bool _sliceMemory = false; + public Span Data { get; } public bool IsEmpty => Data.IsEmpty; @@ -757,7 +763,7 @@ public Span Read(int length) return data; } - public Memory ReadMemory(int length) + private Memory ReadSlicedMemory(int length) { Memory data = Memory.Value.Slice(Position, length); Position += length; @@ -1142,6 +1148,11 @@ public Span DecodeByteArraySpan(bool allowLeadingZeroBytes = true) public Memory? DecodeByteArrayMemory(bool allowLeadingZeroBytes = true) { + if (!_sliceMemory) + { + return DecodeByteArraySpan().ToArray(); + } + if (Memory == null) { throw new RlpException("Rlp not backed by a Memory"); @@ -1166,7 +1177,7 @@ public Span DecodeByteArraySpan(bool allowLeadingZeroBytes = true) if (prefix <= 183) { int length = prefix - 128; - Memory buffer = ReadMemory(length); + Memory buffer = ReadSlicedMemory(length); Span asSpan = buffer.Span; if (length == 1 && asSpan[0] < 128) { @@ -1191,7 +1202,7 @@ public Span DecodeByteArraySpan(bool allowLeadingZeroBytes = true) throw new RlpException("Expected length greater or equal 56 and was {length}"); } - return ReadMemory(length); + return ReadSlicedMemory(length); } throw new RlpException($"Unexpected prefix value of {prefix} when decoding a byte array."); @@ -1562,6 +1573,11 @@ public static int LengthOf(byte[]? array) return LengthOf(array.AsSpan()); } + public static int LengthOf(Memory? memory) + { + return LengthOf(memory.GetValueOrDefault().Span); + } + public static int LengthOf(IReadOnlyList array) { if (array.Count == 0) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs index 962d3f0b745..ecd415d77fc 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs @@ -19,7 +19,6 @@ public enum RlpBehaviors // with additional wrapping for typed transactions we're decoding Uint8Array([TransactionType, TransactionPayload] // without wrapping we're decoding (TransactionType || TransactionPayload) DisableLazyHash = 64, // Disable lazy hash calculation in transaction - SliceMemory = 128, // Slice Memory if available instead of copying span to an array. All = AllowExtraBytes | ForSealing | Storage | Eip658Receipts | AllowUnsigned | SkipTypedWrapping } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs index c77a311d255..c002067ad93 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs @@ -537,6 +537,17 @@ public void Encode(byte[] input) Encode(input.AsSpan()); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Encode(Memory? input) + { + if (input is null) + { + WriteByte(EmptyArrayByte); + return; + } + Encode(input.Value.Span); + } + public void Encode(Span input) { if (input.IsEmpty) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs index b94436a722e..68c077286ea 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs @@ -144,14 +144,7 @@ private void DecodeLegacyPayloadWithoutSig(T transaction, ref Rlp.ValueDecoderCo transaction.GasLimit = decoderContext.DecodeLong(allowLeadingZeroBytes: false); transaction.To = decoderContext.DecodeAddress(); transaction.Value = decoderContext.DecodeUInt256(allowLeadingZeroBytes: false); - if ((rlpBehaviors & RlpBehaviors.SliceMemory) != 0) - { - transaction.Data = decoderContext.DecodeByteArrayMemory(); - } - else - { - transaction.Data = decoderContext.DecodeByteArray(); - } + transaction.Data = decoderContext.DecodeByteArrayMemory(); } private void DecodeAccessListPayloadWithoutSig(T transaction, ref Rlp.ValueDecoderContext decoderContext, RlpBehaviors rlpBehaviors) @@ -162,14 +155,7 @@ private void DecodeAccessListPayloadWithoutSig(T transaction, ref Rlp.ValueDecod transaction.GasLimit = decoderContext.DecodeLong(allowLeadingZeroBytes: false); transaction.To = decoderContext.DecodeAddress(); transaction.Value = decoderContext.DecodeUInt256(allowLeadingZeroBytes: false); - if ((rlpBehaviors & RlpBehaviors.SliceMemory) != 0) - { - transaction.Data = decoderContext.DecodeByteArrayMemory(); - } - else - { - transaction.Data = decoderContext.DecodeByteArray(); - } + transaction.Data = decoderContext.DecodeByteArrayMemory(); transaction.AccessList = _accessListDecoder.Decode(ref decoderContext, rlpBehaviors); } @@ -182,14 +168,7 @@ private void DecodeEip1559PayloadWithoutSig(T transaction, ref Rlp.ValueDecoderC transaction.GasLimit = decoderContext.DecodeLong(allowLeadingZeroBytes: false); transaction.To = decoderContext.DecodeAddress(); transaction.Value = decoderContext.DecodeUInt256(allowLeadingZeroBytes: false); - if ((rlpBehaviors & RlpBehaviors.SliceMemory) != 0) - { - transaction.Data = decoderContext.DecodeByteArrayMemory(); - } - else - { - transaction.Data = decoderContext.DecodeByteArray(); - } + transaction.Data = decoderContext.DecodeByteArrayMemory(); transaction.AccessList = _accessListDecoder.Decode(ref decoderContext, rlpBehaviors); } @@ -200,7 +179,7 @@ private void EncodeLegacyWithoutPayload(T item, RlpStream stream) stream.Encode(item.GasLimit); stream.Encode(item.To); stream.Encode(item.Value); - stream.Encode(item.Data.FasterToArray()); + stream.Encode(item.Data); } private void EncodeAccessListPayloadWithoutPayload(T item, RlpStream stream, RlpBehaviors rlpBehaviors) @@ -211,7 +190,7 @@ private void EncodeAccessListPayloadWithoutPayload(T item, RlpStream stream, Rlp stream.Encode(item.GasLimit); stream.Encode(item.To); stream.Encode(item.Value); - stream.Encode(item.Data.FasterToArray()); + stream.Encode(item.Data); _accessListDecoder.Encode(stream, item.AccessList, rlpBehaviors); } @@ -224,7 +203,7 @@ private void EncodeEip1559PayloadWithoutPayload(T item, RlpStream stream, RlpBeh stream.Encode(item.GasLimit); stream.Encode(item.To); stream.Encode(item.Value); - stream.Encode(item.Data.FasterToArray()); + stream.Encode(item.Data); _accessListDecoder.Encode(stream, item.AccessList, rlpBehaviors); } @@ -474,7 +453,7 @@ private int GetLegacyContentLength(T item) + Rlp.LengthOf(item.GasLimit) + Rlp.LengthOf(item.To) + Rlp.LengthOf(item.Value) - + Rlp.LengthOf(item.Data.FasterToArray()); + + Rlp.LengthOf(item.Data); } private int GetAccessListContentLength(T item) @@ -484,7 +463,7 @@ private int GetAccessListContentLength(T item) + Rlp.LengthOf(item.GasLimit) + Rlp.LengthOf(item.To) + Rlp.LengthOf(item.Value) - + Rlp.LengthOf(item.Data.FasterToArray()) + + Rlp.LengthOf(item.Data) + Rlp.LengthOf(item.ChainId ?? 0) + _accessListDecoder.GetLength(item.AccessList, RlpBehaviors.None); } @@ -497,7 +476,7 @@ private int GetEip1559ContentLength(T item) + Rlp.LengthOf(item.GasLimit) + Rlp.LengthOf(item.To) + Rlp.LengthOf(item.Value) - + Rlp.LengthOf(item.Data.FasterToArray()) + + Rlp.LengthOf(item.Data) + Rlp.LengthOf(item.ChainId ?? 0) + _accessListDecoder.GetLength(item.AccessList, RlpBehaviors.None); } From 3fd0ace6f9a716b52aa8692ef737afd600e839dc Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 19 May 2023 08:13:02 +0800 Subject: [PATCH 12/23] Unit tests --- .../Nethermind.Core.Test/RlpTests.cs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/Nethermind/Nethermind.Core.Test/RlpTests.cs b/src/Nethermind/Nethermind.Core.Test/RlpTests.cs index 451979c827d..9810b2cfcf1 100644 --- a/src/Nethermind/Nethermind.Core.Test/RlpTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/RlpTests.cs @@ -3,6 +3,7 @@ using System; using System.Numerics; +using System.Runtime.InteropServices; using FluentAssertions; using Nethermind.Int256; using Nethermind.Serialization.Rlp; @@ -219,5 +220,32 @@ public void Long_and_big_integer_encoded_the_same(long value) Assert.That(rlpBigInt.Bytes, Is.EqualTo(rlpLong.Bytes)); } + + [TestCase(true)] + [TestCase(false)] + public void RlpContextWithSliceMemory_shouldNotCopyUnderlyingData(bool sliceValue) + { + byte[] randomBytes = new byte[100]; + Random.Shared.NextBytes(randomBytes); + + int requiredLength = Rlp.LengthOf(randomBytes) * 3; + RlpStream stream = new RlpStream(requiredLength); + stream.Encode(randomBytes); + stream.Encode(randomBytes); + stream.Encode(randomBytes); + + Memory memory = stream.Data; + Rlp.ValueDecoderContext context = new Rlp.ValueDecoderContext(memory, sliceValue); + + for (int i = 0; i < 3; i++) + { + Memory? slice = context.DecodeByteArrayMemory(); + slice.Should().NotBeNull(); + MemoryMarshal.TryGetArray(slice.Value, out ArraySegment segment); + + bool isACopy = (segment.Offset == 0 && segment.Count == slice.Value.Length); + isACopy.Should().NotBe(sliceValue); + } + } } } From 1c987a5f297d257e8bd96b277d94b2ab8e2fd26f Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 19 May 2023 10:34:03 +0800 Subject: [PATCH 13/23] Pool tx --- .../ReceiptRecoveryBlock.cs | 7 ++++-- .../Nethermind.Serialization.Rlp/TxDecoder.cs | 23 +++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs index 36c19976824..2221e769f19 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs @@ -41,6 +41,9 @@ public ReceiptRecoveryBlock(IMemoryOwner memoryOwner, BlockHeader header, public BlockHeader Header { get; } public int TransactionCount { get; } + // Use a buffer to avoid reallocation. Surprisingly significant. May produce incorrect transaction, but for recovery, it is correct. + private Transaction? _txBuffer; + public Transaction GetNextTransaction() { if (_transactions != null) @@ -50,10 +53,10 @@ public Transaction GetNextTransaction() Rlp.ValueDecoderContext decoderContext = new(_transactionData, true); decoderContext.Position = _currentTransactionPosition; - Transaction tx = TxDecoder.Instance.Decode(ref decoderContext, RlpBehaviors.AllowUnsigned | RlpBehaviors.DisableLazyHash); + TxDecoder.Instance.Decode(ref decoderContext, ref _txBuffer, RlpBehaviors.AllowUnsigned | RlpBehaviors.DisableLazyHash); _currentTransactionPosition = decoderContext.Position; - return tx; + return _txBuffer; } public Keccak? Hash => Header.Hash; // do not add setter here diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs index 68c077286ea..a95f3ef1448 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs @@ -212,16 +212,32 @@ private void EncodeEip1559PayloadWithoutPayload(T item, RlpStream stream, RlpBeh public T? Decode(ref Rlp.ValueDecoderContext decoderContext, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + + { + T transaction = null; + Decode(ref decoderContext, ref transaction, rlpBehaviors); + + return transaction; + } + + + public void Decode(ref Rlp.ValueDecoderContext decoderContext, ref T? transaction, + RlpBehaviors rlpBehaviors = RlpBehaviors.None) { if (decoderContext.IsNextItemNull()) { decoderContext.ReadByte(); - return null; + transaction = null; + return; } - Span transactionSequence = decoderContext.PeekNextItem(); + if (transaction == null) + { + transaction = new(); + } + transaction.Type = TxType.Legacy; - T transaction = new(); + Span transactionSequence = decoderContext.PeekNextItem(); if ((rlpBehaviors & RlpBehaviors.SkipTypedWrapping) == RlpBehaviors.SkipTypedWrapping) { byte firstByte = decoderContext.PeekByte(); @@ -280,7 +296,6 @@ private void EncodeEip1559PayloadWithoutPayload(T item, RlpStream stream, RlpBeh // Just calculate the Hash immediately as txn too large transaction.Hash = Keccak.Compute(transactionSequence); } - return transaction; } private static void DecodeSignature( From ed766b4e0d7738de40951ba50cb9329d410df243 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 19 May 2023 11:37:11 +0800 Subject: [PATCH 14/23] Use memory manager instead --- src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs | 3 ++- src/Nethermind/Nethermind.Db/IDbWithSpan.cs | 2 +- .../Nethermind.Serialization.Rlp/BlockDecoder.cs | 8 ++++---- .../Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs | 6 +++--- src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs | 6 ++++++ 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index 7fbf65e0167..f0cf6a64b57 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -3,6 +3,7 @@ using System; using System.Buffers; +using System.Runtime.InteropServices; using Nethermind.Core; using Nethermind.Core.Caching; using Nethermind.Core.Crypto; @@ -64,7 +65,7 @@ public void Delete(Keccak blockHash) public ReceiptRecoveryBlock GetReceiptRecoveryBlock(Keccak blockHash) { - IMemoryOwner memory = _blockDbAsSpan.GetOwnedMemory(blockHash.Bytes); + MemoryManager memory = _blockDbAsSpan.GetOwnedMemory(blockHash.Bytes); return _blockDecoder.DecodeToReceiptRecoveryBlock(memory, RlpBehaviors.DisableLazyHash); } diff --git a/src/Nethermind/Nethermind.Db/IDbWithSpan.cs b/src/Nethermind/Nethermind.Db/IDbWithSpan.cs index 716e01ba2b5..d095d18518f 100644 --- a/src/Nethermind/Nethermind.Db/IDbWithSpan.cs +++ b/src/Nethermind/Nethermind.Db/IDbWithSpan.cs @@ -17,7 +17,7 @@ public interface IDbWithSpan : IDb Span GetSpan(ReadOnlySpan key); void PutSpan(ReadOnlySpan key, ReadOnlySpan value); void DangerousReleaseMemory(in Span span); - IMemoryOwner GetOwnedMemory(ReadOnlySpan key) + MemoryManager GetOwnedMemory(ReadOnlySpan key) { Span span = GetSpan(key); return new DbSpanMemoryManager(this, span); diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs index 310e7b17c75..d370ecce25b 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs @@ -249,9 +249,9 @@ public void Encode(RlpStream stream, Block? item, RlpBehaviors rlpBehaviors = Rl } } - public ReceiptRecoveryBlock DecodeToReceiptRecoveryBlock(IMemoryOwner memory, RlpBehaviors rlpBehaviors) + public ReceiptRecoveryBlock DecodeToReceiptRecoveryBlock(MemoryManager memory, RlpBehaviors rlpBehaviors) { - Rlp.ValueDecoderContext decoderContext = new Rlp.ValueDecoderContext(memory.Memory); + Rlp.ValueDecoderContext decoderContext = new Rlp.ValueDecoderContext(memory.Memory, true); if (decoderContext.IsNextItemNull()) { @@ -266,8 +266,8 @@ public ReceiptRecoveryBlock DecodeToReceiptRecoveryBlock(IMemoryOwner memo int contentLength = decoderContext.ReadSequenceLength(); int transactionCount = decoderContext.PeekNumberOfItemsRemaining(decoderContext.Position + contentLength); - Memory transactionMemory = memory.Memory.Slice(decoderContext.Position, contentLength); - decoderContext.Position += contentLength; // Skip the whole tx + + Memory transactionMemory = decoderContext.ReadMemory(contentLength); decoderContext.SkipItem(); // Skip uncles diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs index 2221e769f19..69366386cb1 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs @@ -23,7 +23,7 @@ public ReceiptRecoveryBlock(Block block) TransactionCount = _transactions.Length; } - public ReceiptRecoveryBlock(IMemoryOwner memoryOwner, BlockHeader header, Memory transactionData, int transactionCount) + public ReceiptRecoveryBlock(MemoryManager memoryOwner, BlockHeader header, Memory transactionData, int transactionCount) { Header = header; _memoryOwner = memoryOwner; @@ -31,7 +31,7 @@ public ReceiptRecoveryBlock(IMemoryOwner memoryOwner, BlockHeader header, TransactionCount = transactionCount; } - private IMemoryOwner? _memoryOwner; + private MemoryManager? _memoryOwner; private Memory _transactionData { get; set; } private int _currentTransactionPosition = 0; @@ -64,6 +64,6 @@ public Transaction GetNextTransaction() public void Dispose() { - _memoryOwner?.Dispose(); + ((IMemoryOwner)_memoryOwner)?.Dispose(); } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs index 6e7497b26c0..41083edae4c 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs @@ -763,6 +763,12 @@ public Span Read(int length) return data; } + public Memory ReadMemory(int length) + { + if (_sliceMemory && Memory.HasValue) return ReadSlicedMemory(length); + return Read(length).ToArray(); + } + private Memory ReadSlicedMemory(int length) { Memory data = Memory.Value.Slice(Position, length); From f448d9b5b32147adb7620033a6b5dcc6c0fb0301 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 19 May 2023 16:03:50 +0800 Subject: [PATCH 15/23] Fix whitespace --- src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs | 2 +- src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs | 2 +- .../Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs | 2 +- src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs b/src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs index 298e4c75b7e..e1e7a80cd42 100644 --- a/src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs +++ b/src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs @@ -42,7 +42,7 @@ public void Test(TransactionTest test) Assert.That(decodedUnsigned.Value, Is.EqualTo(test.Value), "value"); Assert.That(decodedUnsigned.GasPrice, Is.EqualTo(test.GasPrice), "gasPrice"); Assert.That(decodedUnsigned.GasLimit, Is.EqualTo(test.StartGas), "gasLimit"); - Assert.That(decodedUnsigned.Data, Is.EqualTo(test.Data), "data"); + Assert.That(decodedUnsigned.Data.FasterToArray(), Is.EqualTo(test.Data), "data"); Assert.That(decodedUnsigned.To, Is.EqualTo(test.To), "to"); Assert.That(decodedUnsigned.Nonce, Is.EqualTo(test.Nonce), "nonce"); diff --git a/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs b/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs index 8558463e335..b8846a8df91 100644 --- a/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs +++ b/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs @@ -179,7 +179,7 @@ private void RunTest(TransactionTest test, IReleaseSpec spec) if (validTest != null) { Assert.That(transaction.Value, Is.EqualTo(validTest.Value), "value"); - Assert.That(transaction.Data, Is.EqualTo(validTest.Data), "data"); + Assert.That(transaction.Data.FasterToArray(), Is.EqualTo(validTest.Data), "data"); Assert.That(transaction.GasLimit, Is.EqualTo(validTest.GasLimit.ToInt64(null)), "gasLimit"); Assert.That(transaction.GasPrice, Is.EqualTo(validTest.GasPrice), "gasPrice"); Assert.That(transaction.Nonce, Is.EqualTo(validTest.Nonce), "nonce"); diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs index 80041df1172..d2cfeba665b 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsRecovery.cs @@ -18,7 +18,7 @@ ReceiptsRecoveryResult TryRecover(Block block, TxReceipt[] receipts, bool forceR IRecoveryContext CreateRecoveryContext(ReceiptRecoveryBlock block, bool forceRecoverSender = false); - public interface IRecoveryContext: IDisposable + public interface IRecoveryContext : IDisposable { void RecoverReceiptData(TxReceipt receipt); void RecoverReceiptData(ref TxReceiptStructRef receipt); diff --git a/src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs b/src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs index c3c074d5aec..3d6d0c646ae 100644 --- a/src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs +++ b/src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs @@ -9,7 +9,7 @@ namespace Nethermind.Core.Buffers; -public class DbSpanMemoryManager: MemoryManager +public class DbSpanMemoryManager : MemoryManager { private readonly IDbWithSpan _db; private readonly unsafe void* _ptr; From 9bdeb50264da7bb86bd6fd3b3f16a2681e692be8 Mon Sep 17 00:00:00 2001 From: benaadams Date: Fri, 19 May 2023 11:28:56 +0100 Subject: [PATCH 16/23] Tweaks --- .../Nethermind.Db/DbSpanMemoryManager.cs | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs b/src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs index 3d6d0c646ae..4411a5843a6 100644 --- a/src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs +++ b/src/Nethermind/Nethermind.Db/DbSpanMemoryManager.cs @@ -3,50 +3,65 @@ using System; using System.Buffers; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Nethermind.Db; namespace Nethermind.Core.Buffers; -public class DbSpanMemoryManager : MemoryManager +public unsafe sealed class DbSpanMemoryManager : MemoryManager { private readonly IDbWithSpan _db; - private readonly unsafe void* _ptr; + private void* _ptr; private readonly int _length; - public DbSpanMemoryManager(IDbWithSpan db, Span span) + public DbSpanMemoryManager(IDbWithSpan db, Span unmanagedSpan) { - unsafe - { - _db = db; - _ptr = Unsafe.AsPointer(ref span.GetPinnableReference()); - _length = span.Length; - } + _db = db; + _ptr = Unsafe.AsPointer(ref MemoryMarshal.GetReference(unmanagedSpan)); + _length = unmanagedSpan.Length; } protected override void Dispose(bool disposing) { - _db.DangerousReleaseMemory(GetSpan()); + if (_ptr != null) + { + _db.DangerousReleaseMemory(GetSpan()); + } + + _ptr = null; } public override Span GetSpan() { - unsafe + if (_ptr == null && _length > 0) { - return new Span(_ptr, _length); + ThrowDisposed(); } + + return new Span(_ptr, _length); } public override MemoryHandle Pin(int elementIndex = 0) { - unsafe + if (_ptr == null && _length > 0) { - return new MemoryHandle(_ptr); + ThrowDisposed(); } + + return new MemoryHandle(_ptr); } public override void Unpin() { } + + [DoesNotReturn] + [StackTraceHidden] + private static void ThrowDisposed() + { + throw new ObjectDisposedException(nameof(DbSpanMemoryManager)); + } } From 8e676fb3a279be189bac67b11f0edd661b4188fd Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 24 May 2023 07:38:20 +0800 Subject: [PATCH 17/23] Use constructor param for lazy hash --- .../Blocks/BlockStore.cs | 2 +- .../ReceiptRecoveryBlock.cs | 2 +- .../RlpBehaviors.cs | 1 - .../Nethermind.Serialization.Rlp/TxDecoder.cs | 19 +++++++++++++++++-- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index f0cf6a64b57..0da5db6044b 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -67,7 +67,7 @@ public ReceiptRecoveryBlock GetReceiptRecoveryBlock(Keccak blockHash) { MemoryManager memory = _blockDbAsSpan.GetOwnedMemory(blockHash.Bytes); - return _blockDecoder.DecodeToReceiptRecoveryBlock(memory, RlpBehaviors.DisableLazyHash); + return _blockDecoder.DecodeToReceiptRecoveryBlock(memory, RlpBehaviors.None); } public void Cache(Block block) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs index 69366386cb1..e92b4574ae1 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs @@ -53,7 +53,7 @@ public Transaction GetNextTransaction() Rlp.ValueDecoderContext decoderContext = new(_transactionData, true); decoderContext.Position = _currentTransactionPosition; - TxDecoder.Instance.Decode(ref decoderContext, ref _txBuffer, RlpBehaviors.AllowUnsigned | RlpBehaviors.DisableLazyHash); + TxDecoder.InstanceWithoutLazyHash.Decode(ref decoderContext, ref _txBuffer, RlpBehaviors.AllowUnsigned); _currentTransactionPosition = decoderContext.Position; return _txBuffer; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs index ecd415d77fc..fb0947e543a 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpBehaviors.cs @@ -18,7 +18,6 @@ public enum RlpBehaviors // when we're calculating tx hash or sending raw transaction we should skip this wrapping // with additional wrapping for typed transactions we're decoding Uint8Array([TransactionType, TransactionPayload] // without wrapping we're decoding (TransactionType || TransactionPayload) - DisableLazyHash = 64, // Disable lazy hash calculation in transaction All = AllowExtraBytes | ForSealing | Storage | Eip658Receipts | AllowUnsigned | SkipTypedWrapping } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs index a95f3ef1448..c017ffdd8ba 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoder.cs @@ -13,6 +13,15 @@ public class TxDecoder : TxDecoder, ITransactionSizeCalculator { public const int MaxDelayedHashTxnSize = 32768; public static TxDecoder Instance = new TxDecoder(); + public static TxDecoder InstanceWithoutLazyHash = new TxDecoder(false); + + public TxDecoder() : base(true) // Rlp will try to find empty constructor. + { + } + + public TxDecoder(bool lazyHash) : base(lazyHash) + { + } public int GetLength(Transaction tx) { @@ -28,6 +37,12 @@ public class TxDecoder : where T : Transaction, new() { private readonly AccessListDecoder _accessListDecoder = new(); + private bool _lazyHash; + + protected TxDecoder(bool lazyHash = true) + { + _lazyHash = lazyHash; + } public T? Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { @@ -88,7 +103,7 @@ public class TxDecoder : rlpStream.Check(lastCheck); } - if (transactionSequence.Length <= TxDecoder.MaxDelayedHashTxnSize && (rlpBehaviors & RlpBehaviors.DisableLazyHash) == 0) + if (transactionSequence.Length <= TxDecoder.MaxDelayedHashTxnSize && _lazyHash) { // Delay hash generation, as may be filtered as having too low gas etc transaction.SetPreHash(transactionSequence); @@ -286,7 +301,7 @@ public void Decode(ref Rlp.ValueDecoderContext decoderContext, ref T? transactio decoderContext.Check(lastCheck); } - if (transactionSequence.Length <= TxDecoder.MaxDelayedHashTxnSize && (rlpBehaviors & RlpBehaviors.DisableLazyHash) == 0) + if (transactionSequence.Length <= TxDecoder.MaxDelayedHashTxnSize && _lazyHash) { // Delay hash generation, as may be filtered as having too low gas etc transaction.SetPreHash(transactionSequence); From 7c52e5a1d8c0d2fdf65f652e1d439a53936fb72f Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 24 May 2023 07:40:22 +0800 Subject: [PATCH 18/23] Fix wrong function called --- src/Nethermind/Nethermind.Core/Transaction.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Transaction.cs b/src/Nethermind/Nethermind.Core/Transaction.cs index 3c05bd139e3..e6544bdef1f 100644 --- a/src/Nethermind/Nethermind.Core/Transaction.cs +++ b/src/Nethermind/Nethermind.Core/Transaction.cs @@ -66,10 +66,7 @@ public Keccak? Hash } set { - if (_preHash.Count != 0) - { - ClearPreHashInternal(); - } + ClearPreHash(); _hash = value; } } From fc7a7d3f2666f9bf10418809a87fb6df8949fd14 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 24 May 2023 07:50:45 +0800 Subject: [PATCH 19/23] In case its not a span --- .../Blocks/BlockStore.cs | 22 +++++++++++--- .../BlockDecoder.cs | 6 ++-- .../ReceiptRecoveryBlock.cs | 30 +++++++++---------- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index 0da5db6044b..8e810a20219 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -15,7 +15,7 @@ namespace Nethermind.Blockchain.Blocks; public class BlockStore : IBlockStore { private readonly IDb _blockDb; - private readonly IDbWithSpan _blockDbAsSpan; + private readonly IDbWithSpan? _blockDbAsSpan; private readonly BlockDecoder _blockDecoder = new(); private const int CacheSize = 64; @@ -25,7 +25,11 @@ private readonly LruCache public BlockStore(IDb blockDb) { _blockDb = blockDb; - _blockDbAsSpan = (IDbWithSpan)_blockDb; + + if (blockDb is IDbWithSpan blockDbAsSpan) + _blockDbAsSpan = blockDbAsSpan; + else + _blockDbAsSpan = null; } public void SetMetadata(byte[] key, byte[] value) @@ -65,9 +69,19 @@ public void Delete(Keccak blockHash) public ReceiptRecoveryBlock GetReceiptRecoveryBlock(Keccak blockHash) { - MemoryManager memory = _blockDbAsSpan.GetOwnedMemory(blockHash.Bytes); + MemoryManager? memoryOwner = null; + Memory memory; + if (_blockDbAsSpan != null) + { + memoryOwner = _blockDbAsSpan.GetOwnedMemory(blockHash.Bytes); + memory = memoryOwner.Memory; + } + else + { + memory = _blockDb.Get(blockHash.Bytes); + } - return _blockDecoder.DecodeToReceiptRecoveryBlock(memory, RlpBehaviors.None); + return _blockDecoder.DecodeToReceiptRecoveryBlock(memoryOwner, memory, RlpBehaviors.None); } public void Cache(Block block) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs index d370ecce25b..d1b9138d0eb 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs @@ -249,9 +249,9 @@ public void Encode(RlpStream stream, Block? item, RlpBehaviors rlpBehaviors = Rl } } - public ReceiptRecoveryBlock DecodeToReceiptRecoveryBlock(MemoryManager memory, RlpBehaviors rlpBehaviors) + public ReceiptRecoveryBlock DecodeToReceiptRecoveryBlock(MemoryManager? memoryManager, Memory memory, RlpBehaviors rlpBehaviors) { - Rlp.ValueDecoderContext decoderContext = new Rlp.ValueDecoderContext(memory.Memory, true); + Rlp.ValueDecoderContext decoderContext = new Rlp.ValueDecoderContext(memory, true); if (decoderContext.IsNextItemNull()) { @@ -280,7 +280,7 @@ public ReceiptRecoveryBlock DecodeToReceiptRecoveryBlock(MemoryManager mem decoderContext.Check(blockCheck); } - return new ReceiptRecoveryBlock(memory, header, transactionMemory, transactionCount); + return new ReceiptRecoveryBlock(memoryManager, header, transactionMemory, transactionCount); } } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs index e92b4574ae1..882b8de81d5 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs @@ -16,6 +16,19 @@ namespace Nethermind.Serialization.Rlp; [DebuggerDisplay("{Hash} ({Number})")] public class ReceiptRecoveryBlock { + private readonly MemoryManager? _memoryOwner; // Can be null if loaded without span + private readonly Memory _transactionData; + private int _currentTransactionPosition = 0; + + private readonly Transaction[]? _transactions = null; + private int _currentTransactionIndex = 0; + + public BlockHeader Header { get; } + public int TransactionCount { get; } + + // Use a buffer to avoid reallocation. Surprisingly significant. May produce incorrect transaction, but for recovery, it is correct. + private Transaction? _txBuffer; + public ReceiptRecoveryBlock(Block block) { Header = block.Header; @@ -23,7 +36,7 @@ public ReceiptRecoveryBlock(Block block) TransactionCount = _transactions.Length; } - public ReceiptRecoveryBlock(MemoryManager memoryOwner, BlockHeader header, Memory transactionData, int transactionCount) + public ReceiptRecoveryBlock(MemoryManager? memoryOwner, BlockHeader header, Memory transactionData, int transactionCount) { Header = header; _memoryOwner = memoryOwner; @@ -31,19 +44,6 @@ public ReceiptRecoveryBlock(MemoryManager memoryOwner, BlockHeader header, TransactionCount = transactionCount; } - private MemoryManager? _memoryOwner; - private Memory _transactionData { get; set; } - private int _currentTransactionPosition = 0; - - private Transaction[]? _transactions = null; - private int _currentTransactionIndex = 0; - - public BlockHeader Header { get; } - public int TransactionCount { get; } - - // Use a buffer to avoid reallocation. Surprisingly significant. May produce incorrect transaction, but for recovery, it is correct. - private Transaction? _txBuffer; - public Transaction GetNextTransaction() { if (_transactions != null) @@ -64,6 +64,6 @@ public Transaction GetNextTransaction() public void Dispose() { - ((IMemoryOwner)_memoryOwner)?.Dispose(); + ((IMemoryOwner?)_memoryOwner)?.Dispose(); } } From fb97c5e154dd6da4a9c3c3c6042702358598587c Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 24 May 2023 08:01:50 +0800 Subject: [PATCH 20/23] Use struct for the recovery block --- .../Blocks/BlockStoreTests.cs | 2 +- .../Nethermind.Blockchain/Blocks/BlockStore.cs | 2 +- .../Nethermind.Blockchain/Blocks/IBlockStore.cs | 2 +- .../Receipts/PersistentReceiptStorage.cs | 10 ++++++++-- .../Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs | 2 +- .../Nethermind.Serialization.Rlp/BlockDecoder.cs | 2 +- .../ReceiptRecoveryBlock.cs | 2 +- 7 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs index 501ba334dfa..a4bd92bf46c 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs @@ -77,7 +77,7 @@ public void Test_getReceiptRecoveryBlock_produce_same_transaction_as_normal_get( store.Insert(block); - ReceiptRecoveryBlock? retrieved = store.GetReceiptRecoveryBlock(block.Hash); + ReceiptRecoveryBlock retrieved = store.GetReceiptRecoveryBlock(block.Hash).Value; retrieved.Should().NotBeNull(); retrieved.Header.Should().BeEquivalentTo(block.Header); diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index 8e810a20219..ea26ad796aa 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -67,7 +67,7 @@ public void Delete(Keccak blockHash) return _blockDb.Get(blockHash, _blockDecoder, _blockCache, shouldCache); } - public ReceiptRecoveryBlock GetReceiptRecoveryBlock(Keccak blockHash) + public ReceiptRecoveryBlock? GetReceiptRecoveryBlock(Keccak blockHash) { MemoryManager? memoryOwner = null; Memory memory; diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs index 219c0adcbf8..a0d8a63299d 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs @@ -16,7 +16,7 @@ public interface IBlockStore void Insert(Block block); void Delete(Keccak blockHash); Block Get(Keccak blockHash, bool shouldCache = true); - ReceiptRecoveryBlock GetReceiptRecoveryBlock(Keccak blockHash); + ReceiptRecoveryBlock? GetReceiptRecoveryBlock(Keccak blockHash); void Cache(Block block); diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs index 7a5f07826bb..d882bd70552 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs @@ -241,8 +241,14 @@ public bool TryGetReceiptsIterator(long blockNumber, Keccak blockHash, out Recei { recoveryContextFactory = () => { - ReceiptRecoveryBlock block = _blockStore.GetReceiptRecoveryBlock(blockHash); - return _receiptsRecovery.CreateRecoveryContext(block); + ReceiptRecoveryBlock? block = _blockStore.GetReceiptRecoveryBlock(blockHash); + + if (!block.HasValue) + { + throw new InvalidOperationException($"Unable to recover receipts for block {blockHash} because of missing block data."); + } + + return _receiptsRecovery.CreateRecoveryContext(block.Value); }; } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs index 9119de757b9..69f9a5f37d8 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs @@ -73,7 +73,7 @@ public bool NeedRecover(TxReceipt[] receipts, bool forceRecoverSender = true, bo private class RecoveryContext : IReceiptsRecovery.IRecoveryContext { private readonly IReleaseSpec _releaseSpec; - private readonly ReceiptRecoveryBlock _block; + private ReceiptRecoveryBlock _block; private readonly bool _forceRecoverSender; private readonly IEthereumEcdsa _ecdsa; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs index d1b9138d0eb..25c95355e5c 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs @@ -249,7 +249,7 @@ public void Encode(RlpStream stream, Block? item, RlpBehaviors rlpBehaviors = Rl } } - public ReceiptRecoveryBlock DecodeToReceiptRecoveryBlock(MemoryManager? memoryManager, Memory memory, RlpBehaviors rlpBehaviors) + public ReceiptRecoveryBlock? DecodeToReceiptRecoveryBlock(MemoryManager? memoryManager, Memory memory, RlpBehaviors rlpBehaviors) { Rlp.ValueDecoderContext decoderContext = new Rlp.ValueDecoderContext(memory, true); diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs index 882b8de81d5..76d7ac991c8 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptRecoveryBlock.cs @@ -14,7 +14,7 @@ namespace Nethermind.Serialization.Rlp; /// Retain span from DB as memory and must be explicitly disposed. ///
[DebuggerDisplay("{Hash} ({Number})")] -public class ReceiptRecoveryBlock +public struct ReceiptRecoveryBlock { private readonly MemoryManager? _memoryOwner; // Can be null if loaded without span private readonly Memory _transactionData; From b4bb45613c98be480b9cf667afe1ae709cfbcad5 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 24 May 2023 08:23:33 +0800 Subject: [PATCH 21/23] Use Using and WhenTypeIs. --- src/Nethermind/Nethermind.Core.Test/TransactionTests.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs b/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs index 40d1115a970..12ef2e21d68 100644 --- a/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs @@ -50,9 +50,11 @@ public static void EqualToTransaction(this Transaction subject, Transaction expe { subject.Should().BeEquivalentTo( expectation, - o => o.ComparingByMembers().Excluding((tx) => tx.Data)); - - subject.Data.FasterToArray().Should().BeEquivalentTo(expectation.Data.FasterToArray()); + o => o + .ComparingByMembers() + .Using>(ctx => ctx.Subject.FasterToArray().Should().BeEquivalentTo(ctx.Expectation.FasterToArray())) + .WhenTypeIs>() + ); } } } From 6b413433a7065afae0540c0430b51071bd419114 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 25 May 2023 07:46:06 +0800 Subject: [PATCH 22/23] To AsArray --- .../Contract/ReportingValidatorContractTests.cs | 4 ++-- .../Reward/AuRaRewardCalculatorTests.cs | 2 +- .../Validators/ContractBasedValidatorTests.cs | 2 +- .../Contracts/TransactionPermissionContractV3.cs | 2 +- .../Contracts/TransactionPermissionContractV4.cs | 2 +- .../Contracts/TxPriorityContract.Destination.cs | 2 +- src/Nethermind/Nethermind.Core.Test/TransactionTests.cs | 2 +- .../Nethermind.Core/Extensions/MemoryExtensions.cs | 6 +++--- src/Nethermind/Nethermind.Core/Transaction.cs | 4 ++-- .../Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs | 2 +- .../Tracing/ParityStyle/ParityLikeTxTracer.cs | 2 +- .../TransactionProcessing/TransactionProcessor.cs | 4 ++-- .../Nethermind.Facade/Proxy/Models/CallTransactionModel.cs | 2 +- src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs | 2 +- src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs | 2 +- .../Nethermind.JsonRpc/Modules/Parity/ParityTransaction.cs | 2 +- .../Nethermind.Network.Test/P2P/SerializerTester.cs | 2 +- 17 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/Nethermind/Nethermind.AuRa.Test/Contract/ReportingValidatorContractTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Contract/ReportingValidatorContractTests.cs index af3dd07c56c..b24804531f2 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Contract/ReportingValidatorContractTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Contract/ReportingValidatorContractTests.cs @@ -19,7 +19,7 @@ public void Should_generate_malicious_transaction() { ReportingValidatorContract contract = new(AbiEncoder.Instance, new Address("0x1000000000000000000000000000000000000001"), Substitute.For()); Transaction transaction = contract.ReportMalicious(new Address("0x75df42383afe6bf5194aa8fa0e9b3d5f9e869441"), 10, new byte[0]); - transaction.Data.FasterToArray().ToHexString().Should().Be("c476dd4000000000000000000000000075df42383afe6bf5194aa8fa0e9b3d5f9e869441000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000000"); + transaction.Data.AsArray().ToHexString().Should().Be("c476dd4000000000000000000000000075df42383afe6bf5194aa8fa0e9b3d5f9e869441000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000000"); } [Test] @@ -27,7 +27,7 @@ public void Should_generate_benign_transaction() { ReportingValidatorContract contract = new(AbiEncoder.Instance, new Address("0x1000000000000000000000000000000000000001"), Substitute.For()); Transaction transaction = contract.ReportBenign(new Address("0x75df42383afe6bf5194aa8fa0e9b3d5f9e869441"), 10); - transaction.Data.FasterToArray().ToHexString().Should().Be("d69f13bb00000000000000000000000075df42383afe6bf5194aa8fa0e9b3d5f9e869441000000000000000000000000000000000000000000000000000000000000000a"); + transaction.Data.AsArray().ToHexString().Should().Be("d69f13bb00000000000000000000000075df42383afe6bf5194aa8fa0e9b3d5f9e869441000000000000000000000000000000000000000000000000000000000000000a"); } } } diff --git a/src/Nethermind/Nethermind.AuRa.Test/Reward/AuRaRewardCalculatorTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Reward/AuRaRewardCalculatorTests.cs index 8ab51b1fe3f..7717e75d0e6 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Reward/AuRaRewardCalculatorTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Reward/AuRaRewardCalculatorTests.cs @@ -214,7 +214,7 @@ private void SetupBlockRewards(IDictionary rewards) private bool CheckTransaction(Transaction t, ICollection
addresses, byte[] transactionData) => t.SenderAddress == Address.SystemUser && (t.To == _auraParameters.BlockRewardContractAddress || addresses.Contains(t.To)) - && t.Data.FasterToArray() == transactionData; + && t.Data.AsArray() == transactionData; private byte[] SetupAbiAddresses(params BlockReward[] rewards) { diff --git a/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs index 227d9ab749d..5511d9db936 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs @@ -652,7 +652,7 @@ private byte[] SetupAbiAddresses(Address[] addresses) private bool CheckTransaction(Transaction t, (Address Sender, byte[] TransactionData) transactionInfo) { - return t.SenderAddress == transactionInfo.Sender && t.To == _contractAddress && t.Data.FasterToArray() == transactionInfo.TransactionData; + return t.SenderAddress == transactionInfo.Sender && t.To == _contractAddress && t.Data.AsArray() == transactionInfo.TransactionData; } public class ConsecutiveInitiateChangeTestParameters diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV3.cs b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV3.cs index 824ddc8de26..99c955616fa 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV3.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV3.cs @@ -43,7 +43,7 @@ protected override object[] GetAllowedTxTypesParameters(Transaction tx, BlockHea return new object[] { - tx.SenderAddress, tx.To ?? Address.Zero, tx.Value, gasPrice, tx.Data.FasterToArray() ?? Array.Empty() + tx.SenderAddress, tx.To ?? Address.Zero, tx.Value, gasPrice, tx.Data.AsArray() ?? Array.Empty() }; } diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV4.cs b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV4.cs index 44a9655cf38..33561997d60 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV4.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContractV4.cs @@ -41,7 +41,7 @@ protected override object[] GetAllowedTxTypesParameters(Transaction tx, BlockHea return new object[] { - tx.SenderAddress, tx.To ?? Address.Zero, tx.Value, tx.MaxFeePerGas, tx.MaxPriorityFeePerGas, tx.GasLimit, tx.Data.FasterToArray() ?? Array.Empty() + tx.SenderAddress, tx.To ?? Address.Zero, tx.Value, tx.MaxFeePerGas, tx.MaxPriorityFeePerGas, tx.GasLimit, tx.Data.AsArray() ?? Array.Empty() }; } diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TxPriorityContract.Destination.cs b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TxPriorityContract.Destination.cs index cec625b75cd..099fb2c0b91 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TxPriorityContract.Destination.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TxPriorityContract.Destination.cs @@ -54,7 +54,7 @@ public static implicit operator DestinationTuple(Destination destination) => public static Destination GetTransactionKey(Transaction tx) { - byte[] fnSignature = tx.Data?.Length >= 4 ? AbiSignature.GetAddress(tx.Data.FasterToArray()) : FnSignatureEmpty; + byte[] fnSignature = tx.Data?.Length >= 4 ? AbiSignature.GetAddress(tx.Data.AsArray()) : FnSignatureEmpty; return new Destination(tx.To, fnSignature, UInt256.Zero); } diff --git a/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs b/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs index 12ef2e21d68..05c5eb706a2 100644 --- a/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/TransactionTests.cs @@ -52,7 +52,7 @@ public static void EqualToTransaction(this Transaction subject, Transaction expe expectation, o => o .ComparingByMembers() - .Using>(ctx => ctx.Subject.FasterToArray().Should().BeEquivalentTo(ctx.Expectation.FasterToArray())) + .Using>(ctx => ctx.Subject.AsArray().Should().BeEquivalentTo(ctx.Expectation.AsArray())) .WhenTypeIs>() ); } diff --git a/src/Nethermind/Nethermind.Core/Extensions/MemoryExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/MemoryExtensions.cs index 643e60d8802..9d6d83799b0 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/MemoryExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/MemoryExtensions.cs @@ -10,14 +10,14 @@ public static class MemoryExtensions /// /// /// - public static byte[]? FasterToArray(this in Memory? memory) + public static byte[]? AsArray(this in Memory? memory) { if (memory == null) return null; - return memory.Value.FasterToArray(); + return memory.Value.AsArray(); } - public static byte[] FasterToArray(this in Memory memory) + public static byte[] AsArray(this in Memory memory) { if ( MemoryMarshal.TryGetArray(memory, out ArraySegment segment) && diff --git a/src/Nethermind/Nethermind.Core/Transaction.cs b/src/Nethermind/Nethermind.Core/Transaction.cs index e6544bdef1f..d9bbad18758 100644 --- a/src/Nethermind/Nethermind.Core/Transaction.cs +++ b/src/Nethermind/Nethermind.Core/Transaction.cs @@ -144,7 +144,7 @@ public string ToShortString() { string gasPriceString = Supports1559 ? $"maxPriorityFeePerGas: {MaxPriorityFeePerGas}, MaxFeePerGas: {MaxFeePerGas}" : $"gas price {GasPrice}"; - return $"[TX: hash {Hash} from {SenderAddress} to {To} with data {Data.FasterToArray()?.ToHexString()}, {gasPriceString} and limit {GasLimit}, nonce {Nonce}]"; + return $"[TX: hash {Hash} from {SenderAddress} to {To} with data {Data.AsArray()?.ToHexString()}, {gasPriceString} and limit {GasLimit}, nonce {Nonce}]"; } public string ToString(string indent) @@ -166,7 +166,7 @@ public string ToString(string indent) builder.AppendLine($"{indent}Gas Limit: {GasLimit}"); builder.AppendLine($"{indent}Nonce: {Nonce}"); builder.AppendLine($"{indent}Value: {Value}"); - builder.AppendLine($"{indent}Data: {(Data.FasterToArray() ?? Array.Empty()).ToHexString()}"); + builder.AppendLine($"{indent}Data: {(Data.AsArray() ?? Array.Empty()).ToHexString()}"); builder.AppendLine($"{indent}Signature: {(Signature?.Bytes ?? Array.Empty()).ToHexString()}"); builder.AppendLine($"{indent}V: {Signature?.V}"); builder.AppendLine($"{indent}ChainId: {Signature?.ChainId}"); diff --git a/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs b/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs index 4fd3fe4beca..c4cc04974c4 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs @@ -46,7 +46,7 @@ public void On_failure_block_and_tx_fields_are_set() Assert.That(trace.TransactionHash, Is.EqualTo(tx.Hash), "tx hash"); Assert.That(trace.Action.Gas, Is.EqualTo((long)tx.GasLimit - 21000), "gas"); Assert.That(trace.Action.Value, Is.EqualTo(tx.Value), "value"); - Assert.That(trace.Action.Input, Is.EqualTo(tx.Data.FasterToArray()), "input"); + Assert.That(trace.Action.Input, Is.EqualTo(tx.Data.AsArray()), "input"); Assert.That(trace.Action.TraceAddress, Is.EqualTo(Array.Empty()), "trace address"); } diff --git a/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs index 66b3a3e5898..c24f907be56 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs @@ -256,7 +256,7 @@ public void MarkAsFailed(Address recipient, long gasSpent, byte[] output, string _trace.Action.From = _tx.SenderAddress; _trace.Action.To = _tx.To; _trace.Action.Value = _tx.Value; - _trace.Action.Input = _tx.Data.FasterToArray(); + _trace.Action.Input = _tx.Data.AsArray(); _trace.Action.Gas = _tx.GasLimit; _trace.Action.CallType = _tx.IsMessageCall ? "call" : "init"; _trace.Action.Error = error; diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs index 6a355c3c004..056750345ce 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs @@ -149,8 +149,8 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra transaction.CalculateEffectiveGasPrice(spec.IsEip1559Enabled, block.BaseFeePerGas); long gasLimit = transaction.GasLimit; - byte[] machineCode = transaction.IsContractCreation ? transaction.Data.FasterToArray() : null; - byte[] data = transaction.IsMessageCall ? transaction.Data.FasterToArray() : Array.Empty(); + byte[] machineCode = transaction.IsContractCreation ? transaction.Data.AsArray() : null; + byte[] data = transaction.IsMessageCall ? transaction.Data.AsArray() : Array.Empty(); Address? caller = transaction.SenderAddress; if (_logger.IsTrace) _logger.Trace($"Executing tx {transaction.Hash}"); diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/CallTransactionModel.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/CallTransactionModel.cs index 18e6ae03e01..697b11eb3d7 100644 --- a/src/Nethermind/Nethermind.Facade/Proxy/Models/CallTransactionModel.cs +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/CallTransactionModel.cs @@ -21,7 +21,7 @@ public static CallTransactionModel FromTransaction(Transaction transaction) { From = transaction.SenderAddress, To = transaction.To, - Data = transaction.Data.FasterToArray(), + Data = transaction.Data.AsArray(), Value = transaction.Value, Gas = (UInt256)transaction.GasLimit, GasPrice = transaction.GasPrice diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs index 39d5b9d19eb..1981a60accf 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs @@ -204,7 +204,7 @@ public void can_convert_from_Transaction_to_TransactionForRpc_and_back(TxType tx Transaction afterConversion = _transactionForRpc.ToTransaction(); afterConversion.Should().BeEquivalentTo(_transaction, option => option.ComparingByMembers().Excluding(tx => tx.Data)); - afterConversion.Data.FasterToArray().Should().BeEquivalentTo(_transaction.Data.FasterToArray()); + afterConversion.Data.AsArray().Should().BeEquivalentTo(_transaction.Data.AsArray()); } } } diff --git a/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs b/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs index c59e0ffa632..bca581c00e0 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs @@ -27,7 +27,7 @@ public TransactionForRpc(Keccak? blockHash, long? blockNumber, int? txIndex, Tra Value = transaction.Value; GasPrice = transaction.GasPrice; Gas = transaction.GasLimit; - Input = Data = transaction.Data.FasterToArray(); + Input = Data = transaction.Data.AsArray(); if (transaction.Supports1559) { GasPrice = baseFee is not null diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Parity/ParityTransaction.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Parity/ParityTransaction.cs index bd522f85eb4..0abbe5f5959 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Parity/ParityTransaction.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Parity/ParityTransaction.cs @@ -57,7 +57,7 @@ public ParityTransaction(Transaction transaction, byte[] raw, PublicKey publicKe GasPrice = transaction.GasPrice; Gas = transaction.GasLimit; Raw = raw; - Input = transaction.Data.FasterToArray(); + Input = transaction.Data.AsArray(); PublicKey = publicKey; ChainId = transaction.Signature.ChainId; R = transaction.Signature.R; diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/SerializerTester.cs b/src/Nethermind/Nethermind.Network.Test/P2P/SerializerTester.cs index 1ff687963ac..efb907cb7bc 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/SerializerTester.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/SerializerTester.cs @@ -26,7 +26,7 @@ public static void TestZero(IZeroMessageSerializer serializer, T message, // RlpLength is calculated explicitly when serializing an object by Calculate method. It's null after deserialization. deserialized.Should().BeEquivalentTo(message, options => options .Excluding(c => c.Name == "RlpLength") - .Using>((context => context.Subject.FasterToArray().Should().BeEquivalentTo(context.Expectation.FasterToArray()))) + .Using>((context => context.Subject.AsArray().Should().BeEquivalentTo(context.Expectation.AsArray()))) .WhenTypeIs>() ); From c7823a6a1e347b2c4e2a6ea78ca01def195850bd Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 25 May 2023 11:21:37 +0800 Subject: [PATCH 23/23] Fix build --- src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs | 2 +- src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs b/src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs index e1e7a80cd42..3d4fdb9fa1f 100644 --- a/src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs +++ b/src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs @@ -42,7 +42,7 @@ public void Test(TransactionTest test) Assert.That(decodedUnsigned.Value, Is.EqualTo(test.Value), "value"); Assert.That(decodedUnsigned.GasPrice, Is.EqualTo(test.GasPrice), "gasPrice"); Assert.That(decodedUnsigned.GasLimit, Is.EqualTo(test.StartGas), "gasLimit"); - Assert.That(decodedUnsigned.Data.FasterToArray(), Is.EqualTo(test.Data), "data"); + Assert.That(decodedUnsigned.Data.AsArray(), Is.EqualTo(test.Data), "data"); Assert.That(decodedUnsigned.To, Is.EqualTo(test.To), "to"); Assert.That(decodedUnsigned.Nonce, Is.EqualTo(test.Nonce), "nonce"); diff --git a/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs b/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs index b8846a8df91..dc872261d15 100644 --- a/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs +++ b/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs @@ -179,7 +179,7 @@ private void RunTest(TransactionTest test, IReleaseSpec spec) if (validTest != null) { Assert.That(transaction.Value, Is.EqualTo(validTest.Value), "value"); - Assert.That(transaction.Data.FasterToArray(), Is.EqualTo(validTest.Data), "data"); + Assert.That(transaction.Data.AsArray(), Is.EqualTo(validTest.Data), "data"); Assert.That(transaction.GasLimit, Is.EqualTo(validTest.GasLimit.ToInt64(null)), "gasLimit"); Assert.That(transaction.GasPrice, Is.EqualTo(validTest.GasPrice), "gasPrice"); Assert.That(transaction.Nonce, Is.EqualTo(validTest.Nonce), "nonce");