From d3a3232c69c9a6f667d288cb13fc42de804503ad Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 12:56:12 -0300 Subject: [PATCH 001/106] Initial Rlp + Writer --- .../Instances/IRlpConverter.cs | 9 + .../Instances/IntRlpConverter.cs | 29 ++++ .../Instances/ReadOnlySpanConverter.cs | 44 +++++ .../Instances/StringRlpConverter.cs | 20 +++ .../Nethermind.Serialization.Rlp.Test.csproj | 24 +++ .../Nethermind.Serialization.Rlp.Test/Rlp.cs | 18 ++ .../RlpWriter.cs | 80 +++++++++ .../RlpWriterTest.cs | 155 ++++++++++++++++++ src/Nethermind/Nethermind.sln | 7 + 9 files changed, 386 insertions(+) create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanConverter.cs create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs new file mode 100644 index 00000000000..934e0cb7bd9 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Serialization.Rlp.Test.Instances; + +public interface IRlpConverter +{ + public static abstract void Write(IRlpWriter writer, T value); +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs new file mode 100644 index 00000000000..24542278dc9 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Buffers.Binary; + +namespace Nethermind.Serialization.Rlp.Test.Instances; + +public abstract class IntRlpConverter : IRlpConverter +{ + public static void Write(IRlpWriter writer, int value) + { + if (value < 0x80) + { + writer.WriteByte((byte)value); + } + else + { + Span bytes = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32BigEndian(bytes, value); + bytes = bytes.TrimStart((byte)0); + writer.Write(bytes); + } + } +} + +public static class IntRlpConverterExt +{ + public static void Write(this IRlpWriter writer, int value) => IntRlpConverter.Write(writer, value); +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanConverter.cs new file mode 100644 index 00000000000..523ac750ce4 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanConverter.cs @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Buffers.Binary; + +namespace Nethermind.Serialization.Rlp.Test.Instances; + +// Spiritually implements `IRlpConverter` but due to restrictions on `ReadOnlySpan` we cannot make it explicit +public abstract class ReadOnlySpanConverter /* : IRlpConverter> */ +{ + public static void Write(IRlpWriter writer, ReadOnlySpan value) + { + // "If a string is 0-55 bytes long, the RLP encoding consists of + // - a single byte with value 0x80 (dec. 128) + // - plus the length of the string + // - followed by the string. + // The range of the first byte is thus [0x80, 0xb7] (dec. [128, 183])". + if (value.Length < 55) + { + writer.WriteByte((byte)(0x80 + value.Length)); + } + // If a string is more than 55 bytes long, the RLP encoding consists of + // - a single byte with value 0xb7 (dec. 183) + // - plus the length in bytes of the length of the string in binary form, + // - followed by the length of the string, + // - followed by the string. + // The range of the first byte is thus [0xb8, 0xbf] (dec. [184, 191]). + else + { + Span binaryLength = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32BigEndian(binaryLength, value.Length); + binaryLength = binaryLength.TrimStart((byte)0); + writer.WriteByte((byte)(0xB7 + binaryLength.Length)); + writer.WriteBytes(binaryLength); + } + + writer.WriteBytes(value); + } +} + +public static class ReadOnlySpanConverterExt +{ + public static void Write(this IRlpWriter writer, ReadOnlySpan value) => ReadOnlySpanConverter.Write(writer, value); +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs new file mode 100644 index 00000000000..5364a82bf03 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text; + +namespace Nethermind.Serialization.Rlp.Test.Instances; + +public abstract class StringRlpConverter : IRlpConverter +{ + public static void Write(IRlpWriter writer, string value) + { + ReadOnlySpan bytes = Encoding.UTF8.GetBytes(value); + writer.Write(bytes); + } +} + +public static class StringRlpConverterExt +{ + public static void Write(this IRlpWriter writer, string value) => StringRlpConverter.Write(writer, value); +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj new file mode 100644 index 00000000000..a15287bde91 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj @@ -0,0 +1,24 @@ + + + + net9.0 + preview + enable + enable + false + + + + + + + + + + + + + + + + diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs new file mode 100644 index 00000000000..61d1fecd6f8 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Serialization.Rlp.Test; + +public static class Rlp +{ + public static byte[] Write(Action action) + { + var lengthWriter = new RlpLengthWriter(); + action(lengthWriter); + var serialized = new byte[lengthWriter.Length]; + var contentWriter = new RlpContentWriter(serialized); + action(contentWriter); + + return serialized; + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs new file mode 100644 index 00000000000..716640271b6 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Serialization.Rlp.Test; + +public interface IRlpWriter +{ + void WriteByte(byte value); + void WriteBytes(scoped ReadOnlySpan value); + void WriteList(Action action); +} + +public sealed class RlpContentWriter : IRlpWriter +{ + private readonly byte[] _buffer; + private int _position; + + public RlpContentWriter(byte[] buffer) + { + _buffer = buffer; + _position = 0; + } + + public void WriteByte(byte value) + { + _buffer[_position++] = value; + } + + public void WriteBytes(ReadOnlySpan value) + { + value.CopyTo(_buffer.AsSpan()[_position..]); + _position += value.Length; + } + + public void WriteList(Action action) + { + var lengthWriter = new RlpLengthWriter(); + action(lengthWriter); + if (lengthWriter.Length < 55) + { + _buffer[_position++] = (byte)(0xC0 + lengthWriter.Length); + } + else + { + throw new NotImplementedException(); + } + + action(this); + } +} + +public sealed class RlpLengthWriter : IRlpWriter +{ + public int Length { get; private set; } + + public RlpLengthWriter() + { + Length = 0; + } + + public void WriteByte(byte value) + { + Length++; + } + + public void WriteBytes(ReadOnlySpan value) + { + Length += value.Length; + } + + public void WriteList(Action action) + { + var inner = new RlpLengthWriter(); + action(inner); + if (inner.Length < 55) + { + Length += 1 + inner.Length; + } + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs new file mode 100644 index 00000000000..6173f479717 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs @@ -0,0 +1,155 @@ +using FluentAssertions; +using Nethermind.Serialization.Rlp.Test.Instances; + +namespace Nethermind.Serialization.Rlp.Test; + +[Parallelizable(ParallelScope.All)] +public class RlpWriterTest +{ + [Test] + public void WriteShortString() + { + var serialized = Rlp.Write(static writer => { writer.Write("dog"); }); + + byte[] expected = [0x83, (byte)'d', (byte)'o', (byte)'g']; + serialized.Should().BeEquivalentTo(expected); + } + + [Test] + public void WriteLongString() + { + var serialized = Rlp.Write(writer => + { + writer.Write("Lorem ipsum dolor sit amet, consectetur adipisicing elit"); + }); + + byte[] expected = [0xb8, 0x38, .."Lorem ipsum dolor sit amet, consectetur adipisicing elit"u8]; + serialized.Should().BeEquivalentTo(expected); + } + + [Test] + public void WriteEmptyString() + { + var serialized = Rlp.Write(static writer => { writer.Write(""); }); + + byte[] expected = [0x80]; + serialized.Should().BeEquivalentTo(expected); + } + + [Test] + public void WriteInteger_1Component() + { + for (int i = 0; i < 0x80; i++) + { + var integer = i; + var serialized = Rlp.Write(writer => { writer.Write(integer); }); + + byte[] expected = [(byte)integer]; + serialized.Should().BeEquivalentTo(expected); + } + } + + [Test] + public void WriteInteger_2Components() + { + byte[] expected = [0x81, 0x00]; + for (int i = 0x80; i < 0x0100; i++) + { + var integer = i; + var serialized = Rlp.Write(writer => { writer.Write(integer); }); + + expected[1] = (byte)integer; + serialized.Should().BeEquivalentTo(expected); + } + } + + [Test] + public void WriteInteger_3Components() + { + byte[] expected = [0x82, 0x00, 0x00]; + for (int i = 0x100; i < 0xFFFF; i++) + { + var integer = i; + var serialized = Rlp.Write(writer => { writer.Write(integer); }); + + expected[1] = (byte)((integer & 0xFF00) >> 8); + expected[2] = (byte)((integer & 0x00FF) >> 0); + serialized.Should().BeEquivalentTo(expected); + } + } + + [Test] + public void WriteStringSequence() + { + var serialized = Rlp.Write(static writer => + { + writer.WriteList(static writer => + { + writer.Write("cat"); + writer.Write("dog"); + }); + }); + + byte[] expected = [0xc8, 0x83, (byte)'c', (byte)'a', (byte)'t', 0x83, (byte)'d', (byte)'o', (byte)'g']; + serialized.Should().BeEquivalentTo(expected); + } + + [Test] + public void WriteEmptySequence() + { + var serialized = Rlp.Write(static writer => { writer.WriteList(static _ => { }); }); + + byte[] expected = [0xc0]; + serialized.Should().BeEquivalentTo(expected); + } + + [Test] + public void WriteByteArray() + { + var serialized = Rlp.Write(static writer => { writer.Write([0x04, 0x00]); }); + + byte[] expected = [0x82, 0x04, 0x00]; + serialized.Should().BeEquivalentTo(expected); + } + + [Test] + public void WriteSetTheoreticalRepresentation() + { + var serialized = Rlp.Write(static writer => + { + writer.WriteList(static root => + { + root.WriteList(static _ => { }); + root.WriteList(static w => { w.WriteList(static _ => { }); }); + root.WriteList(static w => + { + w.WriteList(static _ => { }); + w.WriteList(static w => { w.WriteList(static _ => { }); }); + }); + }); + }); + + byte[] expected = [0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; + serialized.Should().BeEquivalentTo(expected); + } + + // [Test] + // public void WriteSequence() + // { + // var serialized = Rlp.Write(writer => + // { + // writer.WriteSequence(static x => + // { + // x.WriteSequence(static y => { y.Write(0); }); + // x.WriteSequence(static y => + // { + // y.Write("Foo"); + // y.Write("Bar"); + // }); + // }); + // }); + // + // byte[] expected = []; + // serialized.Should().BeEquivalentTo(expected); + // } +} diff --git a/src/Nethermind/Nethermind.sln b/src/Nethermind/Nethermind.sln index 44cf9aa69bd..1e3761fdd31 100644 --- a/src/Nethermind/Nethermind.sln +++ b/src/Nethermind/Nethermind.sln @@ -226,6 +226,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Shutter", "Nethe EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Shutter.Test", "Nethermind.Shutter.Test\Nethermind.Shutter.Test.csproj", "{CEA1C413-A96C-4339-AC1C-839B603DECC8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.Rlp.Test", "Nethermind.Serialization.Rlp.Test\Nethermind.Serialization.Rlp.Test.csproj", "{C8A91B54-F9CA-4211-BE16-F7A0B38223EB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -624,6 +626,10 @@ Global {CEA1C413-A96C-4339-AC1C-839B603DECC8}.Debug|Any CPU.Build.0 = Debug|Any CPU {CEA1C413-A96C-4339-AC1C-839B603DECC8}.Release|Any CPU.ActiveCfg = Release|Any CPU {CEA1C413-A96C-4339-AC1C-839B603DECC8}.Release|Any CPU.Build.0 = Release|Any CPU + {C8A91B54-F9CA-4211-BE16-F7A0B38223EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C8A91B54-F9CA-4211-BE16-F7A0B38223EB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C8A91B54-F9CA-4211-BE16-F7A0B38223EB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C8A91B54-F9CA-4211-BE16-F7A0B38223EB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -684,6 +690,7 @@ Global {E1E7BEFC-52C0-49ED-B0A7-CB8C3250D120} = {4019B82F-1104-4D2C-9F96-05FD7D3575E8} {89311B58-AF36-4956-883D-54531BC1D5A3} = {78BED57D-720E-4E6C-ABA2-397B73B494F9} {6528010D-7DCE-4935-9785-5270FF515F3E} = {89311B58-AF36-4956-883D-54531BC1D5A3} + {C8A91B54-F9CA-4211-BE16-F7A0B38223EB} = {4019B82F-1104-4D2C-9F96-05FD7D3575E8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {092CA5E3-6180-4ED7-A3CB-9B57FAC2AA85} From 9a90ccbd3a843d916bb8943a6be2c9d40c6352d5 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 13:43:38 -0300 Subject: [PATCH 002/106] Rename `Sequence` -> `List` --- .../Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs index 6173f479717..9218546eb08 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs @@ -79,7 +79,7 @@ public void WriteInteger_3Components() } [Test] - public void WriteStringSequence() + public void WriteStringList() { var serialized = Rlp.Write(static writer => { @@ -95,7 +95,7 @@ public void WriteStringSequence() } [Test] - public void WriteEmptySequence() + public void WriteEmptyList() { var serialized = Rlp.Write(static writer => { writer.WriteList(static _ => { }); }); From 30dda9155e0fd5afb5cec25bc5d0968d43be2ed6 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 14:06:20 -0300 Subject: [PATCH 003/106] Initial `RlpReader` --- .../RlpReaderTest.cs | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs new file mode 100644 index 00000000000..ec23713cd0c --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Buffers.Binary; +using System.Text; +using FluentAssertions; + +namespace Nethermind.Serialization.Rlp.Test; + +public class RlpReaderTest +{ + [Test] + public void ReadShortString() + { + byte[] source = [0x83, (byte)'d', (byte)'o', (byte)'g']; + + var reader = new RlpReader(source); + string actual = reader.ReadString(); + + actual.Should().Be("dog"); + } + + [Test] + public void ReadEmptyString() + { + byte[] source = [0x80]; + + var reader = new RlpReader(source); + string actual = reader.ReadString(); + + actual.Should().Be(""); + } + + [Test] + public void ReadLongString() + { + byte[] source = [0xb8, 0x38, .."Lorem ipsum dolor sit amet, consectetur adipisicing elit"u8]; + + var reader = new RlpReader(source); + string actual = reader.ReadString(); + + actual.Should().Be("Lorem ipsum dolor sit amet, consectetur adipisicing elit"); + } + + [Test] + public void ReadShortInteger() + { + for (int i = 0; i < 0x80; i++) + { + var integer = i; + byte[] source = [(byte)integer]; + + var reader = new RlpReader(source); + int actual = reader.ReadInt32(); + + actual.Should().Be(integer); + } + } + + [Test] + public void ReadLongInteger() + { + for (int i = 0x100; i < 0xFFFF; i++) + { + var integer = i; + byte[] source = [0x82, (byte)((integer & 0xFF00) >> 8), (byte)((integer & 0x00FF) >> 0)]; + + var reader = new RlpReader(source); + int actual = reader.ReadInt32(); + + actual.Should().Be(integer); + } + } +} + +public ref struct RlpReader +{ + private readonly ReadOnlySpan _buffer; + private int _position; + + public RlpReader(ReadOnlySpan buffer) + { + _buffer = buffer; + _position = 0; + } + + public ReadOnlySpan ReadObject() + { + ReadOnlySpan result; + var header = _buffer[_position]; + if (header < 0x80) + { + result = _buffer.Slice(_position++, 1); + } + else if (header < 0xB8) + { + header -= 0x80; + result = _buffer.Slice(++_position, header); + _position += header; + } + else if (header < 0xC0) + { + header -= 0xB7; + ReadOnlySpan binaryLength = _buffer.Slice(++_position, header); + _position += header; + var length = Int32Primitive.Read(binaryLength); + result = _buffer.Slice(_position, length); + _position += length; + } + else + { + // Not an object + throw new Exception(); + } + + return result; + } +} + +public static class IntRlpReader +{ + public static Int32 ReadInt32(this RlpReader reader) + { + ReadOnlySpan obj = reader.ReadObject(); + return Int32Primitive.Read(obj); + } +} + +public static class StringRlpReader +{ + public static string ReadString(this RlpReader reader) + { + ReadOnlySpan obj = reader.ReadObject(); + return Encoding.UTF8.GetString(obj); + } +} + +public static class Int32Primitive +{ + /// + /// Reads a from the beginning of a read-only span of bytes, as big endian. + /// + /// The read-only span to read. + /// The big endian value. + /// The span is padded with leading `0`s as needed. + public static int Read(ReadOnlySpan source) + { + Span buffer = stackalloc byte[sizeof(Int32)]; + source.CopyTo(buffer[^source.Length..]); + return BinaryPrimitives.ReadInt32BigEndian(buffer); + } +} From d3d83f9ebabb98016e8bc5b7394125d191212318 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 15:31:03 -0300 Subject: [PATCH 004/106] Initial `ReadList` --- .../RlpReaderTest.cs | 60 ++++++++++++++++++- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs index ec23713cd0c..83a8cb12a58 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs @@ -71,6 +71,34 @@ public void ReadLongInteger() actual.Should().Be(integer); } } + + [Test] + public void ReadStringList() + { + byte[] source = [0xc8, 0x83, .."cat"u8, 0x83, .."dog"u8]; + + var reader = new RlpReader(source); + var actual = reader.ReadList(r => + { + var cat = r.ReadString(); + var dog = r.ReadString(); + + return (cat, dog); + }); + + actual.Should().Be(("cat", "dog")); + } + + [Test] + public void ReadEmptyList() + { + byte[] source = [0xc0]; + + var reader = new RlpReader(source); + object[] actual = reader.ReadList(_ => Array.Empty()); + + actual.Should().BeEmpty(); + } } public ref struct RlpReader @@ -109,17 +137,43 @@ public ReadOnlySpan ReadObject() } else { - // Not an object + // Not an Object throw new Exception(); } return result; } + + public T ReadList(Func func) + { + T result; + var header = _buffer[_position]; + if (header < 0xC0) + { + // Not a List + throw new Exception(); + } + + if (header < 0xF8) + { + _position += 1; + var length = header - 0xC0; + var reader = new RlpReader(_buffer.Slice(_position, length)); + result = func(reader); + _position += length; + } + else + { + throw new NotImplementedException(); + } + + return result; + } } public static class IntRlpReader { - public static Int32 ReadInt32(this RlpReader reader) + public static Int32 ReadInt32(this ref RlpReader reader) { ReadOnlySpan obj = reader.ReadObject(); return Int32Primitive.Read(obj); @@ -128,7 +182,7 @@ public static Int32 ReadInt32(this RlpReader reader) public static class StringRlpReader { - public static string ReadString(this RlpReader reader) + public static string ReadString(this ref RlpReader reader) { ReadOnlySpan obj = reader.ReadObject(); return Encoding.UTF8.GetString(obj); From 65f089186e1ca6064ddb6a40f6893b710fa0bcd1 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 15:44:12 -0300 Subject: [PATCH 005/106] Multiple `ReadList` - Test for Set Theoretical Representation --- .../RlpReaderTest.cs | 47 +++++++++++++++++-- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs index 83a8cb12a58..8ace322dc60 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs @@ -78,7 +78,7 @@ public void ReadStringList() byte[] source = [0xc8, 0x83, .."cat"u8, 0x83, .."dog"u8]; var reader = new RlpReader(source); - var actual = reader.ReadList(r => + var actual = reader.ReadList(static (ref RlpReader r) => { var cat = r.ReadString(); var dog = r.ReadString(); @@ -89,13 +89,50 @@ public void ReadStringList() actual.Should().Be(("cat", "dog")); } + [Test] + public void ReadSetTheoreticalRepresentation() + { + byte[] source = [0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; + + var reader = new RlpReader(source); + object[] actual = reader.ReadList(static (ref RlpReader r) => + { + var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); + var _2 = r.ReadList(static (ref RlpReader r) => + { + var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); + return new object[] { _1 }; + }); + var _3 = r.ReadList(static (ref RlpReader r) => + { + var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); + var _2 = r.ReadList(static (ref RlpReader r) => + { + var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); + return new object[] { _1 }; + }); + + return new object[] { _1, _2 }; + }); + + return new object[] { _1, _2, _3 }; + }); + + actual.Should().BeEquivalentTo(new object[] + { + new object[] { }, + new object[] { new object[] { } }, + new object[] { new object[] { }, new object[] { new object[] { } } }, + }); + } + [Test] public void ReadEmptyList() { byte[] source = [0xc0]; var reader = new RlpReader(source); - object[] actual = reader.ReadList(_ => Array.Empty()); + object[] actual = reader.ReadList((ref RlpReader _) => Array.Empty()); actual.Should().BeEmpty(); } @@ -144,7 +181,9 @@ public ReadOnlySpan ReadObject() return result; } - public T ReadList(Func func) + public delegate TResult ReadListFunc(ref RlpReader arg); + + public T ReadList(ReadListFunc func) { T result; var header = _buffer[_position]; @@ -159,7 +198,7 @@ public T ReadList(Func func) _position += 1; var length = header - 0xC0; var reader = new RlpReader(_buffer.Slice(_position, length)); - result = func(reader); + result = func(ref reader); _position += length; } else From b4656cd8f3b8fbdad082808b2707ed364f388cef Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 16:08:33 -0300 Subject: [PATCH 006/106] Restructure into separate files - Extend instances --- .../Instances/IRlpConverter.cs | 4 +- .../Instances/IntRlpConverter.cs | 7 ++ .../Instances/StringRlpConverter.cs | 7 ++ .../Int32Primitive.cs | 22 ++++ .../Nethermind.Serialization.Rlp.Test/Rlp.cs | 7 ++ .../RlpReader.cs | 77 +++++++++++++ .../RlpReaderTest.cs | 109 +----------------- 7 files changed, 124 insertions(+), 109 deletions(-) create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/Int32Primitive.cs create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs index 934e0cb7bd9..00b6ac9b7a3 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs @@ -3,7 +3,9 @@ namespace Nethermind.Serialization.Rlp.Test.Instances; -public interface IRlpConverter +// TODO: Explore variance annotations +public interface IRlpConverter { + public static abstract T Read(ref RlpReader reader); public static abstract void Write(IRlpWriter writer, T value); } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs index 24542278dc9..9f1c680c88c 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs @@ -7,6 +7,12 @@ namespace Nethermind.Serialization.Rlp.Test.Instances; public abstract class IntRlpConverter : IRlpConverter { + public static int Read(ref RlpReader reader) + { + ReadOnlySpan obj = reader.ReadObject(); + return Int32Primitive.Read(obj); + } + public static void Write(IRlpWriter writer, int value) { if (value < 0x80) @@ -25,5 +31,6 @@ public static void Write(IRlpWriter writer, int value) public static class IntRlpConverterExt { + public static Int32 ReadInt32(this ref RlpReader reader) => IntRlpConverter.Read(ref reader); public static void Write(this IRlpWriter writer, int value) => IntRlpConverter.Write(writer, value); } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs index 5364a82bf03..6e33208cfa7 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs @@ -7,6 +7,12 @@ namespace Nethermind.Serialization.Rlp.Test.Instances; public abstract class StringRlpConverter : IRlpConverter { + public static string Read(ref RlpReader reader) + { + ReadOnlySpan obj = reader.ReadObject(); + return Encoding.UTF8.GetString(obj); + } + public static void Write(IRlpWriter writer, string value) { ReadOnlySpan bytes = Encoding.UTF8.GetBytes(value); @@ -16,5 +22,6 @@ public static void Write(IRlpWriter writer, string value) public static class StringRlpConverterExt { + public static string ReadString(this ref RlpReader reader) => StringRlpConverter.Read(ref reader); public static void Write(this IRlpWriter writer, string value) => StringRlpConverter.Write(writer, value); } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Int32Primitive.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Int32Primitive.cs new file mode 100644 index 00000000000..56882592e1c --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Int32Primitive.cs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Buffers.Binary; + +namespace Nethermind.Serialization.Rlp.Test; + +public static class Int32Primitive +{ + /// + /// Reads a from the beginning of a read-only span of bytes, as big endian. + /// + /// The read-only span to read. + /// The big endian value. + /// The span is padded with leading `0`s as needed. + public static int Read(ReadOnlySpan source) + { + Span buffer = stackalloc byte[sizeof(Int32)]; + source.CopyTo(buffer[^source.Length..]); + return BinaryPrimitives.ReadInt32BigEndian(buffer); + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs index 61d1fecd6f8..d364ddbdfc9 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs @@ -15,4 +15,11 @@ public static byte[] Write(Action action) return serialized; } + + public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) + { + var reader = new RlpReader(source); + // TODO: We might want to add an option to check for no trailing bytes. + return func(ref reader); + } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs new file mode 100644 index 00000000000..02e06ae5196 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Serialization.Rlp.Test; + +public delegate TResult RefRlpReaderFunc(ref RlpReader arg); + +// TODO: We might want to add `IDisposable` to ensure that there are no trailing bytes. +public ref struct RlpReader +{ + private readonly ReadOnlySpan _buffer; + private int _position; + + public RlpReader(ReadOnlySpan buffer) + { + _buffer = buffer; + _position = 0; + } + + public ReadOnlySpan ReadObject() + { + ReadOnlySpan result; + var header = _buffer[_position]; + if (header < 0x80) + { + result = _buffer.Slice(_position++, 1); + } + else if (header < 0xB8) + { + header -= 0x80; + result = _buffer.Slice(++_position, header); + _position += header; + } + else if (header < 0xC0) + { + header -= 0xB7; + ReadOnlySpan binaryLength = _buffer.Slice(++_position, header); + _position += header; + var length = Int32Primitive.Read(binaryLength); + result = _buffer.Slice(_position, length); + _position += length; + } + else + { + // Not an Object + throw new Exception(); + } + + return result; + } + + public T ReadList(RefRlpReaderFunc func) + { + T result; + var header = _buffer[_position]; + if (header < 0xC0) + { + // Not a List + throw new Exception(); + } + + if (header < 0xF8) + { + _position += 1; + var length = header - 0xC0; + var reader = new RlpReader(_buffer.Slice(_position, length)); + result = func(ref reader); + _position += length; + } + else + { + throw new NotImplementedException(); + } + + return result; + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs index 8ace322dc60..2bed0c4e866 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs @@ -1,9 +1,8 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Buffers.Binary; -using System.Text; using FluentAssertions; +using Nethermind.Serialization.Rlp.Test.Instances; namespace Nethermind.Serialization.Rlp.Test; @@ -137,109 +136,3 @@ public void ReadEmptyList() actual.Should().BeEmpty(); } } - -public ref struct RlpReader -{ - private readonly ReadOnlySpan _buffer; - private int _position; - - public RlpReader(ReadOnlySpan buffer) - { - _buffer = buffer; - _position = 0; - } - - public ReadOnlySpan ReadObject() - { - ReadOnlySpan result; - var header = _buffer[_position]; - if (header < 0x80) - { - result = _buffer.Slice(_position++, 1); - } - else if (header < 0xB8) - { - header -= 0x80; - result = _buffer.Slice(++_position, header); - _position += header; - } - else if (header < 0xC0) - { - header -= 0xB7; - ReadOnlySpan binaryLength = _buffer.Slice(++_position, header); - _position += header; - var length = Int32Primitive.Read(binaryLength); - result = _buffer.Slice(_position, length); - _position += length; - } - else - { - // Not an Object - throw new Exception(); - } - - return result; - } - - public delegate TResult ReadListFunc(ref RlpReader arg); - - public T ReadList(ReadListFunc func) - { - T result; - var header = _buffer[_position]; - if (header < 0xC0) - { - // Not a List - throw new Exception(); - } - - if (header < 0xF8) - { - _position += 1; - var length = header - 0xC0; - var reader = new RlpReader(_buffer.Slice(_position, length)); - result = func(ref reader); - _position += length; - } - else - { - throw new NotImplementedException(); - } - - return result; - } -} - -public static class IntRlpReader -{ - public static Int32 ReadInt32(this ref RlpReader reader) - { - ReadOnlySpan obj = reader.ReadObject(); - return Int32Primitive.Read(obj); - } -} - -public static class StringRlpReader -{ - public static string ReadString(this ref RlpReader reader) - { - ReadOnlySpan obj = reader.ReadObject(); - return Encoding.UTF8.GetString(obj); - } -} - -public static class Int32Primitive -{ - /// - /// Reads a from the beginning of a read-only span of bytes, as big endian. - /// - /// The read-only span to read. - /// The big endian value. - /// The span is padded with leading `0`s as needed. - public static int Read(ReadOnlySpan source) - { - Span buffer = stackalloc byte[sizeof(Int32)]; - source.CopyTo(buffer[^source.Length..]); - return BinaryPrimitives.ReadInt32BigEndian(buffer); - } -} From a63b64f3afaacd219124efdb9e12a3d8c0d55f2f Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 16:13:41 -0300 Subject: [PATCH 007/106] Use `Rlp.Read` API --- .../RlpReaderTest.cs | 65 +++++++++---------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs index 2bed0c4e866..4eb4f669674 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs @@ -12,9 +12,7 @@ public class RlpReaderTest public void ReadShortString() { byte[] source = [0x83, (byte)'d', (byte)'o', (byte)'g']; - - var reader = new RlpReader(source); - string actual = reader.ReadString(); + string actual = Rlp.Read(source, static (ref RlpReader r) => r.ReadString()); actual.Should().Be("dog"); } @@ -23,9 +21,7 @@ public void ReadShortString() public void ReadEmptyString() { byte[] source = [0x80]; - - var reader = new RlpReader(source); - string actual = reader.ReadString(); + string actual = Rlp.Read(source, static (ref RlpReader r) => r.ReadString()); actual.Should().Be(""); } @@ -34,9 +30,7 @@ public void ReadEmptyString() public void ReadLongString() { byte[] source = [0xb8, 0x38, .."Lorem ipsum dolor sit amet, consectetur adipisicing elit"u8]; - - var reader = new RlpReader(source); - string actual = reader.ReadString(); + string actual = Rlp.Read(source, static (ref RlpReader r) => r.ReadString()); actual.Should().Be("Lorem ipsum dolor sit amet, consectetur adipisicing elit"); } @@ -48,9 +42,7 @@ public void ReadShortInteger() { var integer = i; byte[] source = [(byte)integer]; - - var reader = new RlpReader(source); - int actual = reader.ReadInt32(); + int actual = Rlp.Read(source, static (ref RlpReader r) => r.ReadInt32()); actual.Should().Be(integer); } @@ -63,9 +55,7 @@ public void ReadLongInteger() { var integer = i; byte[] source = [0x82, (byte)((integer & 0xFF00) >> 8), (byte)((integer & 0x00FF) >> 0)]; - - var reader = new RlpReader(source); - int actual = reader.ReadInt32(); + int actual = Rlp.Read(source, static (ref RlpReader r) => r.ReadInt32()); actual.Should().Be(integer); } @@ -75,14 +65,15 @@ public void ReadLongInteger() public void ReadStringList() { byte[] source = [0xc8, 0x83, .."cat"u8, 0x83, .."dog"u8]; - - var reader = new RlpReader(source); - var actual = reader.ReadList(static (ref RlpReader r) => + var actual = Rlp.Read(source, static (ref RlpReader r) => { - var cat = r.ReadString(); - var dog = r.ReadString(); + return r.ReadList(static (ref RlpReader r) => + { + var cat = r.ReadString(); + var dog = r.ReadString(); - return (cat, dog); + return (cat, dog); + }); }); actual.Should().Be(("cat", "dog")); @@ -93,16 +84,9 @@ public void ReadSetTheoreticalRepresentation() { byte[] source = [0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; - var reader = new RlpReader(source); - object[] actual = reader.ReadList(static (ref RlpReader r) => + object[] actual = Rlp.Read(source, static (ref RlpReader r) => { - var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); - var _2 = r.ReadList(static (ref RlpReader r) => - { - var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); - return new object[] { _1 }; - }); - var _3 = r.ReadList(static (ref RlpReader r) => + return r.ReadList(static (ref RlpReader r) => { var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); var _2 = r.ReadList(static (ref RlpReader r) => @@ -110,11 +94,20 @@ public void ReadSetTheoreticalRepresentation() var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); return new object[] { _1 }; }); + var _3 = r.ReadList(static (ref RlpReader r) => + { + var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); + var _2 = r.ReadList(static (ref RlpReader r) => + { + var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); + return new object[] { _1 }; + }); - return new object[] { _1, _2 }; - }); + return new object[] { _1, _2 }; + }); - return new object[] { _1, _2, _3 }; + return new object[] { _1, _2, _3 }; + }); }); actual.Should().BeEquivalentTo(new object[] @@ -130,8 +123,10 @@ public void ReadEmptyList() { byte[] source = [0xc0]; - var reader = new RlpReader(source); - object[] actual = reader.ReadList((ref RlpReader _) => Array.Empty()); + var actual = Rlp.Read(source, static (ref RlpReader r) => + { + return r.ReadList((ref RlpReader _) => Array.Empty()); + }); actual.Should().BeEmpty(); } From 6735d2993c76284e160900717cc5eaf1a68c9241 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 16:33:03 -0300 Subject: [PATCH 008/106] Add `HeterogeneousList` test --- .../RlpReadWriteTest.cs | 46 +++++++++++++++++++ .../RlpWriterTest.cs | 20 -------- 2 files changed, 46 insertions(+), 20 deletions(-) create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs new file mode 100644 index 00000000000..cef97283857 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Serialization.Rlp.Test.Instances; + +namespace Nethermind.Serialization.Rlp.Test; + +public class RlpReadWriteTest +{ + [Test] + public void HeterogeneousList() + { + var rlp = Rlp.Write(static w => + { + w.WriteList(static w => + { + w.WriteList(static w => { w.Write(42); }); + w.WriteList(static w => + { + w.Write("dog"); + w.Write("cat"); + }); + }); + }); + + var decoded = Rlp.Read(rlp, (ref RlpReader r) => + { + return r.ReadList(static (ref RlpReader r) => + { + var _1 = r.ReadList(static (ref RlpReader r) => r.ReadInt32()); + var _2 = r.ReadList(static (ref RlpReader r) => + { + var _1 = r.ReadString(); + var _2 = r.ReadString(); + + return (_1, _2); + }); + + return (_1, _2); + }); + }); + + decoded.Should().Be((42, ("dog", "cat"))); + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs index 9218546eb08..c0f993059a3 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs @@ -132,24 +132,4 @@ public void WriteSetTheoreticalRepresentation() byte[] expected = [0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; serialized.Should().BeEquivalentTo(expected); } - - // [Test] - // public void WriteSequence() - // { - // var serialized = Rlp.Write(writer => - // { - // writer.WriteSequence(static x => - // { - // x.WriteSequence(static y => { y.Write(0); }); - // x.WriteSequence(static y => - // { - // y.Write("Foo"); - // y.Write("Bar"); - // }); - // }); - // }); - // - // byte[] expected = []; - // serialized.Should().BeEquivalentTo(expected); - // } } From 74cf2fd8bbdf929e8d05f70df5ce3a9dfa6e23f6 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 16:48:47 -0300 Subject: [PATCH 009/106] Add overload for `Action` --- .../Nethermind.Serialization.Rlp.Test/RlpReader.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs index 02e06ae5196..f679fabb6b3 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs @@ -5,6 +5,8 @@ namespace Nethermind.Serialization.Rlp.Test; public delegate TResult RefRlpReaderFunc(ref RlpReader arg); +public delegate void RefRlpReaderAction(ref RlpReader arg); + // TODO: We might want to add `IDisposable` to ensure that there are no trailing bytes. public ref struct RlpReader { @@ -74,4 +76,13 @@ public T ReadList(RefRlpReaderFunc func) return result; } + + public void ReadList(RefRlpReaderAction func) + { + ReadList((ref RlpReader r) => + { + func(ref r); + return null; + }); + } } From 2ef4c4d385a7ea3858ef79b44b47227a1c486042 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 16:50:14 -0300 Subject: [PATCH 010/106] Test `UnknownLengthList` --- .../RlpReadWriteTest.cs | 30 +++++++++++++++++++ .../RlpReader.cs | 2 ++ 2 files changed, 32 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs index cef97283857..b7b01a77c73 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs @@ -43,4 +43,34 @@ public void HeterogeneousList() decoded.Should().Be((42, ("dog", "cat"))); } + + + [TestCase(2)] + public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) + { + var rlp = Rlp.Write(root => + { + root.WriteList(w => + { + for (int i = 0; i < length; i++) + { + w.Write(42); + } + }); + }); + List decoded = Rlp.Read(rlp, (ref RlpReader r) => + { + List result = []; + r.ReadList((ref RlpReader r) => + { + while (r.HasNext) + { + result.Add(r.ReadInt32()); + } + }); + return result; + }); + + decoded.Count.Should().Be(length); + } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs index f679fabb6b3..91fabc70a59 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs @@ -85,4 +85,6 @@ public void ReadList(RefRlpReaderAction func) return null; }); } + + public bool HasNext => _position < _buffer.Length; } From b3dddcb66a63e7e5788c44a21e434d0ba072f98e Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 16:51:43 -0300 Subject: [PATCH 011/106] Rename converter --- .../{ReadOnlySpanConverter.cs => ReadOnlySpanRlpConverter.cs} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/{ReadOnlySpanConverter.cs => ReadOnlySpanRlpConverter.cs} (91%) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanRlpConverter.cs similarity index 91% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanConverter.cs rename to src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanRlpConverter.cs index 523ac750ce4..6c6a0dff33f 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanRlpConverter.cs @@ -6,7 +6,7 @@ namespace Nethermind.Serialization.Rlp.Test.Instances; // Spiritually implements `IRlpConverter` but due to restrictions on `ReadOnlySpan` we cannot make it explicit -public abstract class ReadOnlySpanConverter /* : IRlpConverter> */ +public abstract class ReadOnlySpanRlpConverter /* : IRlpConverter> */ { public static void Write(IRlpWriter writer, ReadOnlySpan value) { @@ -40,5 +40,5 @@ public static void Write(IRlpWriter writer, ReadOnlySpan value) public static class ReadOnlySpanConverterExt { - public static void Write(this IRlpWriter writer, ReadOnlySpan value) => ReadOnlySpanConverter.Write(writer, value); + public static void Write(this IRlpWriter writer, ReadOnlySpan value) => ReadOnlySpanRlpConverter.Write(writer, value); } From 8b19e01f5ccab7bb082a3345ddd24dbadb439a4b Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 17:02:02 -0300 Subject: [PATCH 012/106] Use custom Exception --- .../Nethermind.Serialization.Rlp.Test/RlpReader.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs index 91fabc70a59..2e9d81e559a 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs @@ -7,6 +7,8 @@ namespace Nethermind.Serialization.Rlp.Test; public delegate void RefRlpReaderAction(ref RlpReader arg); +public class RlpReaderException(string message) : Exception(message); + // TODO: We might want to add `IDisposable` to ensure that there are no trailing bytes. public ref struct RlpReader { @@ -44,8 +46,7 @@ public ReadOnlySpan ReadObject() } else { - // Not an Object - throw new Exception(); + throw new RlpReaderException("RLP does not correspond to an object"); } return result; @@ -57,8 +58,7 @@ public T ReadList(RefRlpReaderFunc func) var header = _buffer[_position]; if (header < 0xC0) { - // Not a List - throw new Exception(); + throw new RlpReaderException("RLP does not correspond to a list"); } if (header < 0xF8) From 2c67165341780abef7942de276e722d1cbfa0c3c Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 17:02:37 -0300 Subject: [PATCH 013/106] Add `Action` overload --- src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs index d364ddbdfc9..24d26e339be 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs @@ -22,4 +22,13 @@ public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) // TODO: We might want to add an option to check for no trailing bytes. return func(ref reader); } + + public static void Read(ReadOnlySpan source, RefRlpReaderAction func) + { + Read(source, (ref RlpReader reader) => + { + func(ref reader); + return null; + }); + } } From 52e1f7be85dce513c8cb0749b07b117db9c6037a Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 17:03:00 -0300 Subject: [PATCH 014/106] Test for invalid readings --- .../RlpReadWriteTest.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs index b7b01a77c73..94e73cc69ec 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs @@ -73,4 +73,25 @@ public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) decoded.Count.Should().Be(length); } + + [Test] + public void InvalidObjectReading() + { + var rlp = Rlp.Write(static w => { w.Write(42); }); + Action tryRead = () => Rlp.Read(rlp, (ref RlpReader r) => + { + r.ReadList((ref RlpReader _) => { }); + }); + + tryRead.Should().Throw(); + } + + [Test] + public void InvalidListReading() + { + var rlp = Rlp.Write(static w => { w.WriteList(static _ => { }); }); + Func tryRead = () => Rlp.Read(rlp, (ref RlpReader r) => r.ReadInt32()); + + tryRead.Should().Throw(); + } } From 726240774af78f6f92b8b980c9689b5eb024730a Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 19 Dec 2024 17:27:23 -0300 Subject: [PATCH 015/106] Support long lists (+55 bytes) --- .../RlpReadWriteTest.cs | 98 +++++++++++++++++-- .../RlpReader.cs | 9 +- .../RlpWriter.cs | 16 ++- 3 files changed, 114 insertions(+), 9 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs index 94e73cc69ec..938ce1e7b3f 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs @@ -44,6 +44,91 @@ public void HeterogeneousList() decoded.Should().Be((42, ("dog", "cat"))); } + [Test] + public void LongList() + { + var rlp = Rlp.Write(static w => + { + w.WriteList(static w => + { + for (int i = 0; i < 100; i++) + { + w.Write("dog"); + } + }); + }); + + List decoded = Rlp.Read(rlp, (ref RlpReader r) => + { + return r.ReadList((ref RlpReader r) => + { + List result = []; + for (int i = 0; i < 100; i++) + { + result.Add(r.ReadString()); + } + + return result; + }); + }); + + decoded.Count.Should().Be(100); + decoded.Should().AllBeEquivalentTo("dog"); + } + + [Test] + public void MutlipleLongList() + { + var rlp = Rlp.Write(static w => + { + w.WriteList(static w => + { + for (int i = 0; i < 100; i++) + { + w.Write("dog"); + } + }); + w.WriteList(static w => + { + for (int i = 0; i < 50; i++) + { + w.Write("cat"); + } + }); + }); + + var (dogs, cats) = Rlp.Read(rlp, (ref RlpReader r) => + { + var dogs = r.ReadList((ref RlpReader r) => + { + List result = []; + while (r.HasNext) + { + result.Add(r.ReadString()); + } + + return result; + }); + var cats = r.ReadList((ref RlpReader r) => + { + List result = []; + while (r.HasNext) + { + result.Add(r.ReadString()); + } + + return result; + }); + + return (dogs, cats); + }); + + dogs.Count.Should().Be(100); + dogs.Should().AllBeEquivalentTo("dog"); + + cats.Count.Should().Be(50); + cats.Should().AllBeEquivalentTo("cat"); + } [TestCase(2)] public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) @@ -58,17 +143,19 @@ public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) } }); }); + List decoded = Rlp.Read(rlp, (ref RlpReader r) => { - List result = []; - r.ReadList((ref RlpReader r) => + return r.ReadList((ref RlpReader r) => { + List result = []; while (r.HasNext) { result.Add(r.ReadInt32()); } + + return result; }); - return result; }); decoded.Count.Should().Be(length); @@ -78,10 +165,7 @@ public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) public void InvalidObjectReading() { var rlp = Rlp.Write(static w => { w.Write(42); }); - Action tryRead = () => Rlp.Read(rlp, (ref RlpReader r) => - { - r.ReadList((ref RlpReader _) => { }); - }); + Action tryRead = () => Rlp.Read(rlp, (ref RlpReader r) => { r.ReadList((ref RlpReader _) => { }); }); tryRead.Should().Throw(); } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs index 2e9d81e559a..71aace0fe86 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs @@ -71,7 +71,14 @@ public T ReadList(RefRlpReaderFunc func) } else { - throw new NotImplementedException(); + _position += 1; + var lengthOfLength = header - 0xF7; + ReadOnlySpan binaryLength = _buffer.Slice(_position, lengthOfLength); + _position += lengthOfLength; + int length = Int32Primitive.Read(binaryLength); + var reader = new RlpReader(_buffer.Slice(_position, length)); + result = func(ref reader); + _position += length; } return result; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs index 716640271b6..d3072b58329 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Buffers.Binary; + namespace Nethermind.Serialization.Rlp.Test; public interface IRlpWriter @@ -42,7 +44,12 @@ public void WriteList(Action action) } else { - throw new NotImplementedException(); + Span binaryLength = stackalloc byte[sizeof(Int32)]; + BinaryPrimitives.WriteInt32BigEndian(binaryLength, lengthWriter.Length); + binaryLength = binaryLength.TrimStart((byte)0); + _buffer[_position++] = (byte)(0xF7 + binaryLength.Length); + binaryLength.CopyTo(_buffer.AsSpan()[_position..]); + _position += binaryLength.Length; } action(this); @@ -76,5 +83,12 @@ public void WriteList(Action action) { Length += 1 + inner.Length; } + else + { + Span binaryLength = stackalloc byte[sizeof(Int32)]; + BinaryPrimitives.WriteInt32BigEndian(binaryLength, inner.Length); + binaryLength = binaryLength.TrimStart((byte)0); + Length += 1 + inner.Length + binaryLength.Length; + } } } From e4c2e6357c437cf8f6e5c7b1c1307fd74ab559a8 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 09:24:32 -0300 Subject: [PATCH 016/106] Implement interface on `ReadOnlySpanConverter` --- .../Instances/IRlpConverter.cs | 3 +-- .../Instances/ReadOnlySpanRlpConverter.cs | 9 +++++++-- .../Nethermind.Serialization.Rlp.Test/RlpReader.cs | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs index 00b6ac9b7a3..a3f5d11a31c 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs @@ -3,8 +3,7 @@ namespace Nethermind.Serialization.Rlp.Test.Instances; -// TODO: Explore variance annotations -public interface IRlpConverter +public interface IRlpConverter where T : allows ref struct { public static abstract T Read(ref RlpReader reader); public static abstract void Write(IRlpWriter writer, T value); diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanRlpConverter.cs index 6c6a0dff33f..2640482ce1f 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanRlpConverter.cs @@ -5,9 +5,13 @@ namespace Nethermind.Serialization.Rlp.Test.Instances; -// Spiritually implements `IRlpConverter` but due to restrictions on `ReadOnlySpan` we cannot make it explicit -public abstract class ReadOnlySpanRlpConverter /* : IRlpConverter> */ +public abstract class ReadOnlySpanRlpConverter : IRlpConverter> { + public static ReadOnlySpan Read(ref RlpReader reader) + { + return reader.ReadObject(); + } + public static void Write(IRlpWriter writer, ReadOnlySpan value) { // "If a string is 0-55 bytes long, the RLP encoding consists of @@ -40,5 +44,6 @@ public static void Write(IRlpWriter writer, ReadOnlySpan value) public static class ReadOnlySpanConverterExt { + public static ReadOnlySpan ReadReadOnlySpan(this ref RlpReader reader) => ReadOnlySpanRlpConverter.Read(ref reader); public static void Write(this IRlpWriter writer, ReadOnlySpan value) => ReadOnlySpanRlpConverter.Write(writer, value); } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs index 71aace0fe86..51e6461bd3b 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs @@ -9,7 +9,6 @@ namespace Nethermind.Serialization.Rlp.Test; public class RlpReaderException(string message) : Exception(message); -// TODO: We might want to add `IDisposable` to ensure that there are no trailing bytes. public ref struct RlpReader { private readonly ReadOnlySpan _buffer; From 35b92030cfd465dd9b4c39e993eb6fb3d1ae3c41 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 10:21:11 -0300 Subject: [PATCH 017/106] Make `Rlp[Reader|Writer]` symmetric - Good things happen to those who respect symmetry. --- .../Instances/IntRlpConverter.cs | 2 +- .../Instances/ReadOnlySpanRlpConverter.cs | 49 ------------------- .../Instances/StringRlpConverter.cs | 2 +- .../RlpWriter.cs | 32 ++++++++++-- .../RlpWriterTest.cs | 2 +- 5 files changed, 32 insertions(+), 55 deletions(-) delete mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanRlpConverter.cs diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs index 9f1c680c88c..f1fff1773ad 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs @@ -24,7 +24,7 @@ public static void Write(IRlpWriter writer, int value) Span bytes = stackalloc byte[sizeof(int)]; BinaryPrimitives.WriteInt32BigEndian(bytes, value); bytes = bytes.TrimStart((byte)0); - writer.Write(bytes); + writer.WriteObject(bytes); } } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanRlpConverter.cs deleted file mode 100644 index 2640482ce1f..00000000000 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/ReadOnlySpanRlpConverter.cs +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System.Buffers.Binary; - -namespace Nethermind.Serialization.Rlp.Test.Instances; - -public abstract class ReadOnlySpanRlpConverter : IRlpConverter> -{ - public static ReadOnlySpan Read(ref RlpReader reader) - { - return reader.ReadObject(); - } - - public static void Write(IRlpWriter writer, ReadOnlySpan value) - { - // "If a string is 0-55 bytes long, the RLP encoding consists of - // - a single byte with value 0x80 (dec. 128) - // - plus the length of the string - // - followed by the string. - // The range of the first byte is thus [0x80, 0xb7] (dec. [128, 183])". - if (value.Length < 55) - { - writer.WriteByte((byte)(0x80 + value.Length)); - } - // If a string is more than 55 bytes long, the RLP encoding consists of - // - a single byte with value 0xb7 (dec. 183) - // - plus the length in bytes of the length of the string in binary form, - // - followed by the length of the string, - // - followed by the string. - // The range of the first byte is thus [0xb8, 0xbf] (dec. [184, 191]). - else - { - Span binaryLength = stackalloc byte[sizeof(int)]; - BinaryPrimitives.WriteInt32BigEndian(binaryLength, value.Length); - binaryLength = binaryLength.TrimStart((byte)0); - writer.WriteByte((byte)(0xB7 + binaryLength.Length)); - writer.WriteBytes(binaryLength); - } - - writer.WriteBytes(value); - } -} - -public static class ReadOnlySpanConverterExt -{ - public static ReadOnlySpan ReadReadOnlySpan(this ref RlpReader reader) => ReadOnlySpanRlpConverter.Read(ref reader); - public static void Write(this IRlpWriter writer, ReadOnlySpan value) => ReadOnlySpanRlpConverter.Write(writer, value); -} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs index 6e33208cfa7..40042707b9a 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs @@ -16,7 +16,7 @@ public static string Read(ref RlpReader reader) public static void Write(IRlpWriter writer, string value) { ReadOnlySpan bytes = Encoding.UTF8.GetBytes(value); - writer.Write(bytes); + writer.WriteObject(bytes); } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs index d3072b58329..d600b4a6082 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs @@ -8,7 +8,7 @@ namespace Nethermind.Serialization.Rlp.Test; public interface IRlpWriter { void WriteByte(byte value); - void WriteBytes(scoped ReadOnlySpan value); + void WriteObject(ReadOnlySpan value); void WriteList(Action action); } @@ -28,8 +28,22 @@ public void WriteByte(byte value) _buffer[_position++] = value; } - public void WriteBytes(ReadOnlySpan value) + public void WriteObject(ReadOnlySpan value) { + if (value.Length < 55) + { + _buffer[_position++] = (byte)(0x80 + value.Length); + } + else + { + Span binaryLength = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32BigEndian(binaryLength, value.Length); + binaryLength = binaryLength.TrimStart((byte)0); + _buffer[_position++] = (byte)(0xB7 + binaryLength.Length); + binaryLength.CopyTo(_buffer.AsSpan()[_position..]); + _position += binaryLength.Length; + } + value.CopyTo(_buffer.AsSpan()[_position..]); _position += value.Length; } @@ -70,8 +84,20 @@ public void WriteByte(byte value) Length++; } - public void WriteBytes(ReadOnlySpan value) + public void WriteObject(ReadOnlySpan value) { + if (value.Length < 55) + { + Length += 1; + } + else + { + Span binaryLength = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32BigEndian(binaryLength, value.Length); + binaryLength = binaryLength.TrimStart((byte)0); + Length += 1 + binaryLength.Length; + } + Length += value.Length; } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs index c0f993059a3..045fdc8ad7a 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs @@ -106,7 +106,7 @@ public void WriteEmptyList() [Test] public void WriteByteArray() { - var serialized = Rlp.Write(static writer => { writer.Write([0x04, 0x00]); }); + var serialized = Rlp.Write(static writer => { writer.WriteObject([0x04, 0x00]); }); byte[] expected = [0x82, 0x04, 0x00]; serialized.Should().BeEquivalentTo(expected); From e0ec1ce6fddf399b794f897dc2c716904817c876 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 10:27:47 -0300 Subject: [PATCH 018/106] Support ref struct --- .../Nethermind.Serialization.Rlp.Test/Rlp.cs | 7 +-- .../RlpReader.cs | 4 +- .../RlpReaderTest.cs | 51 ++++++++++++------- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs index 24d26e339be..9640194db95 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs @@ -16,16 +16,17 @@ public static byte[] Write(Action action) return serialized; } - public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) + public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) where T : allows ref struct { var reader = new RlpReader(source); + T result = func(ref reader); // TODO: We might want to add an option to check for no trailing bytes. - return func(ref reader); + return result; } public static void Read(ReadOnlySpan source, RefRlpReaderAction func) { - Read(source, (ref RlpReader reader) => + Read(source, (scoped ref RlpReader reader) => { func(ref reader); return null; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs index 51e6461bd3b..5169725a38d 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs @@ -3,7 +3,7 @@ namespace Nethermind.Serialization.Rlp.Test; -public delegate TResult RefRlpReaderFunc(ref RlpReader arg); +public delegate TResult RefRlpReaderFunc(scoped ref RlpReader arg) where TResult : allows ref struct; public delegate void RefRlpReaderAction(ref RlpReader arg); @@ -85,7 +85,7 @@ public T ReadList(RefRlpReaderFunc func) public void ReadList(RefRlpReaderAction func) { - ReadList((ref RlpReader r) => + ReadList((scoped ref RlpReader r) => { func(ref r); return null; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs index 4eb4f669674..38aba251a42 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs @@ -12,7 +12,7 @@ public class RlpReaderTest public void ReadShortString() { byte[] source = [0x83, (byte)'d', (byte)'o', (byte)'g']; - string actual = Rlp.Read(source, static (ref RlpReader r) => r.ReadString()); + string actual = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadString()); actual.Should().Be("dog"); } @@ -21,7 +21,7 @@ public void ReadShortString() public void ReadEmptyString() { byte[] source = [0x80]; - string actual = Rlp.Read(source, static (ref RlpReader r) => r.ReadString()); + string actual = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadString()); actual.Should().Be(""); } @@ -30,7 +30,7 @@ public void ReadEmptyString() public void ReadLongString() { byte[] source = [0xb8, 0x38, .."Lorem ipsum dolor sit amet, consectetur adipisicing elit"u8]; - string actual = Rlp.Read(source, static (ref RlpReader r) => r.ReadString()); + string actual = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadString()); actual.Should().Be("Lorem ipsum dolor sit amet, consectetur adipisicing elit"); } @@ -42,7 +42,7 @@ public void ReadShortInteger() { var integer = i; byte[] source = [(byte)integer]; - int actual = Rlp.Read(source, static (ref RlpReader r) => r.ReadInt32()); + int actual = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadInt32()); actual.Should().Be(integer); } @@ -55,7 +55,7 @@ public void ReadLongInteger() { var integer = i; byte[] source = [0x82, (byte)((integer & 0xFF00) >> 8), (byte)((integer & 0x00FF) >> 0)]; - int actual = Rlp.Read(source, static (ref RlpReader r) => r.ReadInt32()); + int actual = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadInt32()); actual.Should().Be(integer); } @@ -65,9 +65,9 @@ public void ReadLongInteger() public void ReadStringList() { byte[] source = [0xc8, 0x83, .."cat"u8, 0x83, .."dog"u8]; - var actual = Rlp.Read(source, static (ref RlpReader r) => + var actual = Rlp.Read(source, static (scoped ref RlpReader r) => { - return r.ReadList(static (ref RlpReader r) => + return r.ReadList(static (scoped ref RlpReader r) => { var cat = r.ReadString(); var dog = r.ReadString(); @@ -84,22 +84,22 @@ public void ReadSetTheoreticalRepresentation() { byte[] source = [0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; - object[] actual = Rlp.Read(source, static (ref RlpReader r) => + object[] actual = Rlp.Read(source, static (scoped ref RlpReader r) => { - return r.ReadList(static (ref RlpReader r) => + return r.ReadList(static (scoped ref RlpReader r) => { - var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); - var _2 = r.ReadList(static (ref RlpReader r) => + var _1 = r.ReadList(static (scoped ref RlpReader _) => Array.Empty()); + var _2 = r.ReadList(static (scoped ref RlpReader r) => { - var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); + var _1 = r.ReadList(static (scoped ref RlpReader _) => Array.Empty()); return new object[] { _1 }; }); - var _3 = r.ReadList(static (ref RlpReader r) => + var _3 = r.ReadList(static (scoped ref RlpReader r) => { - var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); - var _2 = r.ReadList(static (ref RlpReader r) => + var _1 = r.ReadList(static (scoped ref RlpReader _) => Array.Empty()); + var _2 = r.ReadList(static (scoped ref RlpReader r) => { - var _1 = r.ReadList(static (ref RlpReader _) => Array.Empty()); + var _1 = r.ReadList(static (scoped ref RlpReader _) => Array.Empty()); return new object[] { _1 }; }); @@ -123,11 +123,26 @@ public void ReadEmptyList() { byte[] source = [0xc0]; - var actual = Rlp.Read(source, static (ref RlpReader r) => + var actual = Rlp.Read(source, static (scoped ref RlpReader r) => { - return r.ReadList((ref RlpReader _) => Array.Empty()); + return r.ReadList((scoped ref RlpReader _) => Array.Empty()); }); actual.Should().BeEmpty(); } + + [Test] + public void ReadSpan() + { + byte[] source = [0x82, 0x04, 0x00]; + + ReadOnlySpan actual = Rlp.Read(source, static (scoped ref RlpReader r) => + { + var result = r.ReadObject(); + return result; + }); + + ReadOnlySpan expected = [0x04, 0x00]; + actual.SequenceEqual(expected).Should().BeTrue(); + } } From d3a2e5b247d16fde36e8f1f35bd87f30f607166d Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 10:30:07 -0300 Subject: [PATCH 019/106] Reorder tests --- .../RlpReaderTest.cs | 52 +++++++++---------- .../RlpWriterTest.cs | 20 +++---- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs index 38aba251a42..0ccbc609363 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs @@ -79,6 +79,30 @@ public void ReadStringList() actual.Should().Be(("cat", "dog")); } + [Test] + public void ReadEmptyList() + { + byte[] source = [0xc0]; + + var actual = Rlp.Read(source, static (scoped ref RlpReader r) => + { + return r.ReadList((scoped ref RlpReader _) => Array.Empty()); + }); + + actual.Should().BeEmpty(); + } + + [Test] + public void ReadSpan() + { + byte[] source = [0x82, 0x04, 0x00]; + + ReadOnlySpan actual = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadObject()); + + ReadOnlySpan expected = [0x04, 0x00]; + actual.SequenceEqual(expected).Should().BeTrue(); + } + [Test] public void ReadSetTheoreticalRepresentation() { @@ -117,32 +141,4 @@ public void ReadSetTheoreticalRepresentation() new object[] { new object[] { }, new object[] { new object[] { } } }, }); } - - [Test] - public void ReadEmptyList() - { - byte[] source = [0xc0]; - - var actual = Rlp.Read(source, static (scoped ref RlpReader r) => - { - return r.ReadList((scoped ref RlpReader _) => Array.Empty()); - }); - - actual.Should().BeEmpty(); - } - - [Test] - public void ReadSpan() - { - byte[] source = [0x82, 0x04, 0x00]; - - ReadOnlySpan actual = Rlp.Read(source, static (scoped ref RlpReader r) => - { - var result = r.ReadObject(); - return result; - }); - - ReadOnlySpan expected = [0x04, 0x00]; - actual.SequenceEqual(expected).Should().BeTrue(); - } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs index 045fdc8ad7a..6f97494bcba 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs @@ -16,23 +16,23 @@ public void WriteShortString() } [Test] - public void WriteLongString() + public void WriteEmptyString() { - var serialized = Rlp.Write(writer => - { - writer.Write("Lorem ipsum dolor sit amet, consectetur adipisicing elit"); - }); + var serialized = Rlp.Write(static writer => { writer.Write(""); }); - byte[] expected = [0xb8, 0x38, .."Lorem ipsum dolor sit amet, consectetur adipisicing elit"u8]; + byte[] expected = [0x80]; serialized.Should().BeEquivalentTo(expected); } [Test] - public void WriteEmptyString() + public void WriteLongString() { - var serialized = Rlp.Write(static writer => { writer.Write(""); }); + var serialized = Rlp.Write(writer => + { + writer.Write("Lorem ipsum dolor sit amet, consectetur adipisicing elit"); + }); - byte[] expected = [0x80]; + byte[] expected = [0xb8, 0x38, .."Lorem ipsum dolor sit amet, consectetur adipisicing elit"u8]; serialized.Should().BeEquivalentTo(expected); } @@ -104,7 +104,7 @@ public void WriteEmptyList() } [Test] - public void WriteByteArray() + public void WriteSpan() { var serialized = Rlp.Write(static writer => { writer.WriteObject([0x04, 0x00]); }); From 67a69df4f41990c8b4f6208b3a77976314a3736d Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 10:32:38 -0300 Subject: [PATCH 020/106] Annotate as scoped - Code works, but compiler complains --- .../RlpReadWriteTest.cs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs index 938ce1e7b3f..8ce1d45a4b9 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs @@ -24,12 +24,12 @@ public void HeterogeneousList() }); }); - var decoded = Rlp.Read(rlp, (ref RlpReader r) => + var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => { - return r.ReadList(static (ref RlpReader r) => + return r.ReadList(static (scoped ref RlpReader r) => { - var _1 = r.ReadList(static (ref RlpReader r) => r.ReadInt32()); - var _2 = r.ReadList(static (ref RlpReader r) => + var _1 = r.ReadList(static (scoped ref RlpReader r) => r.ReadInt32()); + var _2 = r.ReadList(static (scoped ref RlpReader r) => { var _1 = r.ReadString(); var _2 = r.ReadString(); @@ -58,9 +58,9 @@ public void LongList() }); }); - List decoded = Rlp.Read(rlp, (ref RlpReader r) => + List decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => { - return r.ReadList((ref RlpReader r) => + return r.ReadList((scoped ref RlpReader r) => { List result = []; for (int i = 0; i < 100; i++) @@ -97,9 +97,9 @@ public void MutlipleLongList() }); }); - var (dogs, cats) = Rlp.Read(rlp, (ref RlpReader r) => + var (dogs, cats) = Rlp.Read(rlp, (scoped ref RlpReader r) => { - var dogs = r.ReadList((ref RlpReader r) => + var dogs = r.ReadList((scoped ref RlpReader r) => { List result = []; while (r.HasNext) @@ -109,7 +109,7 @@ public void MutlipleLongList() return result; }); - var cats = r.ReadList((ref RlpReader r) => + var cats = r.ReadList((scoped ref RlpReader r) => { List result = []; while (r.HasNext) @@ -144,9 +144,9 @@ public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) }); }); - List decoded = Rlp.Read(rlp, (ref RlpReader r) => + List decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => { - return r.ReadList((ref RlpReader r) => + return r.ReadList((scoped ref RlpReader r) => { List result = []; while (r.HasNext) @@ -165,7 +165,7 @@ public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) public void InvalidObjectReading() { var rlp = Rlp.Write(static w => { w.Write(42); }); - Action tryRead = () => Rlp.Read(rlp, (ref RlpReader r) => { r.ReadList((ref RlpReader _) => { }); }); + Action tryRead = () => Rlp.Read(rlp, (scoped ref RlpReader r) => { r.ReadList((scoped ref RlpReader _) => { }); }); tryRead.Should().Throw(); } @@ -174,7 +174,7 @@ public void InvalidObjectReading() public void InvalidListReading() { var rlp = Rlp.Write(static w => { w.WriteList(static _ => { }); }); - Func tryRead = () => Rlp.Read(rlp, (ref RlpReader r) => r.ReadInt32()); + Func tryRead = () => Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadInt32()); tryRead.Should().Throw(); } From dc3798b83de86fed20155eb236abbe71a9b4062e Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 10:56:01 -0300 Subject: [PATCH 021/106] Initial `Choice` implementation --- .../RlpReadWriteTest.cs | 20 +++++++++++++++++++ .../RlpReader.cs | 19 +++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs index 8ce1d45a4b9..b412f88db54 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs @@ -178,4 +178,24 @@ public void InvalidListReading() tryRead.Should().Throw(); } + + [Test] + public void Choice() + { + var unwrapped = Rlp.Write(static w => { w.Write(42); }); + var wrapped = Rlp.Write(static w => w.WriteList(static w => { w.Write(42); })); + + foreach (var rlp in (byte[][]) [wrapped, unwrapped]) + { + int decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => + { + return r.Choice( + (scoped ref RlpReader r) => r.ReadList(static (scoped ref RlpReader r) => r.ReadInt32()), + (scoped ref RlpReader r) => r.ReadInt32() + ); + }); + + decoded.Should().Be(42); + } + } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs index 5169725a38d..1860501eb19 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs @@ -20,6 +20,8 @@ public RlpReader(ReadOnlySpan buffer) _position = 0; } + public bool HasNext => _position < _buffer.Length; + public ReadOnlySpan ReadObject() { ReadOnlySpan result; @@ -92,5 +94,20 @@ public void ReadList(RefRlpReaderAction func) }); } - public bool HasNext => _position < _buffer.Length; + public T Choice(params ReadOnlySpan> alternatives) + { + int startingPosition = _position; + foreach (var f in alternatives) + { + try + { + return f(ref this); + } + catch (Exception) + { + _position = startingPosition; + } + } + throw new RlpReaderException("No alternative succeeded"); + } } From ffa62f443b4cb409990ec6efc982d508d1fc7bd0 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 11:03:12 -0300 Subject: [PATCH 022/106] Cleanup test --- .../RlpReadWriteTest.cs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs index b412f88db54..a579a3ff218 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs @@ -182,18 +182,15 @@ public void InvalidListReading() [Test] public void Choice() { - var unwrapped = Rlp.Write(static w => { w.Write(42); }); - var wrapped = Rlp.Write(static w => w.WriteList(static w => { w.Write(42); })); + RefRlpReaderFunc intReader = (scoped ref RlpReader r) => r.ReadInt32(); + RefRlpReaderFunc wrappedReader = (scoped ref RlpReader r) => r.ReadList(intReader); - foreach (var rlp in (byte[][]) [wrapped, unwrapped]) + var intRlp = Rlp.Write(static w => { w.Write(42); }); + var wrappedIntRlp = Rlp.Write(static w => w.WriteList(static w => { w.Write(42); })); + + foreach (var rlp in (byte[][]) [intRlp, wrappedIntRlp]) { - int decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => - { - return r.Choice( - (scoped ref RlpReader r) => r.ReadList(static (scoped ref RlpReader r) => r.ReadInt32()), - (scoped ref RlpReader r) => r.ReadInt32() - ); - }); + int decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.Choice(wrappedReader, intReader)); decoded.Should().Be(42); } From e626fc95319f1d6c5b518f25a0a6cf7ad3122efa Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 12:58:57 -0300 Subject: [PATCH 023/106] Restructure internals - Primitives are integers, byte sequences, and lists - No need for `IntRlpConverter` (covered by primitives) --- .../Instances/IntRlpConverter.cs | 36 ------------- .../Instances/IntegerRlpConverter.cs | 14 ++++++ .../Instances/StringRlpConverter.cs | 4 +- .../RlpReader.cs | 40 ++++++++++----- .../RlpReaderTest.cs | 2 +- .../RlpWriter.cs | 50 +++++++++++++++---- .../RlpWriterTest.cs | 2 +- 7 files changed, 87 insertions(+), 61 deletions(-) delete mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntegerRlpConverter.cs diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs deleted file mode 100644 index f1fff1773ad..00000000000 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntRlpConverter.cs +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System.Buffers.Binary; - -namespace Nethermind.Serialization.Rlp.Test.Instances; - -public abstract class IntRlpConverter : IRlpConverter -{ - public static int Read(ref RlpReader reader) - { - ReadOnlySpan obj = reader.ReadObject(); - return Int32Primitive.Read(obj); - } - - public static void Write(IRlpWriter writer, int value) - { - if (value < 0x80) - { - writer.WriteByte((byte)value); - } - else - { - Span bytes = stackalloc byte[sizeof(int)]; - BinaryPrimitives.WriteInt32BigEndian(bytes, value); - bytes = bytes.TrimStart((byte)0); - writer.WriteObject(bytes); - } - } -} - -public static class IntRlpConverterExt -{ - public static Int32 ReadInt32(this ref RlpReader reader) => IntRlpConverter.Read(ref reader); - public static void Write(this IRlpWriter writer, int value) => IntRlpConverter.Write(writer, value); -} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntegerRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntegerRlpConverter.cs new file mode 100644 index 00000000000..8fcea3afb47 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntegerRlpConverter.cs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Serialization.Rlp.Test.Instances; + +// NOTE: No need for `Write` overloads since they're covered by generic primitives +// `Read` methods are provided for a consistent API (instead of using generics primitives) +public static class IntegerRlpConverterExt +{ + public static Int16 ReadInt16(this ref RlpReader reader) => reader.ReadInteger(); + public static Int32 ReadInt32(this ref RlpReader reader) => reader.ReadInteger(); + public static Int64 ReadInt64(this ref RlpReader reader) => reader.ReadInteger(); + public static Int128 ReadInt128(this ref RlpReader reader) => reader.ReadInteger(); +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs index 40042707b9a..8f47164ae2c 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs @@ -9,14 +9,14 @@ public abstract class StringRlpConverter : IRlpConverter { public static string Read(ref RlpReader reader) { - ReadOnlySpan obj = reader.ReadObject(); + ReadOnlySpan obj = reader.ReadBytes(); return Encoding.UTF8.GetString(obj); } public static void Write(IRlpWriter writer, string value) { ReadOnlySpan bytes = Encoding.UTF8.GetBytes(value); - writer.WriteObject(bytes); + writer.Write(bytes); } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs index 1860501eb19..2500de531a6 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Numerics; +using System.Runtime.InteropServices; + namespace Nethermind.Serialization.Rlp.Test; public delegate TResult RefRlpReaderFunc(scoped ref RlpReader arg) where TResult : allows ref struct; @@ -22,21 +25,40 @@ public RlpReader(ReadOnlySpan buffer) public bool HasNext => _position < _buffer.Length; - public ReadOnlySpan ReadObject() + public T ReadInteger() where T : IBinaryInteger, ISignedNumber { - ReadOnlySpan result; + ReadOnlySpan bigEndian; var header = _buffer[_position]; if (header < 0x80) { - result = _buffer.Slice(_position++, 1); + bigEndian = _buffer.Slice(_position++, 1); } - else if (header < 0xB8) + else + { + bigEndian = ReadBytes(); + } + + Span buffer = stackalloc byte[Marshal.SizeOf()]; + bigEndian.CopyTo(buffer[^bigEndian.Length..]); + return T.ReadBigEndian(buffer, false); + } + + public ReadOnlySpan ReadBytes() + { + ReadOnlySpan result; + var header = _buffer[_position]; + if (header < 0x80 || header >= 0xC0) + { + throw new RlpReaderException("RLP does not correspond to a byte string"); + } + + if (header < 0xB8) { header -= 0x80; result = _buffer.Slice(++_position, header); _position += header; } - else if (header < 0xC0) + else { header -= 0xB7; ReadOnlySpan binaryLength = _buffer.Slice(++_position, header); @@ -45,10 +67,6 @@ public ReadOnlySpan ReadObject() result = _buffer.Slice(_position, length); _position += length; } - else - { - throw new RlpReaderException("RLP does not correspond to an object"); - } return result; } @@ -56,7 +74,7 @@ public ReadOnlySpan ReadObject() public T ReadList(RefRlpReaderFunc func) { T result; - var header = _buffer[_position]; + var header = _buffer[_position++]; if (header < 0xC0) { throw new RlpReaderException("RLP does not correspond to a list"); @@ -64,7 +82,6 @@ public T ReadList(RefRlpReaderFunc func) if (header < 0xF8) { - _position += 1; var length = header - 0xC0; var reader = new RlpReader(_buffer.Slice(_position, length)); result = func(ref reader); @@ -72,7 +89,6 @@ public T ReadList(RefRlpReaderFunc func) } else { - _position += 1; var lengthOfLength = header - 0xF7; ReadOnlySpan binaryLength = _buffer.Slice(_position, lengthOfLength); _position += lengthOfLength; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs index 0ccbc609363..23ad0612f02 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs @@ -97,7 +97,7 @@ public void ReadSpan() { byte[] source = [0x82, 0x04, 0x00]; - ReadOnlySpan actual = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadObject()); + ReadOnlySpan actual = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadBytes()); ReadOnlySpan expected = [0x04, 0x00]; actual.SequenceEqual(expected).Should().BeTrue(); diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs index d600b4a6082..64ffbbe5652 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs @@ -2,13 +2,15 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Buffers.Binary; +using System.Numerics; +using System.Runtime.InteropServices; namespace Nethermind.Serialization.Rlp.Test; public interface IRlpWriter { - void WriteByte(byte value); - void WriteObject(ReadOnlySpan value); + void Write(T value) where T: IBinaryInteger, ISignedNumber; + void Write(ReadOnlySpan value); void WriteList(Action action); } @@ -23,12 +25,27 @@ public RlpContentWriter(byte[] buffer) _position = 0; } - public void WriteByte(byte value) + public void Write(T value) where T : IBinaryInteger, ISignedNumber { - _buffer[_position++] = value; + var size = Marshal.SizeOf(); + Span bigEndian = stackalloc byte[size]; + value.WriteBigEndian(bigEndian); + bigEndian = bigEndian.TrimStart((byte)0); + + if (bigEndian.Length == 0) + { + _buffer[_position++] = 0; + } else if (bigEndian.Length == 1 && bigEndian[0] < 0x80) + { + _buffer[_position++] = bigEndian[0]; + } + else + { + Write(bigEndian); + } } - public void WriteObject(ReadOnlySpan value) + public void Write(ReadOnlySpan value) { if (value.Length < 55) { @@ -79,16 +96,31 @@ public RlpLengthWriter() Length = 0; } - public void WriteByte(byte value) + public void Write(T value) where T : IBinaryInteger, ISignedNumber { - Length++; + var size = Marshal.SizeOf(); + Span bigEndian = stackalloc byte[size]; + value.WriteBigEndian(bigEndian); + bigEndian = bigEndian.TrimStart((byte)0); + + if (bigEndian.Length == 0) + { + Length++; + } else if (bigEndian.Length == 1 && bigEndian[0] < 0x80) + { + Length++; + } + else + { + Write(bigEndian); + } } - public void WriteObject(ReadOnlySpan value) + public void Write(ReadOnlySpan value) { if (value.Length < 55) { - Length += 1; + Length++; } else { diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs index 6f97494bcba..bd899bcbd65 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs @@ -106,7 +106,7 @@ public void WriteEmptyList() [Test] public void WriteSpan() { - var serialized = Rlp.Write(static writer => { writer.WriteObject([0x04, 0x00]); }); + var serialized = Rlp.Write(static writer => { writer.Write([0x04, 0x00]); }); byte[] expected = [0x82, 0x04, 0x00]; serialized.Should().BeEquivalentTo(expected); From 6f3efbc3e1874b165276b0c456bda32e163f8208 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 12:59:22 -0300 Subject: [PATCH 024/106] Test for deep `Choice` (backtracking) --- .../RlpReadWriteTest.cs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs index a579a3ff218..f98ef8d1dce 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs @@ -195,4 +195,44 @@ public void Choice() decoded.Should().Be(42); } } + + [Test] + public void ChoiceDeep() + { + RefRlpReaderFunc<(string, string, string)> readerA = static (scoped ref RlpReader r) => + { + return r.ReadList(static (scoped ref RlpReader r) => + { + var _1 = r.ReadString(); + var _2 = r.ReadString(); + var _3 = r.ReadString(); + + return (_1, _2, _3); + }); + }; + RefRlpReaderFunc<(string, string, string)> readerB = static (scoped ref RlpReader r) => + { + return r.ReadList(static (scoped ref RlpReader r) => + { + var _1 = r.ReadString(); + var _2 = r.ReadString(); + var _3 = r.ReadInt32(); + + return (_1, _2, _3.ToString()); + }); + }; + + var rlp = Rlp.Write(static w => + { + w.WriteList(static w => + { + w.Write("dog"); + w.Write("cat"); + w.Write(42); + }); + }); + + var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.Choice(readerA, readerB)); + decoded.Should().Be(("dog", "cat", "42")); + } } From 2d2eedabdf21902bd40da85e8b94976408377ee8 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 13:09:46 -0300 Subject: [PATCH 025/106] Move `IRlpConverter` --- .../{Instances => }/IRlpConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/Nethermind/Nethermind.Serialization.Rlp.Test/{Instances => }/IRlpConverter.cs (84%) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/IRlpConverter.cs similarity index 84% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs rename to src/Nethermind/Nethermind.Serialization.Rlp.Test/IRlpConverter.cs index a3f5d11a31c..a09ecea6ec8 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/IRlpConverter.cs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -namespace Nethermind.Serialization.Rlp.Test.Instances; +namespace Nethermind.Serialization.Rlp.Test; public interface IRlpConverter where T : allows ref struct { From 67abc2bbc0a243187711d633562c60800366f21c Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 13:23:22 -0300 Subject: [PATCH 026/106] Demo user-defined record support --- .../Instances/Student.cs | 57 +++++++++++++++++++ .../RlpReadWriteTest.cs | 53 +++++++++++++++-- 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/Student.cs diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/Student.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/Student.cs new file mode 100644 index 00000000000..a8daa1e6a54 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/Student.cs @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Serialization.Rlp.Test.Instances; + +public record Student(string Name, int Age, Dictionary Scores); + +// NOTE: All the following code can be automatically derived using Source Generators +public abstract class StudentRlpConverter : IRlpConverter +{ + public static Student Read(ref RlpReader reader) + { + return reader.ReadList(static (scoped ref RlpReader r) => + { + var name = r.ReadString(); + var age = r.ReadInt32(); + var scores = r.ReadList(static (scoped ref RlpReader r) => + { + Dictionary scores = []; + while (r.HasNext) + { + var subject = r.ReadString(); + var score = r.ReadInt32(); + + scores[subject] = score; + } + + return scores; + }); + + return new Student(name, age, scores); + }); + } + + public static void Write(IRlpWriter writer, Student value) + { + writer.WriteList(r => + { + r.Write(value.Name); + r.Write(value.Age); + r.WriteList(r => + { + foreach (var (subject, score) in value.Scores) + { + r.Write(subject); + r.Write(score); + } + }); + }); + } +} + +public static class StudentExt +{ + public static Student ReadStudent(this ref RlpReader reader) => StudentRlpConverter.Read(ref reader); + public static void Write(this IRlpWriter writer, Student value) => StudentRlpConverter.Write(writer, value); +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs index f98ef8d1dce..d8189830644 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs @@ -203,11 +203,11 @@ public void ChoiceDeep() { return r.ReadList(static (scoped ref RlpReader r) => { - var _1 = r.ReadString(); - var _2 = r.ReadString(); - var _3 = r.ReadString(); + var _1 = r.ReadString(); + var _2 = r.ReadString(); + var _3 = r.ReadString(); - return (_1, _2, _3); + return (_1, _2, _3); }); }; RefRlpReaderFunc<(string, string, string)> readerB = static (scoped ref RlpReader r) => @@ -235,4 +235,49 @@ public void ChoiceDeep() var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.Choice(readerA, readerB)); decoded.Should().Be(("dog", "cat", "42")); } + + [Test] + public void UserDefinedRecord() + { + List students = + [ + new("Ana", 23, new Dictionary + { + { "Math", 7 }, + { "Literature", 9 } + }), + new("Bob", 25, new Dictionary + { + { "Math", 9 }, + { "Literature", 6 } + }), + ]; + + var rlp = Rlp.Write(w => + { + w.WriteList(w => + { + foreach (var student in students) + { + w.Write(student); + } + }); + }); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => + { + return r.ReadList(static (scoped ref RlpReader r) => + { + List result = []; + while (r.HasNext) + { + result.Add(r.ReadStudent()); + } + + return result; + }); + }); + + decoded.Should().BeEquivalentTo(students); + } } From 8f4b02a5e92c3035252ff67ffaa58b240a934895 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 14:18:19 -0300 Subject: [PATCH 027/106] Remove versions --- .../Nethermind.Serialization.Rlp.Test.csproj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj index a15287bde91..62c29abfb0c 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj @@ -9,12 +9,12 @@ - + - - - - + + + + From 9db2ebb85fab0f7b5cf2ead9acdd97d4c756be6e Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 14:24:26 -0300 Subject: [PATCH 028/106] Use `ref struct` over `Interface` - Replace virtual calls with conditional and inner state flag - Refactor call sites --- .../IRlpConverter.cs | 2 +- .../Instances/StringRlpConverter.cs | 4 +- .../Instances/Student.cs | 16 +- .../Nethermind.Serialization.Rlp.Test/Rlp.cs | 10 +- .../RlpReadWriteTest.cs | 38 ++-- .../RlpWriter.cs | 176 +++++++++++------- .../RlpWriterTest.cs | 40 ++-- 7 files changed, 164 insertions(+), 122 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/IRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/IRlpConverter.cs index a09ecea6ec8..9934fa62264 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/IRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/IRlpConverter.cs @@ -6,5 +6,5 @@ namespace Nethermind.Serialization.Rlp.Test; public interface IRlpConverter where T : allows ref struct { public static abstract T Read(ref RlpReader reader); - public static abstract void Write(IRlpWriter writer, T value); + public static abstract void Write(ref RlpWriter writer, T value); } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs index 8f47164ae2c..6235d9ba25f 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs @@ -13,7 +13,7 @@ public static string Read(ref RlpReader reader) return Encoding.UTF8.GetString(obj); } - public static void Write(IRlpWriter writer, string value) + public static void Write(ref RlpWriter writer, string value) { ReadOnlySpan bytes = Encoding.UTF8.GetBytes(value); writer.Write(bytes); @@ -23,5 +23,5 @@ public static void Write(IRlpWriter writer, string value) public static class StringRlpConverterExt { public static string ReadString(this ref RlpReader reader) => StringRlpConverter.Read(ref reader); - public static void Write(this IRlpWriter writer, string value) => StringRlpConverter.Write(writer, value); + public static void Write(this ref RlpWriter writer, string value) => StringRlpConverter.Write(ref writer, value); } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/Student.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/Student.cs index a8daa1e6a54..027d18568f2 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/Student.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/Student.cs @@ -32,18 +32,18 @@ public static Student Read(ref RlpReader reader) }); } - public static void Write(IRlpWriter writer, Student value) + public static void Write(ref RlpWriter writer, Student value) { - writer.WriteList(r => + writer.WriteList((ref RlpWriter w) => { - r.Write(value.Name); - r.Write(value.Age); - r.WriteList(r => + w.Write(value.Name); + w.Write(value.Age); + w.WriteList((ref RlpWriter w) => { foreach (var (subject, score) in value.Scores) { - r.Write(subject); - r.Write(score); + w.Write(subject); + w.Write(score); } }); }); @@ -53,5 +53,5 @@ public static void Write(IRlpWriter writer, Student value) public static class StudentExt { public static Student ReadStudent(this ref RlpReader reader) => StudentRlpConverter.Read(ref reader); - public static void Write(this IRlpWriter writer, Student value) => StudentRlpConverter.Write(writer, value); + public static void Write(this ref RlpWriter writer, Student value) => StudentRlpConverter.Write(ref writer, value); } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs index 9640194db95..c5cd273c9f7 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs @@ -5,13 +5,13 @@ namespace Nethermind.Serialization.Rlp.Test; public static class Rlp { - public static byte[] Write(Action action) + public static byte[] Write(RefRlpWriterAction action) { - var lengthWriter = new RlpLengthWriter(); - action(lengthWriter); + var lengthWriter = RlpWriter.LengthWriter(); + action(ref lengthWriter); var serialized = new byte[lengthWriter.Length]; - var contentWriter = new RlpContentWriter(serialized); - action(contentWriter); + var contentWriter = RlpWriter.ContentWriter(serialized); + action(ref contentWriter); return serialized; } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs index d8189830644..9ce48639af3 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs @@ -11,12 +11,12 @@ public class RlpReadWriteTest [Test] public void HeterogeneousList() { - var rlp = Rlp.Write(static w => + var rlp = Rlp.Write(static (ref RlpWriter w) => { - w.WriteList(static w => + w.WriteList(static (ref RlpWriter w) => { - w.WriteList(static w => { w.Write(42); }); - w.WriteList(static w => + w.WriteList(static (ref RlpWriter w) => { w.Write(42); }); + w.WriteList(static (ref RlpWriter w) => { w.Write("dog"); w.Write("cat"); @@ -47,9 +47,9 @@ public void HeterogeneousList() [Test] public void LongList() { - var rlp = Rlp.Write(static w => + var rlp = Rlp.Write(static (ref RlpWriter w) => { - w.WriteList(static w => + w.WriteList(static (ref RlpWriter w) => { for (int i = 0; i < 100; i++) { @@ -79,16 +79,16 @@ public void LongList() [Test] public void MutlipleLongList() { - var rlp = Rlp.Write(static w => + var rlp = Rlp.Write(static (ref RlpWriter w) => { - w.WriteList(static w => + w.WriteList(static (ref RlpWriter w) => { for (int i = 0; i < 100; i++) { w.Write("dog"); } }); - w.WriteList(static w => + w.WriteList(static (ref RlpWriter w) => { for (int i = 0; i < 50; i++) { @@ -133,9 +133,9 @@ public void MutlipleLongList() [TestCase(2)] public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) { - var rlp = Rlp.Write(root => + var rlp = Rlp.Write((ref RlpWriter root) => { - root.WriteList(w => + root.WriteList((ref RlpWriter w) => { for (int i = 0; i < length; i++) { @@ -164,7 +164,7 @@ public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) [Test] public void InvalidObjectReading() { - var rlp = Rlp.Write(static w => { w.Write(42); }); + var rlp = Rlp.Write(static (ref RlpWriter w) => { w.Write(42); }); Action tryRead = () => Rlp.Read(rlp, (scoped ref RlpReader r) => { r.ReadList((scoped ref RlpReader _) => { }); }); tryRead.Should().Throw(); @@ -173,7 +173,7 @@ public void InvalidObjectReading() [Test] public void InvalidListReading() { - var rlp = Rlp.Write(static w => { w.WriteList(static _ => { }); }); + var rlp = Rlp.Write(static (ref RlpWriter w) => { w.WriteList(static (ref RlpWriter _) => { }); }); Func tryRead = () => Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadInt32()); tryRead.Should().Throw(); @@ -185,8 +185,8 @@ public void Choice() RefRlpReaderFunc intReader = (scoped ref RlpReader r) => r.ReadInt32(); RefRlpReaderFunc wrappedReader = (scoped ref RlpReader r) => r.ReadList(intReader); - var intRlp = Rlp.Write(static w => { w.Write(42); }); - var wrappedIntRlp = Rlp.Write(static w => w.WriteList(static w => { w.Write(42); })); + var intRlp = Rlp.Write(static (ref RlpWriter w) => { w.Write(42); }); + var wrappedIntRlp = Rlp.Write(static (ref RlpWriter w) => w.WriteList(static (ref RlpWriter w) => { w.Write(42); })); foreach (var rlp in (byte[][]) [intRlp, wrappedIntRlp]) { @@ -222,9 +222,9 @@ public void ChoiceDeep() }); }; - var rlp = Rlp.Write(static w => + var rlp = Rlp.Write(static (ref RlpWriter w) => { - w.WriteList(static w => + w.WriteList(static (ref RlpWriter w) => { w.Write("dog"); w.Write("cat"); @@ -253,9 +253,9 @@ public void UserDefinedRecord() }), ]; - var rlp = Rlp.Write(w => + var rlp = Rlp.Write((ref RlpWriter w) => { - w.WriteList(w => + w.WriteList((ref RlpWriter w) => { foreach (var student in students) { diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs index 64ffbbe5652..fe27bbe44ea 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs @@ -7,25 +7,71 @@ namespace Nethermind.Serialization.Rlp.Test; -public interface IRlpWriter -{ - void Write(T value) where T: IBinaryInteger, ISignedNumber; - void Write(ReadOnlySpan value); - void WriteList(Action action); -} +public delegate void RefRlpWriterAction(ref RlpWriter arg); -public sealed class RlpContentWriter : IRlpWriter +public ref struct RlpWriter { - private readonly byte[] _buffer; + private const bool LengthMode = false; + private const bool ContentMode = true; + + private bool _mode; + + public int Length { get; private set; } + + private byte[] _buffer; private int _position; - public RlpContentWriter(byte[] buffer) + public static RlpWriter LengthWriter() + { + return new RlpWriter + { + _mode = LengthMode + }; + } + + public static RlpWriter ContentWriter(byte[] buffer) { - _buffer = buffer; - _position = 0; + return new RlpWriter + { + _mode = ContentMode, + _buffer = buffer + }; } public void Write(T value) where T : IBinaryInteger, ISignedNumber + { + switch (_mode) + { + case LengthMode: + LengthWrite(value); + break; + case ContentMode: + ContentWrite(value); + break; + } + } + + private void LengthWrite(T value) where T : IBinaryInteger, ISignedNumber + { + var size = Marshal.SizeOf(); + Span bigEndian = stackalloc byte[size]; + value.WriteBigEndian(bigEndian); + bigEndian = bigEndian.TrimStart((byte)0); + + if (bigEndian.Length == 0) + { + Length++; + } else if (bigEndian.Length == 1 && bigEndian[0] < 0x80) + { + Length++; + } + else + { + LengthWrite(bigEndian); + } + } + + private void ContentWrite(T value) where T : IBinaryInteger, ISignedNumber { var size = Marshal.SizeOf(); Span bigEndian = stackalloc byte[size]; @@ -41,112 +87,108 @@ public void Write(T value) where T : IBinaryInteger, ISignedNumber } else { - Write(bigEndian); + ContentWrite(bigEndian); } } public void Write(ReadOnlySpan value) + { + switch (_mode) + { + case LengthMode: + LengthWrite(value); + break; + case ContentMode: + ContentWrite(value); + break; + } + } + + private void LengthWrite(scoped ReadOnlySpan value) { if (value.Length < 55) { - _buffer[_position++] = (byte)(0x80 + value.Length); + Length++; } else { Span binaryLength = stackalloc byte[sizeof(int)]; BinaryPrimitives.WriteInt32BigEndian(binaryLength, value.Length); binaryLength = binaryLength.TrimStart((byte)0); - _buffer[_position++] = (byte)(0xB7 + binaryLength.Length); - binaryLength.CopyTo(_buffer.AsSpan()[_position..]); - _position += binaryLength.Length; + Length += 1 + binaryLength.Length; } - value.CopyTo(_buffer.AsSpan()[_position..]); - _position += value.Length; + Length += value.Length; } - public void WriteList(Action action) + private void ContentWrite(scoped ReadOnlySpan value) { - var lengthWriter = new RlpLengthWriter(); - action(lengthWriter); - if (lengthWriter.Length < 55) + if (value.Length < 55) { - _buffer[_position++] = (byte)(0xC0 + lengthWriter.Length); + _buffer[_position++] = (byte)(0x80 + value.Length); } else { - Span binaryLength = stackalloc byte[sizeof(Int32)]; - BinaryPrimitives.WriteInt32BigEndian(binaryLength, lengthWriter.Length); + Span binaryLength = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32BigEndian(binaryLength, value.Length); binaryLength = binaryLength.TrimStart((byte)0); - _buffer[_position++] = (byte)(0xF7 + binaryLength.Length); + _buffer[_position++] = (byte)(0xB7 + binaryLength.Length); binaryLength.CopyTo(_buffer.AsSpan()[_position..]); _position += binaryLength.Length; } - action(this); - } -} - -public sealed class RlpLengthWriter : IRlpWriter -{ - public int Length { get; private set; } - - public RlpLengthWriter() - { - Length = 0; + value.CopyTo(_buffer.AsSpan()[_position..]); + _position += value.Length; } - public void Write(T value) where T : IBinaryInteger, ISignedNumber + public void WriteList(RefRlpWriterAction action) { - var size = Marshal.SizeOf(); - Span bigEndian = stackalloc byte[size]; - value.WriteBigEndian(bigEndian); - bigEndian = bigEndian.TrimStart((byte)0); - - if (bigEndian.Length == 0) - { - Length++; - } else if (bigEndian.Length == 1 && bigEndian[0] < 0x80) - { - Length++; - } - else + switch (_mode) { - Write(bigEndian); + case LengthMode: + LengthWriteList(action); + break; + case ContentMode: + ContentWriteList(action); + break; } } - public void Write(ReadOnlySpan value) + private void LengthWriteList(RefRlpWriterAction action) { - if (value.Length < 55) + var inner = LengthWriter(); + action(ref inner); + if (inner.Length < 55) { - Length++; + Length += 1 + inner.Length; } else { - Span binaryLength = stackalloc byte[sizeof(int)]; - BinaryPrimitives.WriteInt32BigEndian(binaryLength, value.Length); + Span binaryLength = stackalloc byte[sizeof(Int32)]; + BinaryPrimitives.WriteInt32BigEndian(binaryLength, inner.Length); binaryLength = binaryLength.TrimStart((byte)0); - Length += 1 + binaryLength.Length; + Length += 1 + inner.Length + binaryLength.Length; } - - Length += value.Length; } - public void WriteList(Action action) + private void ContentWriteList(RefRlpWriterAction action) { - var inner = new RlpLengthWriter(); - action(inner); - if (inner.Length < 55) + var lengthWriter = LengthWriter(); + action(ref lengthWriter); + if (lengthWriter.Length < 55) { - Length += 1 + inner.Length; + _buffer[_position++] = (byte)(0xC0 + lengthWriter.Length); } else { Span binaryLength = stackalloc byte[sizeof(Int32)]; - BinaryPrimitives.WriteInt32BigEndian(binaryLength, inner.Length); + BinaryPrimitives.WriteInt32BigEndian(binaryLength, lengthWriter.Length); binaryLength = binaryLength.TrimStart((byte)0); - Length += 1 + inner.Length + binaryLength.Length; + _buffer[_position++] = (byte)(0xF7 + binaryLength.Length); + binaryLength.CopyTo(_buffer.AsSpan()[_position..]); + _position += binaryLength.Length; } + + action(ref this); } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs index bd899bcbd65..d17c3253578 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs @@ -9,7 +9,7 @@ public class RlpWriterTest [Test] public void WriteShortString() { - var serialized = Rlp.Write(static writer => { writer.Write("dog"); }); + var serialized = Rlp.Write(static (ref RlpWriter w) => { w.Write("dog"); }); byte[] expected = [0x83, (byte)'d', (byte)'o', (byte)'g']; serialized.Should().BeEquivalentTo(expected); @@ -18,7 +18,7 @@ public void WriteShortString() [Test] public void WriteEmptyString() { - var serialized = Rlp.Write(static writer => { writer.Write(""); }); + var serialized = Rlp.Write(static (ref RlpWriter w) => { w.Write(""); }); byte[] expected = [0x80]; serialized.Should().BeEquivalentTo(expected); @@ -27,9 +27,9 @@ public void WriteEmptyString() [Test] public void WriteLongString() { - var serialized = Rlp.Write(writer => + var serialized = Rlp.Write(static (ref RlpWriter w) => { - writer.Write("Lorem ipsum dolor sit amet, consectetur adipisicing elit"); + w.Write("Lorem ipsum dolor sit amet, consectetur adipisicing elit"); }); byte[] expected = [0xb8, 0x38, .."Lorem ipsum dolor sit amet, consectetur adipisicing elit"u8]; @@ -42,7 +42,7 @@ public void WriteInteger_1Component() for (int i = 0; i < 0x80; i++) { var integer = i; - var serialized = Rlp.Write(writer => { writer.Write(integer); }); + var serialized = Rlp.Write((ref RlpWriter w) => { w.Write(integer); }); byte[] expected = [(byte)integer]; serialized.Should().BeEquivalentTo(expected); @@ -56,7 +56,7 @@ public void WriteInteger_2Components() for (int i = 0x80; i < 0x0100; i++) { var integer = i; - var serialized = Rlp.Write(writer => { writer.Write(integer); }); + var serialized = Rlp.Write((ref RlpWriter w) => { w.Write(integer); }); expected[1] = (byte)integer; serialized.Should().BeEquivalentTo(expected); @@ -70,7 +70,7 @@ public void WriteInteger_3Components() for (int i = 0x100; i < 0xFFFF; i++) { var integer = i; - var serialized = Rlp.Write(writer => { writer.Write(integer); }); + var serialized = Rlp.Write((ref RlpWriter w) => { w.Write(integer); }); expected[1] = (byte)((integer & 0xFF00) >> 8); expected[2] = (byte)((integer & 0x00FF) >> 0); @@ -81,12 +81,12 @@ public void WriteInteger_3Components() [Test] public void WriteStringList() { - var serialized = Rlp.Write(static writer => + var serialized = Rlp.Write(static (ref RlpWriter w) => { - writer.WriteList(static writer => + w.WriteList(static (ref RlpWriter w) => { - writer.Write("cat"); - writer.Write("dog"); + w.Write("cat"); + w.Write("dog"); }); }); @@ -97,7 +97,7 @@ public void WriteStringList() [Test] public void WriteEmptyList() { - var serialized = Rlp.Write(static writer => { writer.WriteList(static _ => { }); }); + var serialized = Rlp.Write(static (ref RlpWriter w) => { w.WriteList(static (ref RlpWriter _) => { }); }); byte[] expected = [0xc0]; serialized.Should().BeEquivalentTo(expected); @@ -106,7 +106,7 @@ public void WriteEmptyList() [Test] public void WriteSpan() { - var serialized = Rlp.Write(static writer => { writer.Write([0x04, 0x00]); }); + var serialized = Rlp.Write(static (ref RlpWriter w) => { w.Write([0x04, 0x00]); }); byte[] expected = [0x82, 0x04, 0x00]; serialized.Should().BeEquivalentTo(expected); @@ -115,16 +115,16 @@ public void WriteSpan() [Test] public void WriteSetTheoreticalRepresentation() { - var serialized = Rlp.Write(static writer => + var serialized = Rlp.Write(static (ref RlpWriter w) => { - writer.WriteList(static root => + w.WriteList(static (ref RlpWriter w) => { - root.WriteList(static _ => { }); - root.WriteList(static w => { w.WriteList(static _ => { }); }); - root.WriteList(static w => + w.WriteList(static (ref RlpWriter _) => { }); + w.WriteList(static (ref RlpWriter w) => { w.WriteList(static (ref RlpWriter _) => { }); }); + w.WriteList(static (ref RlpWriter w) => { - w.WriteList(static _ => { }); - w.WriteList(static w => { w.WriteList(static _ => { }); }); + w.WriteList(static (ref RlpWriter _) => { }); + w.WriteList(static (ref RlpWriter w) => { w.WriteList(static (ref RlpWriter _) => { }); }); }); }); }); From 22238451d9f55580535d106776ab03d1f3af2f19 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 14:35:30 -0300 Subject: [PATCH 029/106] Consistent error --- src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs index 2500de531a6..7e894ac5e2c 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs @@ -124,6 +124,6 @@ public T Choice(params ReadOnlySpan> alternatives) _position = startingPosition; } } - throw new RlpReaderException("No alternative succeeded"); + throw new RlpReaderException("RLP does not correspond to any alternative"); } } From 51fd03856b1a78314b548ec85461c6fb83a761da Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 16:33:29 -0300 Subject: [PATCH 030/106] Split tests from library - Rename to `FastRlp` to avoid conflicts --- .../Nethermind.Serialization.FastRlp.Test.csproj} | 4 ++++ .../RlpReadWriteTest.cs | 4 ++-- .../RlpReaderTest.cs | 4 ++-- .../RlpWriterTest.cs | 9 ++++++--- .../Student.cs | 4 +++- .../IRlpConverter.cs | 2 +- .../Instances/IntegerRlpConverter.cs | 4 +++- .../Instances/StringRlpConverter.cs | 3 ++- .../Int32Primitive.cs | 5 +++-- .../Nethermind.Serialization.FastRlp.csproj | 7 +++++++ .../Rlp.cs | 4 +++- .../RlpReader.cs | 3 ++- .../RlpWriter.cs | 3 ++- src/Nethermind/Nethermind.sln | 8 +++++++- 14 files changed, 47 insertions(+), 17 deletions(-) rename src/Nethermind/{Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj => Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj} (83%) rename src/Nethermind/{Nethermind.Serialization.Rlp.Test => Nethermind.Serialization.FastRlp.Test}/RlpReadWriteTest.cs (98%) rename src/Nethermind/{Nethermind.Serialization.Rlp.Test => Nethermind.Serialization.FastRlp.Test}/RlpReaderTest.cs (97%) rename src/Nethermind/{Nethermind.Serialization.Rlp.Test => Nethermind.Serialization.FastRlp.Test}/RlpWriterTest.cs (94%) rename src/Nethermind/{Nethermind.Serialization.Rlp.Test/Instances => Nethermind.Serialization.FastRlp.Test}/Student.cs (94%) rename src/Nethermind/{Nethermind.Serialization.Rlp.Test => Nethermind.Serialization.FastRlp}/IRlpConverter.cs (86%) rename src/Nethermind/{Nethermind.Serialization.Rlp.Test => Nethermind.Serialization.FastRlp}/Instances/IntegerRlpConverter.cs (91%) rename src/Nethermind/{Nethermind.Serialization.Rlp.Test => Nethermind.Serialization.FastRlp}/Instances/StringRlpConverter.cs (92%) rename src/Nethermind/{Nethermind.Serialization.Rlp.Test => Nethermind.Serialization.FastRlp}/Int32Primitive.cs (88%) create mode 100644 src/Nethermind/Nethermind.Serialization.FastRlp/Nethermind.Serialization.FastRlp.csproj rename src/Nethermind/{Nethermind.Serialization.Rlp.Test => Nethermind.Serialization.FastRlp}/Rlp.cs (94%) rename src/Nethermind/{Nethermind.Serialization.Rlp.Test => Nethermind.Serialization.FastRlp}/RlpReader.cs (98%) rename src/Nethermind/{Nethermind.Serialization.Rlp.Test => Nethermind.Serialization.FastRlp}/RlpWriter.cs (98%) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj similarity index 83% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj rename to src/Nethermind/Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj index 62c29abfb0c..7e06a02335a 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Nethermind.Serialization.Rlp.Test.csproj +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj @@ -21,4 +21,8 @@ + + + + diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs similarity index 98% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs rename to src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs index 9ce48639af3..25dbb347198 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using FluentAssertions; -using Nethermind.Serialization.Rlp.Test.Instances; +using Nethermind.Serialization.FastRlp.Instances; -namespace Nethermind.Serialization.Rlp.Test; +namespace Nethermind.Serialization.FastRlp.Test; public class RlpReadWriteTest { diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs similarity index 97% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs rename to src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs index 23ad0612f02..d97c67b2292 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using FluentAssertions; -using Nethermind.Serialization.Rlp.Test.Instances; +using Nethermind.Serialization.FastRlp.Instances; -namespace Nethermind.Serialization.Rlp.Test; +namespace Nethermind.Serialization.FastRlp.Test; public class RlpReaderTest { diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs similarity index 94% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs rename to src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs index d17c3253578..daa59b9c604 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs @@ -1,7 +1,10 @@ -using FluentAssertions; -using Nethermind.Serialization.Rlp.Test.Instances; +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only -namespace Nethermind.Serialization.Rlp.Test; +using FluentAssertions; +using Nethermind.Serialization.FastRlp.Instances; + +namespace Nethermind.Serialization.FastRlp.Test; [Parallelizable(ParallelScope.All)] public class RlpWriterTest diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/Student.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs similarity index 94% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/Student.cs rename to src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs index 027d18568f2..85a6971d1fb 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/Student.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -namespace Nethermind.Serialization.Rlp.Test.Instances; +using Nethermind.Serialization.FastRlp.Instances; + +namespace Nethermind.Serialization.FastRlp.Test; public record Student(string Name, int Age, Dictionary Scores); diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/IRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/IRlpConverter.cs similarity index 86% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/IRlpConverter.cs rename to src/Nethermind/Nethermind.Serialization.FastRlp/IRlpConverter.cs index 9934fa62264..4a3a8b8a415 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/IRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/IRlpConverter.cs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -namespace Nethermind.Serialization.Rlp.Test; +namespace Nethermind.Serialization.FastRlp; public interface IRlpConverter where T : allows ref struct { diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntegerRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/IntegerRlpConverter.cs similarity index 91% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntegerRlpConverter.cs rename to src/Nethermind/Nethermind.Serialization.FastRlp/Instances/IntegerRlpConverter.cs index 8fcea3afb47..2af8b5344ae 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/IntegerRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/IntegerRlpConverter.cs @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -namespace Nethermind.Serialization.Rlp.Test.Instances; +using System; + +namespace Nethermind.Serialization.FastRlp.Instances; // NOTE: No need for `Write` overloads since they're covered by generic primitives // `Read` methods are provided for a consistent API (instead of using generics primitives) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/StringRlpConverter.cs similarity index 92% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs rename to src/Nethermind/Nethermind.Serialization.FastRlp/Instances/StringRlpConverter.cs index 6235d9ba25f..0d5ecff6354 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Instances/StringRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/StringRlpConverter.cs @@ -1,9 +1,10 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Text; -namespace Nethermind.Serialization.Rlp.Test.Instances; +namespace Nethermind.Serialization.FastRlp.Instances; public abstract class StringRlpConverter : IRlpConverter { diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Int32Primitive.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Int32Primitive.cs similarity index 88% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/Int32Primitive.cs rename to src/Nethermind/Nethermind.Serialization.FastRlp/Int32Primitive.cs index 56882592e1c..a028914072f 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Int32Primitive.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Int32Primitive.cs @@ -1,11 +1,12 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Buffers.Binary; -namespace Nethermind.Serialization.Rlp.Test; +namespace Nethermind.Serialization.FastRlp; -public static class Int32Primitive +internal static class Int32Primitive { /// /// Reads a from the beginning of a read-only span of bytes, as big endian. diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Nethermind.Serialization.FastRlp.csproj b/src/Nethermind/Nethermind.Serialization.FastRlp/Nethermind.Serialization.FastRlp.csproj new file mode 100644 index 00000000000..f1e6730cc6e --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Nethermind.Serialization.FastRlp.csproj @@ -0,0 +1,7 @@ + + + + enable + + + diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs similarity index 94% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs rename to src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs index c5cd273c9f7..257fce868f2 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -namespace Nethermind.Serialization.Rlp.Test; +using System; + +namespace Nethermind.Serialization.FastRlp; public static class Rlp { diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs similarity index 98% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs rename to src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs index 7e894ac5e2c..70bb3deee32 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs @@ -1,10 +1,11 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Numerics; using System.Runtime.InteropServices; -namespace Nethermind.Serialization.Rlp.Test; +namespace Nethermind.Serialization.FastRlp; public delegate TResult RefRlpReaderFunc(scoped ref RlpReader arg) where TResult : allows ref struct; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs similarity index 98% rename from src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs rename to src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs index fe27bbe44ea..9a571172462 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Test/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs @@ -1,11 +1,12 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Buffers.Binary; using System.Numerics; using System.Runtime.InteropServices; -namespace Nethermind.Serialization.Rlp.Test; +namespace Nethermind.Serialization.FastRlp; public delegate void RefRlpWriterAction(ref RlpWriter arg); diff --git a/src/Nethermind/Nethermind.sln b/src/Nethermind/Nethermind.sln index 1e3761fdd31..4f29e6cb856 100644 --- a/src/Nethermind/Nethermind.sln +++ b/src/Nethermind/Nethermind.sln @@ -226,7 +226,9 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Shutter", "Nethe EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Shutter.Test", "Nethermind.Shutter.Test\Nethermind.Shutter.Test.csproj", "{CEA1C413-A96C-4339-AC1C-839B603DECC8}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.Rlp.Test", "Nethermind.Serialization.Rlp.Test\Nethermind.Serialization.Rlp.Test.csproj", "{C8A91B54-F9CA-4211-BE16-F7A0B38223EB}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.FastRlp.Test", "Nethermind.Serialization.FastRlp.Test\Nethermind.Serialization.FastRlp.Test.csproj", "{C8A91B54-F9CA-4211-BE16-F7A0B38223EB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.FastRlp", "Nethermind.Serialization.FastRlp\Nethermind.Serialization.FastRlp.csproj", "{3D358FD8-E047-4770-BCDF-8FBBBBDDE03A}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -630,6 +632,10 @@ Global {C8A91B54-F9CA-4211-BE16-F7A0B38223EB}.Debug|Any CPU.Build.0 = Debug|Any CPU {C8A91B54-F9CA-4211-BE16-F7A0B38223EB}.Release|Any CPU.ActiveCfg = Release|Any CPU {C8A91B54-F9CA-4211-BE16-F7A0B38223EB}.Release|Any CPU.Build.0 = Release|Any CPU + {3D358FD8-E047-4770-BCDF-8FBBBBDDE03A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3D358FD8-E047-4770-BCDF-8FBBBBDDE03A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3D358FD8-E047-4770-BCDF-8FBBBBDDE03A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3D358FD8-E047-4770-BCDF-8FBBBBDDE03A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 5fad6ef80b5d0d462cc07e532ed8da63b8424528 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 17:27:06 -0300 Subject: [PATCH 031/106] Initial source generator for `RlpSerializable` --- src/Nethermind/Directory.Packages.props | 1 + ...ind.Serialization.FastRlp.Generator.csproj | 13 ++ .../RlpSourceGenerator.cs | 174 ++++++++++++++++++ ...thermind.Serialization.FastRlp.Test.csproj | 1 + .../Player.cs | 10 + src/Nethermind/Nethermind.sln | 6 + 6 files changed, 205 insertions(+) create mode 100644 src/Nethermind/Nethermind.Serialization.FastRlp.Generator/Nethermind.Serialization.FastRlp.Generator.csproj create mode 100644 src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs create mode 100644 src/Nethermind/Nethermind.Serialization.FastRlp.Test/Player.cs diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props index f3cb0784482..4090b304488 100644 --- a/src/Nethermind/Directory.Packages.props +++ b/src/Nethermind/Directory.Packages.props @@ -36,6 +36,7 @@ + diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/Nethermind.Serialization.FastRlp.Generator.csproj b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/Nethermind.Serialization.FastRlp.Generator.csproj new file mode 100644 index 00000000000..b47244e5b00 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/Nethermind.Serialization.FastRlp.Generator.csproj @@ -0,0 +1,13 @@ + + + + netstandard2.1 + enable + true + + + + + + + diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs new file mode 100644 index 00000000000..5cc0e88ef3e --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs @@ -0,0 +1,174 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Text; +using System.Text; + +namespace Nethermind.Serialization.FastRlp.Generator; + +[AttributeUsage(AttributeTargets.Class)] +public sealed class RlpSerializable : Attribute; + +/// +/// A source generator that finds all records with [RlpSerializable] and +/// generates an abstract RlpConverter class with a Read method. +/// +[Generator] +public sealed class RlpSourceGenerator : IIncrementalGenerator +{ + public void Initialize(IncrementalGeneratorInitializationContext context) + { + var provider = context.SyntaxProvider.CreateSyntaxProvider( + predicate: static (s, _) => s is RecordDeclarationSyntax, + transform: static (ctx, _) => (RecordDeclarationSyntax)ctx.Node + ); + + var compilation = context.CompilationProvider.Combine(provider.Collect()); + + context.RegisterSourceOutput(compilation, Execute); + } + + private void Execute(SourceProductionContext context, + (Compilation Compilation, ImmutableArray RecordsDeclarationSyntaxes) p) + { + // For each record with the attribute, generate the RlpConverter class + foreach (var recordDecl in p.RecordsDeclarationSyntaxes) + { + // Check if the record has the [RlpSerializable] attribute + SemanticModel semanticModel = p.Compilation.GetSemanticModel(recordDecl.SyntaxTree); + ISymbol? symbol = semanticModel.GetDeclaredSymbol(recordDecl); + if (symbol is null) continue; + + // If not, skip the record + var attributes = symbol.GetAttributes(); + if (!attributes.Any(a => + a.AttributeClass?.Name == nameof(RlpSerializable) || + a.AttributeClass?.ToDisplayString() == nameof(RlpSerializable))) + { + continue; + } + + // Extract the `using` statements from the file containing the record + var usingDirectiveSyntaxes = recordDecl + .SyntaxTree + .GetRoot() + .DescendantNodes() + .OfType() + .Select(syntax => syntax.Name!.ToString()); + + // Extract the record name with namespace if needed + var recordName = symbol.Name; + var fullTypeName = symbol.ToDisplayString(); + + // Gather recursively all members that are fields or primary constructor parameters + // so we can read them in the same order they are declared. + var parameters = GetRecordParameters(recordDecl); + + // Build the converter class source + var generatedCode = GenerateConverterClass(usingDirectiveSyntaxes, fullTypeName, recordName, parameters); + + // Add to the compilation + context.AddSource($"{recordName}RlpConverter.g.cs", SourceText.From(generatedCode, Encoding.UTF8)); + } + } + + /// + /// Gathers the record’s primary constructor parameters and public fields/properties + /// in the order they appear in the record declaration. + /// + private static List<(string Name, string TypeName)> GetRecordParameters(RecordDeclarationSyntax recordDecl) + { + List<(string, string)> parameters = []; + + // Primary constructor parameters + if (recordDecl.ParameterList is not null) + { + foreach (var param in recordDecl.ParameterList.Parameters) + { + var paramName = param.Identifier.Text; + var paramType = param.Type?.ToString() ?? "object"; + parameters.Add((paramName, paramType)); + } + } + + return parameters; + } + + private static string GenerateConverterClass( + IEnumerable usingDirectives, + string fullTypeName, + string recordName, + List<(string Name, string TypeName)> parameters) + { + var sb = new StringBuilder(); + + sb.AppendLine("// "); + sb.AppendLine("using System;"); + sb.AppendLine("using Nethermind.Serialization.FastRlp;"); + sb.AppendLine("using Nethermind.Serialization.FastRlp.Instances;"); + foreach (var usingDirective in usingDirectives) + { + sb.AppendLine($"using {usingDirective};"); + } + sb.AppendLine(); + sb.AppendLine("namespace Nethermind.Serialization.FastRlp.Derived;"); + sb.AppendLine(""); + sb.AppendLine($"public abstract class {recordName}RlpConverter : IRlpConverter<{fullTypeName}>"); + sb.AppendLine("{"); + sb.AppendLine($"public static void Write(ref RlpWriter writer, {fullTypeName} value)"); + sb.AppendLine("{"); + sb.AppendLine(" throw new NotImplementedException();"); + sb.AppendLine("}"); + sb.AppendLine($"public static {fullTypeName} Read(ref RlpReader reader)"); + sb.AppendLine("{"); + sb.AppendLine(" return reader.ReadList(static (scoped ref RlpReader r) =>"); + sb.AppendLine(" {"); + + foreach (var (name, typeName) in parameters) + { + var readCall = MapTypeToReadCall(typeName); + sb.AppendLine($" var {name} = r.{readCall};"); + } + + // Reconstruct the record + sb.Append($" return new {fullTypeName}("); + for (int i = 0; i < parameters.Count; i++) + { + sb.Append(parameters[i].Name); + if (i < parameters.Count - 1) sb.Append(", "); + } + + sb.AppendLine(");"); + + sb.AppendLine(" });"); + sb.AppendLine("}"); + sb.AppendLine("}"); + + return sb.ToString(); + } + + /// + /// Map the type name to the appropriate Read method on the `RlpReader` + /// Extend this mapping for more types as needed. + /// + private static string MapTypeToReadCall(string typeName) + { + return typeName switch + { + "byte[]" or "System.Byte[]" or "Span" or "System.Span" or "System.ReadOnlySpan" => "ReadBytes()", + "int" => "ReadInt32()", + _ => $"Read{typeName.Capitalize()}()" + }; + } +} + +public static class StringExt +{ + public static string Capitalize (this string str) => str[0].ToString().ToUpper() + str[1..]; +} diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj index 7e06a02335a..4f49e9de0d2 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj @@ -23,6 +23,7 @@ + diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Player.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Player.cs new file mode 100644 index 00000000000..21e2a925fbb --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Player.cs @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Serialization.FastRlp.Generator; + +namespace Nethermind.Serialization.FastRlp.Test; + +[RlpSerializable] +public record Player(int Id, string Username); + diff --git a/src/Nethermind/Nethermind.sln b/src/Nethermind/Nethermind.sln index 4f29e6cb856..69cf44f6bf5 100644 --- a/src/Nethermind/Nethermind.sln +++ b/src/Nethermind/Nethermind.sln @@ -230,6 +230,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.Fa EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.FastRlp", "Nethermind.Serialization.FastRlp\Nethermind.Serialization.FastRlp.csproj", "{3D358FD8-E047-4770-BCDF-8FBBBBDDE03A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.FastRlp.Generator", "Nethermind.Serialization.FastRlp.Generator\Nethermind.Serialization.FastRlp.Generator.csproj", "{838687B6-9915-4BD2-98CC-79A4130B89B0}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -636,6 +638,10 @@ Global {3D358FD8-E047-4770-BCDF-8FBBBBDDE03A}.Debug|Any CPU.Build.0 = Debug|Any CPU {3D358FD8-E047-4770-BCDF-8FBBBBDDE03A}.Release|Any CPU.ActiveCfg = Release|Any CPU {3D358FD8-E047-4770-BCDF-8FBBBBDDE03A}.Release|Any CPU.Build.0 = Release|Any CPU + {838687B6-9915-4BD2-98CC-79A4130B89B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {838687B6-9915-4BD2-98CC-79A4130B89B0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {838687B6-9915-4BD2-98CC-79A4130B89B0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {838687B6-9915-4BD2-98CC-79A4130B89B0}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From c22f1df01aa7b77b82d355f72f293fb79904e2bd Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 17:44:06 -0300 Subject: [PATCH 032/106] Implement `Write` --- .../RlpSourceGenerator.cs | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs index 5cc0e88ef3e..2b108dc351d 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs @@ -22,6 +22,9 @@ public sealed class RlpSerializable : Attribute; [Generator] public sealed class RlpSourceGenerator : IIncrementalGenerator { + private const string Version = "0.1"; + private const string GeneratedCodeAttribute = $"""[GeneratedCode("{nameof(RlpSourceGenerator)}", "{Version}")]"""; + public void Initialize(IncrementalGeneratorInitializationContext context) { var provider = context.SyntaxProvider.CreateSyntaxProvider( @@ -110,6 +113,7 @@ private static string GenerateConverterClass( sb.AppendLine("// "); sb.AppendLine("using System;"); + sb.AppendLine("using System.CodeDom.Compiler;"); sb.AppendLine("using Nethermind.Serialization.FastRlp;"); sb.AppendLine("using Nethermind.Serialization.FastRlp.Instances;"); foreach (var usingDirective in usingDirectives) @@ -119,35 +123,53 @@ private static string GenerateConverterClass( sb.AppendLine(); sb.AppendLine("namespace Nethermind.Serialization.FastRlp.Derived;"); sb.AppendLine(""); + sb.AppendLine(GeneratedCodeAttribute); sb.AppendLine($"public abstract class {recordName}RlpConverter : IRlpConverter<{fullTypeName}>"); sb.AppendLine("{"); - sb.AppendLine($"public static void Write(ref RlpWriter writer, {fullTypeName} value)"); + + // `Write` method + sb.AppendLine($"public static void Write(ref RlpWriter w, {fullTypeName} value)"); + sb.AppendLine("{"); + sb.AppendLine("w.WriteList((ref RlpWriter w) => "); sb.AppendLine("{"); - sb.AppendLine(" throw new NotImplementedException();"); + + foreach (var (name, _) in parameters) + { + sb.AppendLine($"w.Write(value.{name});"); + } + sb.AppendLine("});"); sb.AppendLine("}"); - sb.AppendLine($"public static {fullTypeName} Read(ref RlpReader reader)"); + + // `Read` method + sb.AppendLine($"public static {fullTypeName} Read(ref RlpReader r)"); + sb.AppendLine("{"); + sb.AppendLine("return r.ReadList(static (scoped ref RlpReader r) =>"); sb.AppendLine("{"); - sb.AppendLine(" return reader.ReadList(static (scoped ref RlpReader r) =>"); - sb.AppendLine(" {"); foreach (var (name, typeName) in parameters) { var readCall = MapTypeToReadCall(typeName); - sb.AppendLine($" var {name} = r.{readCall};"); + sb.AppendLine($"var {name} = r.{readCall};"); } - // Reconstruct the record - sb.Append($" return new {fullTypeName}("); + sb.Append($"return new {fullTypeName}("); for (int i = 0; i < parameters.Count; i++) { sb.Append(parameters[i].Name); if (i < parameters.Count - 1) sb.Append(", "); } - sb.AppendLine(");"); - sb.AppendLine(" });"); + sb.AppendLine("});"); + sb.AppendLine("}"); sb.AppendLine("}"); + + sb.AppendLine(); + sb.AppendLine(GeneratedCodeAttribute); + sb.AppendLine($"public static class {recordName}Ext"); + sb.AppendLine("{"); + sb.AppendLine($"public static {fullTypeName} Read{recordName}(this ref RlpReader reader) => {recordName}RlpConverter.Read(ref reader);"); + sb.AppendLine($"public static void Write(this ref RlpWriter writer, {fullTypeName} value) => {recordName}RlpConverter.Write(ref writer, value);"); sb.AppendLine("}"); return sb.ToString(); From 70c78e2974f79726fa57b9a3f9c2cea9e6e48aa6 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 17:44:20 -0300 Subject: [PATCH 033/106] Test for Source Generated instances --- .../RlpDerivedTest.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs new file mode 100644 index 00000000000..faf96f4f996 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Serialization.FastRlp.Derived; + +namespace Nethermind.Serialization.FastRlp.Test; + +public class RlpDerivedTest +{ + [Test] + public void ReadDerivedRecord() + { + var player = new Player(42, "SuperUser"); + ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); + + var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadPlayer()); + decoded.Should().Be(player); + } +} From f11ff9a8d7fbd91adec4d10afca347c913aa9515 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 17:46:41 -0300 Subject: [PATCH 034/106] Move record --- .../Nethermind.Serialization.FastRlp.Test/Player.cs | 10 ---------- .../RlpDerivedTest.cs | 8 +++++++- 2 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 src/Nethermind/Nethermind.Serialization.FastRlp.Test/Player.cs diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Player.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Player.cs deleted file mode 100644 index 21e2a925fbb..00000000000 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Player.cs +++ /dev/null @@ -1,10 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Serialization.FastRlp.Generator; - -namespace Nethermind.Serialization.FastRlp.Test; - -[RlpSerializable] -public record Player(int Id, string Username); - diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs index faf96f4f996..6ed824e8d6f 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -2,16 +2,22 @@ // SPDX-License-Identifier: LGPL-3.0-only using FluentAssertions; +// Automatically imports all source-generated instances using Nethermind.Serialization.FastRlp.Derived; +// Imports `RlpSerializable` attribute +using Nethermind.Serialization.FastRlp.Generator; namespace Nethermind.Serialization.FastRlp.Test; +[RlpSerializable] +public record Player(int Id, string Username); + public class RlpDerivedTest { [Test] public void ReadDerivedRecord() { - var player = new Player(42, "SuperUser"); + var player = new Player(Id: 42, Username: "SuperUser"); ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadPlayer()); From 38cbb3d07a02922dfa0e9ba38a93e4da4093d506 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 20 Dec 2024 17:48:00 -0300 Subject: [PATCH 035/106] No need to copy `using` directives --- .../RlpSourceGenerator.cs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs index 2b108dc351d..493675f0c13 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs @@ -57,14 +57,6 @@ private void Execute(SourceProductionContext context, continue; } - // Extract the `using` statements from the file containing the record - var usingDirectiveSyntaxes = recordDecl - .SyntaxTree - .GetRoot() - .DescendantNodes() - .OfType() - .Select(syntax => syntax.Name!.ToString()); - // Extract the record name with namespace if needed var recordName = symbol.Name; var fullTypeName = symbol.ToDisplayString(); @@ -74,7 +66,7 @@ private void Execute(SourceProductionContext context, var parameters = GetRecordParameters(recordDecl); // Build the converter class source - var generatedCode = GenerateConverterClass(usingDirectiveSyntaxes, fullTypeName, recordName, parameters); + var generatedCode = GenerateConverterClass(fullTypeName, recordName, parameters); // Add to the compilation context.AddSource($"{recordName}RlpConverter.g.cs", SourceText.From(generatedCode, Encoding.UTF8)); @@ -104,7 +96,6 @@ private void Execute(SourceProductionContext context, } private static string GenerateConverterClass( - IEnumerable usingDirectives, string fullTypeName, string recordName, List<(string Name, string TypeName)> parameters) @@ -116,10 +107,6 @@ private static string GenerateConverterClass( sb.AppendLine("using System.CodeDom.Compiler;"); sb.AppendLine("using Nethermind.Serialization.FastRlp;"); sb.AppendLine("using Nethermind.Serialization.FastRlp.Instances;"); - foreach (var usingDirective in usingDirectives) - { - sb.AppendLine($"using {usingDirective};"); - } sb.AppendLine(); sb.AppendLine("namespace Nethermind.Serialization.FastRlp.Derived;"); sb.AppendLine(""); From f284c281584823482d3042c138e1753f8d29a5a5 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 10:48:27 -0300 Subject: [PATCH 036/106] Remove `Derived` namespace - Derived code lives in the same namespace as the input record --- .../RlpSourceGenerator.cs | 22 +++++++++++++------ .../RlpDerivedTest.cs | 3 --- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs index 493675f0c13..08fac5531f8 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs @@ -37,7 +37,8 @@ public void Initialize(IncrementalGeneratorInitializationContext context) context.RegisterSourceOutput(compilation, Execute); } - private void Execute(SourceProductionContext context, + private void Execute( + SourceProductionContext context, (Compilation Compilation, ImmutableArray RecordsDeclarationSyntaxes) p) { // For each record with the attribute, generate the RlpConverter class @@ -57,16 +58,18 @@ private void Execute(SourceProductionContext context, continue; } - // Extract the record name with namespace if needed + // Extract the fully qualified record name with its namespace var recordName = symbol.Name; var fullTypeName = symbol.ToDisplayString(); + // TODO: Deal with missing and nested namespaces + var @namespace = symbol.ContainingNamespace?.ToDisplayString(); // Gather recursively all members that are fields or primary constructor parameters // so we can read them in the same order they are declared. var parameters = GetRecordParameters(recordDecl); // Build the converter class source - var generatedCode = GenerateConverterClass(fullTypeName, recordName, parameters); + var generatedCode = GenerateConverterClass(@namespace, fullTypeName, recordName, parameters); // Add to the compilation context.AddSource($"{recordName}RlpConverter.g.cs", SourceText.From(generatedCode, Encoding.UTF8)); @@ -96,6 +99,7 @@ private void Execute(SourceProductionContext context, } private static string GenerateConverterClass( + string? @namespace, string fullTypeName, string recordName, List<(string Name, string TypeName)> parameters) @@ -108,7 +112,7 @@ private static string GenerateConverterClass( sb.AppendLine("using Nethermind.Serialization.FastRlp;"); sb.AppendLine("using Nethermind.Serialization.FastRlp.Instances;"); sb.AppendLine(); - sb.AppendLine("namespace Nethermind.Serialization.FastRlp.Derived;"); + if (@namespace is not null) sb.AppendLine($"namespace {@namespace};"); sb.AppendLine(""); sb.AppendLine(GeneratedCodeAttribute); sb.AppendLine($"public abstract class {recordName}RlpConverter : IRlpConverter<{fullTypeName}>"); @@ -124,6 +128,7 @@ private static string GenerateConverterClass( { sb.AppendLine($"w.Write(value.{name});"); } + sb.AppendLine("});"); sb.AppendLine("}"); @@ -145,6 +150,7 @@ private static string GenerateConverterClass( sb.Append(parameters[i].Name); if (i < parameters.Count - 1) sb.Append(", "); } + sb.AppendLine(");"); sb.AppendLine("});"); @@ -155,8 +161,10 @@ private static string GenerateConverterClass( sb.AppendLine(GeneratedCodeAttribute); sb.AppendLine($"public static class {recordName}Ext"); sb.AppendLine("{"); - sb.AppendLine($"public static {fullTypeName} Read{recordName}(this ref RlpReader reader) => {recordName}RlpConverter.Read(ref reader);"); - sb.AppendLine($"public static void Write(this ref RlpWriter writer, {fullTypeName} value) => {recordName}RlpConverter.Write(ref writer, value);"); + sb.AppendLine( + $"public static {fullTypeName} Read{recordName}(this ref RlpReader reader) => {recordName}RlpConverter.Read(ref reader);"); + sb.AppendLine( + $"public static void Write(this ref RlpWriter writer, {fullTypeName} value) => {recordName}RlpConverter.Write(ref writer, value);"); sb.AppendLine("}"); return sb.ToString(); @@ -179,5 +187,5 @@ private static string MapTypeToReadCall(string typeName) public static class StringExt { - public static string Capitalize (this string str) => str[0].ToString().ToUpper() + str[1..]; + public static string Capitalize(this string str) => str[0].ToString().ToUpper() + str[1..]; } diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs index 6ed824e8d6f..9ccf2518801 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -2,9 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using FluentAssertions; -// Automatically imports all source-generated instances -using Nethermind.Serialization.FastRlp.Derived; -// Imports `RlpSerializable` attribute using Nethermind.Serialization.FastRlp.Generator; namespace Nethermind.Serialization.FastRlp.Test; From 6e91e0d827c6b27b669a9300e0c971e1faf249d3 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 10:52:33 -0300 Subject: [PATCH 037/106] Update docs --- .../Nethermind.Serialization.FastRlp.Test/Student.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs index 85a6971d1fb..733dafc620d 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs @@ -1,13 +1,16 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Nethermind.Serialization.FastRlp.Generator; using Nethermind.Serialization.FastRlp.Instances; namespace Nethermind.Serialization.FastRlp.Test; public record Student(string Name, int Age, Dictionary Scores); -// NOTE: All the following code can be automatically derived using Source Generators +/// +/// The following class can be automatically generated by annotating with . +/// public abstract class StudentRlpConverter : IRlpConverter { public static Student Read(ref RlpReader reader) From ec2fdbc96b7c8d12bcd1f87c522491b479bf47a9 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 10:56:49 -0300 Subject: [PATCH 038/106] Rename `List` to `Sequence` - Avoids conflicts with C#'s `List` --- .../RlpSourceGenerator.cs | 4 +- .../RlpDerivedTest.cs | 2 +- .../RlpReadWriteTest.cs | 46 +++++++++---------- .../RlpReaderTest.cs | 20 ++++---- .../RlpWriterTest.cs | 16 +++---- .../Student.cs | 8 ++-- .../RlpReader.cs | 8 ++-- .../RlpWriter.cs | 10 ++-- 8 files changed, 57 insertions(+), 57 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs index 08fac5531f8..cf2b48701b7 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs @@ -121,7 +121,7 @@ private static string GenerateConverterClass( // `Write` method sb.AppendLine($"public static void Write(ref RlpWriter w, {fullTypeName} value)"); sb.AppendLine("{"); - sb.AppendLine("w.WriteList((ref RlpWriter w) => "); + sb.AppendLine("w.WriteSequence((ref RlpWriter w) => "); sb.AppendLine("{"); foreach (var (name, _) in parameters) @@ -135,7 +135,7 @@ private static string GenerateConverterClass( // `Read` method sb.AppendLine($"public static {fullTypeName} Read(ref RlpReader r)"); sb.AppendLine("{"); - sb.AppendLine("return r.ReadList(static (scoped ref RlpReader r) =>"); + sb.AppendLine("return r.ReadSequence(static (scoped ref RlpReader r) =>"); sb.AppendLine("{"); foreach (var (name, typeName) in parameters) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs index 9ccf2518801..26022413f7b 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -12,7 +12,7 @@ public record Player(int Id, string Username); public class RlpDerivedTest { [Test] - public void ReadDerivedRecord() + public void FlatRecord() { var player = new Player(Id: 42, Username: "SuperUser"); ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs index 25dbb347198..f0cd2135d71 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs @@ -13,10 +13,10 @@ public void HeterogeneousList() { var rlp = Rlp.Write(static (ref RlpWriter w) => { - w.WriteList(static (ref RlpWriter w) => + w.WriteSequence(static (ref RlpWriter w) => { - w.WriteList(static (ref RlpWriter w) => { w.Write(42); }); - w.WriteList(static (ref RlpWriter w) => + w.WriteSequence(static (ref RlpWriter w) => { w.Write(42); }); + w.WriteSequence(static (ref RlpWriter w) => { w.Write("dog"); w.Write("cat"); @@ -26,10 +26,10 @@ public void HeterogeneousList() var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => { - return r.ReadList(static (scoped ref RlpReader r) => + return r.ReadSequence(static (scoped ref RlpReader r) => { - var _1 = r.ReadList(static (scoped ref RlpReader r) => r.ReadInt32()); - var _2 = r.ReadList(static (scoped ref RlpReader r) => + var _1 = r.ReadSequence(static (scoped ref RlpReader r) => r.ReadInt32()); + var _2 = r.ReadSequence(static (scoped ref RlpReader r) => { var _1 = r.ReadString(); var _2 = r.ReadString(); @@ -49,7 +49,7 @@ public void LongList() { var rlp = Rlp.Write(static (ref RlpWriter w) => { - w.WriteList(static (ref RlpWriter w) => + w.WriteSequence(static (ref RlpWriter w) => { for (int i = 0; i < 100; i++) { @@ -60,7 +60,7 @@ public void LongList() List decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => { - return r.ReadList((scoped ref RlpReader r) => + return r.ReadSequence((scoped ref RlpReader r) => { List result = []; for (int i = 0; i < 100; i++) @@ -81,14 +81,14 @@ public void MutlipleLongList() { var rlp = Rlp.Write(static (ref RlpWriter w) => { - w.WriteList(static (ref RlpWriter w) => + w.WriteSequence(static (ref RlpWriter w) => { for (int i = 0; i < 100; i++) { w.Write("dog"); } }); - w.WriteList(static (ref RlpWriter w) => + w.WriteSequence(static (ref RlpWriter w) => { for (int i = 0; i < 50; i++) { @@ -99,7 +99,7 @@ public void MutlipleLongList() var (dogs, cats) = Rlp.Read(rlp, (scoped ref RlpReader r) => { - var dogs = r.ReadList((scoped ref RlpReader r) => + var dogs = r.ReadSequence((scoped ref RlpReader r) => { List result = []; while (r.HasNext) @@ -109,7 +109,7 @@ public void MutlipleLongList() return result; }); - var cats = r.ReadList((scoped ref RlpReader r) => + var cats = r.ReadSequence((scoped ref RlpReader r) => { List result = []; while (r.HasNext) @@ -135,7 +135,7 @@ public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) { var rlp = Rlp.Write((ref RlpWriter root) => { - root.WriteList((ref RlpWriter w) => + root.WriteSequence((ref RlpWriter w) => { for (int i = 0; i < length; i++) { @@ -146,7 +146,7 @@ public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) List decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => { - return r.ReadList((scoped ref RlpReader r) => + return r.ReadSequence((scoped ref RlpReader r) => { List result = []; while (r.HasNext) @@ -165,7 +165,7 @@ public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) public void InvalidObjectReading() { var rlp = Rlp.Write(static (ref RlpWriter w) => { w.Write(42); }); - Action tryRead = () => Rlp.Read(rlp, (scoped ref RlpReader r) => { r.ReadList((scoped ref RlpReader _) => { }); }); + Action tryRead = () => Rlp.Read(rlp, (scoped ref RlpReader r) => { r.ReadSequence((scoped ref RlpReader _) => { }); }); tryRead.Should().Throw(); } @@ -173,7 +173,7 @@ public void InvalidObjectReading() [Test] public void InvalidListReading() { - var rlp = Rlp.Write(static (ref RlpWriter w) => { w.WriteList(static (ref RlpWriter _) => { }); }); + var rlp = Rlp.Write(static (ref RlpWriter w) => { w.WriteSequence(static (ref RlpWriter _) => { }); }); Func tryRead = () => Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadInt32()); tryRead.Should().Throw(); @@ -183,10 +183,10 @@ public void InvalidListReading() public void Choice() { RefRlpReaderFunc intReader = (scoped ref RlpReader r) => r.ReadInt32(); - RefRlpReaderFunc wrappedReader = (scoped ref RlpReader r) => r.ReadList(intReader); + RefRlpReaderFunc wrappedReader = (scoped ref RlpReader r) => r.ReadSequence(intReader); var intRlp = Rlp.Write(static (ref RlpWriter w) => { w.Write(42); }); - var wrappedIntRlp = Rlp.Write(static (ref RlpWriter w) => w.WriteList(static (ref RlpWriter w) => { w.Write(42); })); + var wrappedIntRlp = Rlp.Write(static (ref RlpWriter w) => w.WriteSequence(static (ref RlpWriter w) => { w.Write(42); })); foreach (var rlp in (byte[][]) [intRlp, wrappedIntRlp]) { @@ -201,7 +201,7 @@ public void ChoiceDeep() { RefRlpReaderFunc<(string, string, string)> readerA = static (scoped ref RlpReader r) => { - return r.ReadList(static (scoped ref RlpReader r) => + return r.ReadSequence(static (scoped ref RlpReader r) => { var _1 = r.ReadString(); var _2 = r.ReadString(); @@ -212,7 +212,7 @@ public void ChoiceDeep() }; RefRlpReaderFunc<(string, string, string)> readerB = static (scoped ref RlpReader r) => { - return r.ReadList(static (scoped ref RlpReader r) => + return r.ReadSequence(static (scoped ref RlpReader r) => { var _1 = r.ReadString(); var _2 = r.ReadString(); @@ -224,7 +224,7 @@ public void ChoiceDeep() var rlp = Rlp.Write(static (ref RlpWriter w) => { - w.WriteList(static (ref RlpWriter w) => + w.WriteSequence(static (ref RlpWriter w) => { w.Write("dog"); w.Write("cat"); @@ -255,7 +255,7 @@ public void UserDefinedRecord() var rlp = Rlp.Write((ref RlpWriter w) => { - w.WriteList((ref RlpWriter w) => + w.WriteSequence((ref RlpWriter w) => { foreach (var student in students) { @@ -266,7 +266,7 @@ public void UserDefinedRecord() var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => { - return r.ReadList(static (scoped ref RlpReader r) => + return r.ReadSequence(static (scoped ref RlpReader r) => { List result = []; while (r.HasNext) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs index d97c67b2292..201198ec3e2 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs @@ -67,7 +67,7 @@ public void ReadStringList() byte[] source = [0xc8, 0x83, .."cat"u8, 0x83, .."dog"u8]; var actual = Rlp.Read(source, static (scoped ref RlpReader r) => { - return r.ReadList(static (scoped ref RlpReader r) => + return r.ReadSequence(static (scoped ref RlpReader r) => { var cat = r.ReadString(); var dog = r.ReadString(); @@ -86,7 +86,7 @@ public void ReadEmptyList() var actual = Rlp.Read(source, static (scoped ref RlpReader r) => { - return r.ReadList((scoped ref RlpReader _) => Array.Empty()); + return r.ReadSequence((scoped ref RlpReader _) => Array.Empty()); }); actual.Should().BeEmpty(); @@ -110,20 +110,20 @@ public void ReadSetTheoreticalRepresentation() object[] actual = Rlp.Read(source, static (scoped ref RlpReader r) => { - return r.ReadList(static (scoped ref RlpReader r) => + return r.ReadSequence(static (scoped ref RlpReader r) => { - var _1 = r.ReadList(static (scoped ref RlpReader _) => Array.Empty()); - var _2 = r.ReadList(static (scoped ref RlpReader r) => + var _1 = r.ReadSequence(static (scoped ref RlpReader _) => Array.Empty()); + var _2 = r.ReadSequence(static (scoped ref RlpReader r) => { - var _1 = r.ReadList(static (scoped ref RlpReader _) => Array.Empty()); + var _1 = r.ReadSequence(static (scoped ref RlpReader _) => Array.Empty()); return new object[] { _1 }; }); - var _3 = r.ReadList(static (scoped ref RlpReader r) => + var _3 = r.ReadSequence(static (scoped ref RlpReader r) => { - var _1 = r.ReadList(static (scoped ref RlpReader _) => Array.Empty()); - var _2 = r.ReadList(static (scoped ref RlpReader r) => + var _1 = r.ReadSequence(static (scoped ref RlpReader _) => Array.Empty()); + var _2 = r.ReadSequence(static (scoped ref RlpReader r) => { - var _1 = r.ReadList(static (scoped ref RlpReader _) => Array.Empty()); + var _1 = r.ReadSequence(static (scoped ref RlpReader _) => Array.Empty()); return new object[] { _1 }; }); diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs index daa59b9c604..48164c084e3 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs @@ -86,7 +86,7 @@ public void WriteStringList() { var serialized = Rlp.Write(static (ref RlpWriter w) => { - w.WriteList(static (ref RlpWriter w) => + w.WriteSequence(static (ref RlpWriter w) => { w.Write("cat"); w.Write("dog"); @@ -100,7 +100,7 @@ public void WriteStringList() [Test] public void WriteEmptyList() { - var serialized = Rlp.Write(static (ref RlpWriter w) => { w.WriteList(static (ref RlpWriter _) => { }); }); + var serialized = Rlp.Write(static (ref RlpWriter w) => { w.WriteSequence(static (ref RlpWriter _) => { }); }); byte[] expected = [0xc0]; serialized.Should().BeEquivalentTo(expected); @@ -120,14 +120,14 @@ public void WriteSetTheoreticalRepresentation() { var serialized = Rlp.Write(static (ref RlpWriter w) => { - w.WriteList(static (ref RlpWriter w) => + w.WriteSequence(static (ref RlpWriter w) => { - w.WriteList(static (ref RlpWriter _) => { }); - w.WriteList(static (ref RlpWriter w) => { w.WriteList(static (ref RlpWriter _) => { }); }); - w.WriteList(static (ref RlpWriter w) => + w.WriteSequence(static (ref RlpWriter _) => { }); + w.WriteSequence(static (ref RlpWriter w) => { w.WriteSequence(static (ref RlpWriter _) => { }); }); + w.WriteSequence(static (ref RlpWriter w) => { - w.WriteList(static (ref RlpWriter _) => { }); - w.WriteList(static (ref RlpWriter w) => { w.WriteList(static (ref RlpWriter _) => { }); }); + w.WriteSequence(static (ref RlpWriter _) => { }); + w.WriteSequence(static (ref RlpWriter w) => { w.WriteSequence(static (ref RlpWriter _) => { }); }); }); }); }); diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs index 733dafc620d..50ccfeb9f29 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs @@ -15,11 +15,11 @@ public abstract class StudentRlpConverter : IRlpConverter { public static Student Read(ref RlpReader reader) { - return reader.ReadList(static (scoped ref RlpReader r) => + return reader.ReadSequence(static (scoped ref RlpReader r) => { var name = r.ReadString(); var age = r.ReadInt32(); - var scores = r.ReadList(static (scoped ref RlpReader r) => + var scores = r.ReadSequence(static (scoped ref RlpReader r) => { Dictionary scores = []; while (r.HasNext) @@ -39,11 +39,11 @@ public static Student Read(ref RlpReader reader) public static void Write(ref RlpWriter writer, Student value) { - writer.WriteList((ref RlpWriter w) => + writer.WriteSequence((ref RlpWriter w) => { w.Write(value.Name); w.Write(value.Age); - w.WriteList((ref RlpWriter w) => + w.WriteSequence((ref RlpWriter w) => { foreach (var (subject, score) in value.Scores) { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs index 70bb3deee32..591ded438dc 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs @@ -72,13 +72,13 @@ public ReadOnlySpan ReadBytes() return result; } - public T ReadList(RefRlpReaderFunc func) + public T ReadSequence(RefRlpReaderFunc func) { T result; var header = _buffer[_position++]; if (header < 0xC0) { - throw new RlpReaderException("RLP does not correspond to a list"); + throw new RlpReaderException("RLP does not correspond to a sequence"); } if (header < 0xF8) @@ -102,9 +102,9 @@ public T ReadList(RefRlpReaderFunc func) return result; } - public void ReadList(RefRlpReaderAction func) + public void ReadSequence(RefRlpReaderAction func) { - ReadList((scoped ref RlpReader r) => + ReadSequence((scoped ref RlpReader r) => { func(ref r); return null; diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs index 9a571172462..b8e70ba0c12 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs @@ -142,20 +142,20 @@ private void ContentWrite(scoped ReadOnlySpan value) _position += value.Length; } - public void WriteList(RefRlpWriterAction action) + public void WriteSequence(RefRlpWriterAction action) { switch (_mode) { case LengthMode: - LengthWriteList(action); + LengthWriteSequence(action); break; case ContentMode: - ContentWriteList(action); + ContentWriteSequence(action); break; } } - private void LengthWriteList(RefRlpWriterAction action) + private void LengthWriteSequence(RefRlpWriterAction action) { var inner = LengthWriter(); action(ref inner); @@ -172,7 +172,7 @@ private void LengthWriteList(RefRlpWriterAction action) } } - private void ContentWriteList(RefRlpWriterAction action) + private void ContentWriteSequence(RefRlpWriterAction action) { var lengthWriter = LengthWriter(); action(ref lengthWriter); From f1584632eb9a765fddf73a02a31ffdcd30e1b7ca Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 12:06:51 -0300 Subject: [PATCH 039/106] Add support for `List` collection --- .../RlpReaderTest.cs | 9 ++++ .../RlpWriterTest.cs | 10 ++++ .../Instances/ListRlpConverter.cs | 47 +++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 src/Nethermind/Nethermind.Serialization.FastRlp/Instances/ListRlpConverter.cs diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs index 201198ec3e2..a39a628035e 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs @@ -79,6 +79,15 @@ public void ReadStringList() actual.Should().Be(("cat", "dog")); } + [Test] + public void ReadStringListCollection() + { + byte[] source = [0xc8, 0x83, .."cat"u8, 0x83, .."dog"u8]; + var actual = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadList()); + + actual.Should().BeEquivalentTo("cat", "dog"); + } + [Test] public void ReadEmptyList() { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs index 48164c084e3..6638b815963 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs @@ -97,6 +97,16 @@ public void WriteStringList() serialized.Should().BeEquivalentTo(expected); } + [Test] + public void WriteStringListCollection() + { + var list = new List { "cat", "dog" }; + var serialized = Rlp.Write((ref RlpWriter w) => w.Write(list)); + + byte[] expected = [0xc8, 0x83, (byte)'c', (byte)'a', (byte)'t', 0x83, (byte)'d', (byte)'o', (byte)'g']; + serialized.Should().BeEquivalentTo(expected); + } + [Test] public void WriteEmptyList() { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/ListRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/ListRlpConverter.cs new file mode 100644 index 00000000000..25a3ad1476a --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/ListRlpConverter.cs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; + +namespace Nethermind.Serialization.FastRlp.Instances; + +// TODO: We might want to introduce an interface for collection types (ex. List) +// Main issue are HKT (aka Generics of Generics) +public abstract class ListRlpConverter +{ + public static List Read(ref RlpReader reader) where TConverter : IRlpConverter + { + return reader.ReadSequence((scoped ref RlpReader r) => + { + List result = []; + while (r.HasNext) + { + result.Add(TConverter.Read(ref r)); + } + + return result; + }); + } + + public static void Write(ref RlpWriter writer, List value) where TConverter : IRlpConverter + { + writer.WriteSequence((ref RlpWriter w) => + { + foreach (T v in value) + { + TConverter.Write(ref w, v); + } + }); + } +} + +public static class ListRlpConverterExt +{ + public static List ReadList(this ref RlpReader reader) + where TConverter : IRlpConverter + => ListRlpConverter.Read(ref reader); + + public static void Write(this ref RlpWriter writer, List value) + where TConverter : IRlpConverter + => ListRlpConverter.Write(ref writer, value); +} From 66772226e35f4c9deecca741b45496e47918f094 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 12:29:24 -0300 Subject: [PATCH 040/106] Test for equivalence in collections with explicit --- .../RlpReaderTest.cs | 32 +++++++++++++------ .../RlpWriterTest.cs | 27 ++++++++++------ 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs index a39a628035e..ccb8e4fced7 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs @@ -79,15 +79,6 @@ public void ReadStringList() actual.Should().Be(("cat", "dog")); } - [Test] - public void ReadStringListCollection() - { - byte[] source = [0xc8, 0x83, .."cat"u8, 0x83, .."dog"u8]; - var actual = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadList()); - - actual.Should().BeEquivalentTo("cat", "dog"); - } - [Test] public void ReadEmptyList() { @@ -150,4 +141,27 @@ public void ReadSetTheoreticalRepresentation() new object[] { new object[] { }, new object[] { new object[] { } } }, }); } + + + [Test] + public void ReadListEquivalentToExplicit() + { + byte[] source = [0xc8, 0x83, .."cat"u8, 0x83, .."dog"u8]; + var decodedList = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadList()); + var decodedExplicit = Rlp.Read(source, static (scoped ref RlpReader r) => + { + return r.ReadSequence(static (scoped ref RlpReader r) => + { + List result = []; + while (r.HasNext) + { + result.Add(r.ReadString()); + } + + return result; + }); + }); + + decodedList.Should().BeEquivalentTo(decodedExplicit); + } } diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs index 6638b815963..6607804b3ef 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs @@ -97,16 +97,6 @@ public void WriteStringList() serialized.Should().BeEquivalentTo(expected); } - [Test] - public void WriteStringListCollection() - { - var list = new List { "cat", "dog" }; - var serialized = Rlp.Write((ref RlpWriter w) => w.Write(list)); - - byte[] expected = [0xc8, 0x83, (byte)'c', (byte)'a', (byte)'t', 0x83, (byte)'d', (byte)'o', (byte)'g']; - serialized.Should().BeEquivalentTo(expected); - } - [Test] public void WriteEmptyList() { @@ -145,4 +135,21 @@ public void WriteSetTheoreticalRepresentation() byte[] expected = [0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; serialized.Should().BeEquivalentTo(expected); } + + [Test] + public void WriteListEquivalentToExplicit() + { + var list = new List { "cat", "dog" }; + var rlpList = Rlp.Write((ref RlpWriter w) => w.Write(list)); + var rlpExplicit = Rlp.Write(static (ref RlpWriter w) => + { + w.WriteSequence(static (ref RlpWriter w) => + { + w.Write("cat"); + w.Write("dog"); + }); + }); + + rlpList.Should().BeEquivalentTo(rlpExplicit); + } } From e32a33c0a14ec7f92fc1e7ec7b2c36379c9e7a51 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 12:29:55 -0300 Subject: [PATCH 041/106] Add explicit converters for IntXX - Required as Generics in Collections --- .../Instances/IntegerRlpConverter.cs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/IntegerRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/IntegerRlpConverter.cs index 2af8b5344ae..f1ebd0ae2da 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/IntegerRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/IntegerRlpConverter.cs @@ -5,6 +5,34 @@ namespace Nethermind.Serialization.FastRlp.Instances; +public abstract class Int16RlpConverter : IRlpConverter +{ + public static Int16 Read(ref RlpReader reader) => reader.ReadInteger(); + + public static void Write(ref RlpWriter writer, Int16 value) => writer.Write(value); +} + +public abstract class Int32RlpConverter : IRlpConverter +{ + public static Int32 Read(ref RlpReader reader) => reader.ReadInteger(); + + public static void Write(ref RlpWriter writer, Int32 value) => writer.Write(value); +} + +public abstract class Int64RlpConverter : IRlpConverter +{ + public static Int64 Read(ref RlpReader reader) => reader.ReadInteger(); + + public static void Write(ref RlpWriter writer, Int64 value) => writer.Write(value); +} + +public abstract class Int128RlpConverter : IRlpConverter +{ + public static Int128 Read(ref RlpReader reader) => reader.ReadInteger(); + + public static void Write(ref RlpWriter writer, Int128 value) => writer.Write(value); +} + // NOTE: No need for `Write` overloads since they're covered by generic primitives // `Read` methods are provided for a consistent API (instead of using generics primitives) public static class IntegerRlpConverterExt From f665779eb4bd4dcbaf12d771737dc05237bd6de3 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 12:37:55 -0300 Subject: [PATCH 042/106] Reduce duplication --- .../RlpReadWriteTest.cs | 21 +++++++++++++++++ .../RlpReaderTest.cs | 23 ------------------- .../RlpWriterTest.cs | 17 -------------- 3 files changed, 21 insertions(+), 40 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs index f0cd2135d71..7f394c400bf 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs @@ -280,4 +280,25 @@ public void UserDefinedRecord() decoded.Should().BeEquivalentTo(students); } + + [Test] + public void ListCollection() + { + var list = new List { "cat", "dog" }; + var rlp = Rlp.Write((ref RlpWriter w) => w.Write(list)); + + var rlpExplicit = Rlp.Write(static (ref RlpWriter w) => + { + w.WriteSequence(static (ref RlpWriter w) => + { + w.Write("cat"); + w.Write("dog"); + }); + }); + rlpExplicit.Should().BeEquivalentTo(rlp); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadList()); + + list.Should().BeEquivalentTo(decoded); + } } diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs index ccb8e4fced7..201198ec3e2 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs @@ -141,27 +141,4 @@ public void ReadSetTheoreticalRepresentation() new object[] { new object[] { }, new object[] { new object[] { } } }, }); } - - - [Test] - public void ReadListEquivalentToExplicit() - { - byte[] source = [0xc8, 0x83, .."cat"u8, 0x83, .."dog"u8]; - var decodedList = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadList()); - var decodedExplicit = Rlp.Read(source, static (scoped ref RlpReader r) => - { - return r.ReadSequence(static (scoped ref RlpReader r) => - { - List result = []; - while (r.HasNext) - { - result.Add(r.ReadString()); - } - - return result; - }); - }); - - decodedList.Should().BeEquivalentTo(decodedExplicit); - } } diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs index 6607804b3ef..48164c084e3 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs @@ -135,21 +135,4 @@ public void WriteSetTheoreticalRepresentation() byte[] expected = [0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; serialized.Should().BeEquivalentTo(expected); } - - [Test] - public void WriteListEquivalentToExplicit() - { - var list = new List { "cat", "dog" }; - var rlpList = Rlp.Write((ref RlpWriter w) => w.Write(list)); - var rlpExplicit = Rlp.Write(static (ref RlpWriter w) => - { - w.WriteSequence(static (ref RlpWriter w) => - { - w.Write("cat"); - w.Write("dog"); - }); - }); - - rlpList.Should().BeEquivalentTo(rlpExplicit); - } } From 2e75a7d70864551094e25235099599774db22221 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 12:42:00 -0300 Subject: [PATCH 043/106] Add `Dictionary` collection --- .../RlpReadWriteTest.cs | 33 +++++++++ .../Instances/DictionaryRlpConverter.cs | 70 +++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 src/Nethermind/Nethermind.Serialization.FastRlp/Instances/DictionaryRlpConverter.cs diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs index 7f394c400bf..74eefe6d09d 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs @@ -285,6 +285,7 @@ public void UserDefinedRecord() public void ListCollection() { var list = new List { "cat", "dog" }; + var rlp = Rlp.Write((ref RlpWriter w) => w.Write(list)); var rlpExplicit = Rlp.Write(static (ref RlpWriter w) => @@ -301,4 +302,36 @@ public void ListCollection() list.Should().BeEquivalentTo(decoded); } + + [Test] + public void DictionaryCollection() + { + var dictionary = new Dictionary + { + { 1, "dog" }, + { 2, "cat" }, + }; + + var rlp = Rlp.Write((ref RlpWriter w) => w.Write(dictionary)); + + var rlpExplicit = Rlp.Write((ref RlpWriter w) => + { + w.WriteSequence((ref RlpWriter w) => + { + foreach (var (k, v) in dictionary) + { + w.WriteSequence((ref RlpWriter w) => + { + w.Write(k); + w.Write(v); + }); + } + }); + }); + rlp.Should().BeEquivalentTo(rlpExplicit); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadDictionary()); + + decoded.Should().BeEquivalentTo(dictionary); + } } diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/DictionaryRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/DictionaryRlpConverter.cs new file mode 100644 index 00000000000..430dee24d35 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/DictionaryRlpConverter.cs @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; + +namespace Nethermind.Serialization.FastRlp.Instances; + +public abstract class DictionaryRlpConverter where TKey : notnull +{ + public static Dictionary Read(ref RlpReader reader) + where TKeyConverter : IRlpConverter + where TValueConverter : IRlpConverter + { + return reader.ReadSequence((scoped ref RlpReader r) => + { + Dictionary result = []; + while (r.HasNext) + { + (TKey key, TValue value) = r.ReadSequence((scoped ref RlpReader r) => + { + TKey key = TKeyConverter.Read(ref r); + TValue value = TValueConverter.Read(ref r); + + return (key, value); + }); + + result.Add(key, value); + } + + return result; + }); + } + + public static void Write(ref RlpWriter writer, Dictionary value) + where TKeyConverter : IRlpConverter + where TValueConverter : IRlpConverter + { + writer.WriteSequence((ref RlpWriter w) => + { + foreach ((TKey k, TValue v) in value) + { + w.WriteSequence((ref RlpWriter w) => + { + TKeyConverter.Write(ref w, k); + TValueConverter.Write(ref w, v); + }); + } + }); + } +} + +public static class DictionaryRlpConverterExt +{ + public static Dictionary ReadDictionary( + this ref RlpReader reader + ) + where TKey : notnull + where TKeyConverter : IRlpConverter + where TValueConverter : IRlpConverter + => DictionaryRlpConverter.Read(ref reader); + + public static void Write( + this ref RlpWriter writer, + Dictionary value + ) + where TKey : notnull + where TValueConverter : IRlpConverter + where TKeyConverter : IRlpConverter + => DictionaryRlpConverter.Write(ref writer, value); +} From aa6b4bd1a79493935dac88a58861948a2fa6607a Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 13:12:21 -0300 Subject: [PATCH 044/106] Extend generator to support Generics (WIP) --- .../RlpSourceGenerator.cs | 71 +++++++++++++++---- .../RlpDerivedTest.cs | 15 +++- 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs index cf2b48701b7..a32a91211f7 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs @@ -80,9 +80,9 @@ private void Execute( /// Gathers the record’s primary constructor parameters and public fields/properties /// in the order they appear in the record declaration. /// - private static List<(string Name, string TypeName)> GetRecordParameters(RecordDeclarationSyntax recordDecl) + private static List<(string Name, TypeSyntax TypeName)> GetRecordParameters(RecordDeclarationSyntax recordDecl) { - List<(string, string)> parameters = []; + List<(string, TypeSyntax)> parameters = []; // Primary constructor parameters if (recordDecl.ParameterList is not null) @@ -90,7 +90,8 @@ private void Execute( foreach (var param in recordDecl.ParameterList.Parameters) { var paramName = param.Identifier.Text; - var paramType = param.Type?.ToString() ?? "object"; + var paramType = param.Type!; + parameters.Add((paramName, paramType)); } } @@ -102,7 +103,7 @@ private static string GenerateConverterClass( string? @namespace, string fullTypeName, string recordName, - List<(string Name, string TypeName)> parameters) + List<(string Name, TypeSyntax TypeName)> parameters) { var sb = new StringBuilder(); @@ -124,9 +125,10 @@ private static string GenerateConverterClass( sb.AppendLine("w.WriteSequence((ref RlpWriter w) => "); sb.AppendLine("{"); - foreach (var (name, _) in parameters) + foreach (var (name, typeName) in parameters) { - sb.AppendLine($"w.Write(value.{name});"); + var writeCall = MapTypeToWriteCall(name, typeName); + sb.AppendLine($"w.{writeCall};"); } sb.AppendLine("});"); @@ -174,14 +176,59 @@ private static string GenerateConverterClass( /// Map the type name to the appropriate Read method on the `RlpReader` /// Extend this mapping for more types as needed. /// - private static string MapTypeToReadCall(string typeName) + private static string MapTypeToReadCall(TypeSyntax syntax) { - return typeName switch + // Hard-coded cases + switch (syntax.ToString()) + { + case "byte[]" or "System.Byte[]" or "Span" or "System.Span" or "System.ReadOnlySpan": + return "ReadBytes()"; + case "int": + return "ReadInt32()"; + } + + // Generics + if (syntax is GenericNameSyntax generic) { - "byte[]" or "System.Byte[]" or "Span" or "System.Span" or "System.ReadOnlySpan" => "ReadBytes()", - "int" => "ReadInt32()", - _ => $"Read{typeName.Capitalize()}()" - }; + var typeConstructor = generic.Identifier.ToString(); + var typeParameters = generic.TypeArgumentList.Arguments; + + var sb = new StringBuilder("Read"); + sb.Append(typeConstructor.Capitalize()); + sb.Append("<"); + foreach (var typeParameter in typeParameters) + { + sb.Append($"{typeParameter.ToString()}, {typeParameter.ToString().Capitalize()}RlpConverter"); + } + sb.Append(">()"); + + return sb.ToString(); + } + + // Defaults + return $"Read{syntax.ToString().Capitalize()}()"; + } + + private static string MapTypeToWriteCall(string name, TypeSyntax syntax) + { + // Generics + if (syntax is GenericNameSyntax generic) + { + var typeParameters = generic.TypeArgumentList.Arguments; + + var sb = new StringBuilder("Write"); + sb.Append("<"); + foreach (var typeParameter in typeParameters) + { + sb.Append($"{typeParameter.ToString()}, {typeParameter.ToString().Capitalize()}RlpConverter"); + } + sb.Append($">(value.{name})"); + + return sb.ToString(); + } + + // Defaults + return $"Write(value.{name})"; } } diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs index 26022413f7b..49f9491ae89 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -9,6 +9,9 @@ namespace Nethermind.Serialization.FastRlp.Test; [RlpSerializable] public record Player(int Id, string Username); +[RlpSerializable] +public record PlayerWithFriends(int Id, string Username, List Friends); + public class RlpDerivedTest { [Test] @@ -18,6 +21,16 @@ public void FlatRecord() ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadPlayer()); - decoded.Should().Be(player); + decoded.Should().BeEquivalentTo(player); + } + + [Test] + public void RecordWithList() + { + var player = new PlayerWithFriends(Id: 42, Username: "SuperUser", Friends: ["ana", "bob"]); + ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); + + var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadPlayerWithFriends()); + decoded.Should().BeEquivalentTo(player); } } From fcde60781571c9d8778adf7eb3e207fdfcd7ba4a Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 13:43:01 -0300 Subject: [PATCH 045/106] Support multi-param generics --- .../RlpSourceGenerator.cs | 72 ++++++++++++++----- .../RlpDerivedTest.cs | 17 +++++ 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs index a32a91211f7..2f63c428676 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs @@ -127,8 +127,8 @@ private static string GenerateConverterClass( foreach (var (name, typeName) in parameters) { - var writeCall = MapTypeToWriteCall(name, typeName); - sb.AppendLine($"w.{writeCall};"); + var writeCall = MapTypeToWriteCall(typeName); + sb.AppendLine($"w.{writeCall}(value.{name});"); } sb.AppendLine("});"); @@ -143,7 +143,7 @@ private static string GenerateConverterClass( foreach (var (name, typeName) in parameters) { var readCall = MapTypeToReadCall(typeName); - sb.AppendLine($"var {name} = r.{readCall};"); + sb.AppendLine($"var {name} = r.{readCall}();"); } sb.Append($"return new {fullTypeName}("); @@ -182,9 +182,7 @@ private static string MapTypeToReadCall(TypeSyntax syntax) switch (syntax.ToString()) { case "byte[]" or "System.Byte[]" or "Span" or "System.Span" or "System.ReadOnlySpan": - return "ReadBytes()"; - case "int": - return "ReadInt32()"; + return "ReadBytes"; } // Generics @@ -196,20 +194,28 @@ private static string MapTypeToReadCall(TypeSyntax syntax) var sb = new StringBuilder("Read"); sb.Append(typeConstructor.Capitalize()); sb.Append("<"); - foreach (var typeParameter in typeParameters) + + var genericTypes = typeParameters + .Select(t => t.ToString()); + var rlpConverterTypes = typeParameters + .Select(t => $"{MapTypeAlias(t.ToString())}RlpConverter"); + + var readTypeParameters = genericTypes.Concat(rlpConverterTypes).Intersperse(", "); + foreach (string rtp in readTypeParameters) { - sb.Append($"{typeParameter.ToString()}, {typeParameter.ToString().Capitalize()}RlpConverter"); + sb.Append(rtp); } - sb.Append(">()"); + + sb.Append(">"); return sb.ToString(); } - // Defaults - return $"Read{syntax.ToString().Capitalize()}()"; + // Default + return $"Read{MapTypeAlias(syntax.ToString())}"; } - private static string MapTypeToWriteCall(string name, TypeSyntax syntax) + private static string MapTypeToWriteCall(TypeSyntax syntax) { // Generics if (syntax is GenericNameSyntax generic) @@ -218,21 +224,53 @@ private static string MapTypeToWriteCall(string name, TypeSyntax syntax) var sb = new StringBuilder("Write"); sb.Append("<"); - foreach (var typeParameter in typeParameters) + + var genericTypes = typeParameters + .Select(t => t.ToString()); + var rlpConverterTypes = typeParameters + .Select(t => $"{MapTypeAlias(t.ToString())}RlpConverter"); + + var readTypeParameters = genericTypes.Concat(rlpConverterTypes).Intersperse(", "); + foreach (string rtp in readTypeParameters) { - sb.Append($"{typeParameter.ToString()}, {typeParameter.ToString().Capitalize()}RlpConverter"); + sb.Append(rtp); } - sb.Append($">(value.{name})"); + + sb.Append(">"); return sb.ToString(); } - // Defaults - return $"Write(value.{name})"; + // Default + return "Write"; } + + private static string MapTypeAlias(string alias) => + alias switch + { + "string" => "String", + "short" => "Int16", + "int" => "Int32", + "long" => "Int64", + _ => alias + }; } public static class StringExt { public static string Capitalize(this string str) => str[0].ToString().ToUpper() + str[1..]; } + +public static class EnumerableExt +{ + public static IEnumerable Intersperse(this IEnumerable source, T element) + { + bool first = true; + foreach (T value in source) + { + if (!first) yield return element; + yield return value; + first = false; + } + } +} diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs index 49f9491ae89..898c67f06a6 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -12,6 +12,9 @@ public record Player(int Id, string Username); [RlpSerializable] public record PlayerWithFriends(int Id, string Username, List Friends); +[RlpSerializable] +public record PlayerWithScores(int Id, string Username, Dictionary Scores); + public class RlpDerivedTest { [Test] @@ -33,4 +36,18 @@ public void RecordWithList() var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadPlayerWithFriends()); decoded.Should().BeEquivalentTo(player); } + + [Test] + public void RecordWithDictionary() + { + var player = new PlayerWithScores(Id: 42, Username: "SuperUser", Scores: new() + { + { "foo", 42 }, + { "bar", 1337 } + }); + ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); + + var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadPlayerWithScores()); + decoded.Should().BeEquivalentTo(player); + } } From 416a45b548189dda35597bb838bbc539c2b814ed Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 14:05:42 -0300 Subject: [PATCH 046/106] Add Generic writer Action - Required for Collections --- src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs index b8e70ba0c12..e445d78de5c 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs @@ -8,6 +8,8 @@ namespace Nethermind.Serialization.FastRlp; +public delegate void RefRlpWriterAction(ref RlpWriter arg, T value) where T: allows ref struct; + public delegate void RefRlpWriterAction(ref RlpWriter arg); public ref struct RlpWriter From 837c2e482d430801d4cb5353f81edc5aeb43fbd7 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 15:14:06 -0300 Subject: [PATCH 047/106] Rewrite Generics - Remove Converter type parameters - Blocks nested Collections --- .../RlpSourceGenerator.cs | 53 ++++++++++--------- .../RlpReadWriteTest.cs | 52 ++++++++++++++++-- .../Instances/DictionaryRlpConverter.cs | 42 +++++++-------- .../Instances/ListRlpConverter.cs | 20 +++---- 4 files changed, 101 insertions(+), 66 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs index 2f63c428676..eb4e9d3a160 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs @@ -127,8 +127,8 @@ private static string GenerateConverterClass( foreach (var (name, typeName) in parameters) { - var writeCall = MapTypeToWriteCall(typeName); - sb.AppendLine($"w.{writeCall}(value.{name});"); + var writeCall = MapTypeToWriteCall(name, typeName); + sb.AppendLine($"w.{writeCall};"); } sb.AppendLine("});"); @@ -143,7 +143,7 @@ private static string GenerateConverterClass( foreach (var (name, typeName) in parameters) { var readCall = MapTypeToReadCall(typeName); - sb.AppendLine($"var {name} = r.{readCall}();"); + sb.AppendLine($"var {name} = r.{readCall};"); } sb.Append($"return new {fullTypeName}("); @@ -193,29 +193,31 @@ private static string MapTypeToReadCall(TypeSyntax syntax) var sb = new StringBuilder("Read"); sb.Append(typeConstructor.Capitalize()); - sb.Append("<"); + sb.AppendLine("("); - var genericTypes = typeParameters - .Select(t => t.ToString()); - var rlpConverterTypes = typeParameters - .Select(t => $"{MapTypeAlias(t.ToString())}RlpConverter"); - - var readTypeParameters = genericTypes.Concat(rlpConverterTypes).Intersperse(", "); - foreach (string rtp in readTypeParameters) + foreach (var typeParameter in typeParameters) { - sb.Append(rtp); + sb.Append("static (scoped ref RlpReader r) =>"); + sb.Append("{"); + sb.Append($"return r.{MapTypeToReadCall(typeParameter)};"); + sb.Append("},"); } - sb.Append(">"); + sb.Length -= 1; // Remove the trailing `,` + sb.Append(")"); return sb.ToString(); } // Default - return $"Read{MapTypeAlias(syntax.ToString())}"; + return $"Read{MapTypeAlias(syntax.ToString())}()"; } - private static string MapTypeToWriteCall(TypeSyntax syntax) + /// + /// Map the type name to the appropriate Write method on the `RlpWriter` + /// Extend this mapping for more types as needed. + /// + private static string MapTypeToWriteCall(string name, TypeSyntax syntax) { // Generics if (syntax is GenericNameSyntax generic) @@ -223,26 +225,25 @@ private static string MapTypeToWriteCall(TypeSyntax syntax) var typeParameters = generic.TypeArgumentList.Arguments; var sb = new StringBuilder("Write"); - sb.Append("<"); - - var genericTypes = typeParameters - .Select(t => t.ToString()); - var rlpConverterTypes = typeParameters - .Select(t => $"{MapTypeAlias(t.ToString())}RlpConverter"); + sb.AppendLine("("); - var readTypeParameters = genericTypes.Concat(rlpConverterTypes).Intersperse(", "); - foreach (string rtp in readTypeParameters) + sb.AppendLine($"value.{name},"); + foreach (var typeParameter in typeParameters) { - sb.Append(rtp); + sb.Append($"static (ref RlpWriter w, {typeParameter.ToString()} value) =>"); + sb.Append("{"); + sb.Append("w.Write(value);"); + sb.Append("},"); } - sb.Append(">"); + sb.Length -= 1; // Remove the trailing `,` + sb.Append(")"); return sb.ToString(); } // Default - return "Write"; + return $"Write(value.{name})"; } private static string MapTypeAlias(string alias) => diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs index 74eefe6d09d..e638270ce4a 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs @@ -286,7 +286,7 @@ public void ListCollection() { var list = new List { "cat", "dog" }; - var rlp = Rlp.Write((ref RlpWriter w) => w.Write(list)); + var rlp = Rlp.Write((ref RlpWriter w) => w.Write(list, StringRlpConverter.Write)); var rlpExplicit = Rlp.Write(static (ref RlpWriter w) => { @@ -298,7 +298,49 @@ public void ListCollection() }); rlpExplicit.Should().BeEquivalentTo(rlp); - var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadList()); + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadList(StringRlpConverter.Read)); + + list.Should().BeEquivalentTo(decoded); + } + + [Test] + public void ListOfListCollection() + { + List> list = [ + ["dog", "cat"], + ["foo"], + [] + ]; + + var rlp = Rlp.Write((ref RlpWriter w) => + w.Write(list, (ref RlpWriter w, List v) => + w.Write(v, StringRlpConverter.Write))); + + var rlpExplicit = Rlp.Write(static (ref RlpWriter w) => + { + w.WriteSequence((ref RlpWriter w) => + { + w.WriteSequence(static (ref RlpWriter w) => + { + w.Write("dog"); + w.Write("cat"); + }); + + w.WriteSequence(static (ref RlpWriter w) => + { + w.Write("foo"); + }); + + w.WriteSequence(static (ref RlpWriter _) => + { + }); + }); + }); + rlpExplicit.Should().BeEquivalentTo(rlp); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => + r.ReadList((scoped ref RlpReader r) => + r.ReadList(StringRlpConverter.Read))); list.Should().BeEquivalentTo(decoded); } @@ -312,7 +354,8 @@ public void DictionaryCollection() { 2, "cat" }, }; - var rlp = Rlp.Write((ref RlpWriter w) => w.Write(dictionary)); + var rlp = Rlp.Write((ref RlpWriter w) => + w.Write(dictionary, Int32RlpConverter.Write, StringRlpConverter.Write)); var rlpExplicit = Rlp.Write((ref RlpWriter w) => { @@ -330,7 +373,8 @@ public void DictionaryCollection() }); rlp.Should().BeEquivalentTo(rlpExplicit); - var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadDictionary()); + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => + r.ReadDictionary(Int32RlpConverter.Read, StringRlpConverter.Read)); decoded.Should().BeEquivalentTo(dictionary); } diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/DictionaryRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/DictionaryRlpConverter.cs index 430dee24d35..e83ea46d966 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/DictionaryRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/DictionaryRlpConverter.cs @@ -7,9 +7,7 @@ namespace Nethermind.Serialization.FastRlp.Instances; public abstract class DictionaryRlpConverter where TKey : notnull { - public static Dictionary Read(ref RlpReader reader) - where TKeyConverter : IRlpConverter - where TValueConverter : IRlpConverter + public static Dictionary Read(ref RlpReader reader, RefRlpReaderFunc readKey, RefRlpReaderFunc readValue) { return reader.ReadSequence((scoped ref RlpReader r) => { @@ -18,8 +16,8 @@ public static Dictionary Read(ref { (TKey key, TValue value) = r.ReadSequence((scoped ref RlpReader r) => { - TKey key = TKeyConverter.Read(ref r); - TValue value = TValueConverter.Read(ref r); + TKey key = readKey(ref r); + TValue value = readValue(ref r); return (key, value); }); @@ -31,9 +29,7 @@ public static Dictionary Read(ref }); } - public static void Write(ref RlpWriter writer, Dictionary value) - where TKeyConverter : IRlpConverter - where TValueConverter : IRlpConverter + public static void Write(ref RlpWriter writer, Dictionary value, RefRlpWriterAction writeKey, RefRlpWriterAction writeValue) { writer.WriteSequence((ref RlpWriter w) => { @@ -41,8 +37,8 @@ public static void Write(ref RlpWriter writer, D { w.WriteSequence((ref RlpWriter w) => { - TKeyConverter.Write(ref w, k); - TValueConverter.Write(ref w, v); + writeKey(ref w, k); + writeValue(ref w, v); }); } }); @@ -51,20 +47,18 @@ public static void Write(ref RlpWriter writer, D public static class DictionaryRlpConverterExt { - public static Dictionary ReadDictionary( - this ref RlpReader reader - ) - where TKey : notnull - where TKeyConverter : IRlpConverter - where TValueConverter : IRlpConverter - => DictionaryRlpConverter.Read(ref reader); + public static Dictionary ReadDictionary( + this ref RlpReader reader, + RefRlpReaderFunc readKey, + RefRlpReaderFunc readValue + ) where TKey : notnull + => DictionaryRlpConverter.Read(ref reader, readKey, readValue); - public static void Write( + public static void Write( this ref RlpWriter writer, - Dictionary value - ) - where TKey : notnull - where TValueConverter : IRlpConverter - where TKeyConverter : IRlpConverter - => DictionaryRlpConverter.Write(ref writer, value); + Dictionary value, + RefRlpWriterAction writeKey, + RefRlpWriterAction writeValue + ) where TKey : notnull + => DictionaryRlpConverter.Write(ref writer, value, writeKey, writeValue); } diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/ListRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/ListRlpConverter.cs index 25a3ad1476a..d6511be8f01 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/ListRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/ListRlpConverter.cs @@ -5,31 +5,29 @@ namespace Nethermind.Serialization.FastRlp.Instances; -// TODO: We might want to introduce an interface for collection types (ex. List) -// Main issue are HKT (aka Generics of Generics) public abstract class ListRlpConverter { - public static List Read(ref RlpReader reader) where TConverter : IRlpConverter + public static List Read(ref RlpReader reader, RefRlpReaderFunc func) { return reader.ReadSequence((scoped ref RlpReader r) => { List result = []; while (r.HasNext) { - result.Add(TConverter.Read(ref r)); + result.Add(func(ref r)); } return result; }); } - public static void Write(ref RlpWriter writer, List value) where TConverter : IRlpConverter + public static void Write(ref RlpWriter writer, List value, RefRlpWriterAction action) { writer.WriteSequence((ref RlpWriter w) => { foreach (T v in value) { - TConverter.Write(ref w, v); + action(ref w, v); } }); } @@ -37,11 +35,9 @@ public static void Write(ref RlpWriter writer, List value) where public static class ListRlpConverterExt { - public static List ReadList(this ref RlpReader reader) - where TConverter : IRlpConverter - => ListRlpConverter.Read(ref reader); + public static List ReadList(this ref RlpReader reader, RefRlpReaderFunc func) + => ListRlpConverter.Read(ref reader, func); - public static void Write(this ref RlpWriter writer, List value) - where TConverter : IRlpConverter - => ListRlpConverter.Write(ref writer, value); + public static void Write(this ref RlpWriter writer, List value, RefRlpWriterAction action) + => ListRlpConverter.Write(ref writer, value, action); } From f484b8babb53d513dab4c50607cc319e17284139 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 16:20:09 -0300 Subject: [PATCH 048/106] Add recursive record test --- .../RlpDerivedTest.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs index 898c67f06a6..7247205f663 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -15,6 +15,9 @@ public record PlayerWithFriends(int Id, string Username, List Friends); [RlpSerializable] public record PlayerWithScores(int Id, string Username, Dictionary Scores); +[RlpSerializable] +public record Tree(string Value, List Children); + public class RlpDerivedTest { [Test] @@ -50,4 +53,20 @@ public void RecordWithDictionary() var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadPlayerWithScores()); decoded.Should().BeEquivalentTo(player); } + + [Test] + public void RecursiveRecord() + { + var tree = new Tree("foo", + [ + new Tree("bar", + [new Tree("dog", [])]), + new Tree("qux", + [new Tree("cat", [])]) + ]); + ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(tree)); + + var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadTree()); + decoded.Should().BeEquivalentTo(tree); + } } From 599da399c6335433bfa2fff2e1a2e2e97204d257 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 16:22:26 -0300 Subject: [PATCH 049/106] Enable `nullable` --- .../RlpSourceGenerator.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs index eb4e9d3a160..ceba931b9e8 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs @@ -108,6 +108,7 @@ private static string GenerateConverterClass( var sb = new StringBuilder(); sb.AppendLine("// "); + sb.AppendLine("#nullable enable"); sb.AppendLine("using System;"); sb.AppendLine("using System.CodeDom.Compiler;"); sb.AppendLine("using Nethermind.Serialization.FastRlp;"); From 3eff5ead0ae7ad4eb00984c5c9e98adf37aa5866 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 16:24:18 -0300 Subject: [PATCH 050/106] Fix `ReadBytes` --- .../RlpSourceGenerator.cs | 6 ++++-- .../Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs index ceba931b9e8..754e42c9026 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs @@ -182,8 +182,10 @@ private static string MapTypeToReadCall(TypeSyntax syntax) // Hard-coded cases switch (syntax.ToString()) { - case "byte[]" or "System.Byte[]" or "Span" or "System.Span" or "System.ReadOnlySpan": - return "ReadBytes"; + case "byte[]" or "Byte[]" or "System.Byte[]": + return "ReadBytes().ToArray()"; + case "Span" or "System.Span" or "ReadOnlySpan" or "System.ReadOnlySpan": + return "ReadBytes()"; } // Generics diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs index 7247205f663..5d60b0e6828 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -18,6 +18,9 @@ public record PlayerWithScores(int Id, string Username, Dictionary [RlpSerializable] public record Tree(string Value, List Children); +[RlpSerializable] +public record RawData(int Tag, byte[] Data); + public class RlpDerivedTest { [Test] From b7889edcc3a5063c8d0fc778f064182e4d7de979 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 16:31:22 -0300 Subject: [PATCH 051/106] Test for all base Integer types --- .../Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs index 5d60b0e6828..dfd88c82fee 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -21,6 +21,9 @@ public record Tree(string Value, List Children); [RlpSerializable] public record RawData(int Tag, byte[] Data); +[RlpSerializable] +public record Integers(short A, int B, long C, Int128 D); + public class RlpDerivedTest { [Test] From d41ecd3fedb4e41325ee5b70a323956868e67391 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 23 Dec 2024 16:36:36 -0300 Subject: [PATCH 052/106] Initial README --- .../Nethermind.Serialization.FastRlp/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/Nethermind/Nethermind.Serialization.FastRlp/README.md diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/README.md b/src/Nethermind/Nethermind.Serialization.FastRlp/README.md new file mode 100644 index 00000000000..c5e657290ec --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/README.md @@ -0,0 +1,11 @@ +# FastRlp + +Declarative RLP encoding a decoding with support for extensibility through manually written `IRlpConverter`s and automatically generated through attributes. + +## TODO + +- Avoid the need for closures through the usage of an extra "context" argument +- Add support more instances for base types +- Add support for generic Arrays (`X[]`) +- Alternative API for writing based on `Async` and `Stream` +- Support for parameterizable names when using attributes From 03325a044c9804fc8ed18745424c3016765dc80c Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 10:36:13 -0300 Subject: [PATCH 053/106] Make `Read` calls static --- .../RlpDerivedTest.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs index dfd88c82fee..7ec8447820b 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -32,7 +32,7 @@ public void FlatRecord() var player = new Player(Id: 42, Username: "SuperUser"); ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); - var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadPlayer()); + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayer()); decoded.Should().BeEquivalentTo(player); } @@ -42,7 +42,7 @@ public void RecordWithList() var player = new PlayerWithFriends(Id: 42, Username: "SuperUser", Friends: ["ana", "bob"]); ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); - var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadPlayerWithFriends()); + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayerWithFriends()); decoded.Should().BeEquivalentTo(player); } @@ -56,7 +56,7 @@ public void RecordWithDictionary() }); ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); - var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadPlayerWithScores()); + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayerWithScores()); decoded.Should().BeEquivalentTo(player); } @@ -72,7 +72,7 @@ [new Tree("cat", [])]) ]); ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(tree)); - var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadTree()); + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadTree()); decoded.Should().BeEquivalentTo(tree); } } From 914505a04fc774642b6faf591cc859eb717149d9 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 10:42:12 -0300 Subject: [PATCH 054/106] Add `context` overload - Allows for pure static lambdas --- .../RlpDerivedTest.cs | 2 +- .../Nethermind.Serialization.FastRlp/Rlp.cs | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs index 7ec8447820b..d16b53b56ec 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -30,7 +30,7 @@ public class RlpDerivedTest public void FlatRecord() { var player = new Player(Id: 42, Username: "SuperUser"); - ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); + ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, Player player) => w.Write(player)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayer()); decoded.Should().BeEquivalentTo(player); diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs index 257fce868f2..1c9fc39b618 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs @@ -18,7 +18,20 @@ public static byte[] Write(RefRlpWriterAction action) return serialized; } - public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) where T : allows ref struct + public static ReadOnlySpan Write(TContext ctx, RefRlpWriterAction action) + where TContext : allows ref struct + { + var lengthWriter = RlpWriter.LengthWriter(); + action(ref lengthWriter, ctx); + var serialized = new byte[lengthWriter.Length]; + var contentWriter = RlpWriter.ContentWriter(serialized); + action(ref contentWriter, ctx); + + return serialized; + } + + public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) + where T : allows ref struct { var reader = new RlpReader(source); T result = func(ref reader); From 7dbc2dc5a659fb0d5f44f2e8b01ed30edb4dcce8 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 10:42:24 -0300 Subject: [PATCH 055/106] Remove allocations --- src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs index 1c9fc39b618..b825db9a86f 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs @@ -41,10 +41,7 @@ public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) public static void Read(ReadOnlySpan source, RefRlpReaderAction func) { - Read(source, (scoped ref RlpReader reader) => - { - func(ref reader); - return null; - }); + var reader = new RlpReader(source); + func(ref reader); } } From 15eb6b683efb68d339b5acf5ccb71807ca5784d2 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 10:44:58 -0300 Subject: [PATCH 056/106] Use `static` on `Write` --- .../Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs index d16b53b56ec..8a22c1580f7 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs @@ -40,7 +40,7 @@ public void FlatRecord() public void RecordWithList() { var player = new PlayerWithFriends(Id: 42, Username: "SuperUser", Friends: ["ana", "bob"]); - ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); + ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithFriends player) => w.Write(player)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayerWithFriends()); decoded.Should().BeEquivalentTo(player); @@ -54,7 +54,7 @@ public void RecordWithDictionary() { "foo", 42 }, { "bar", 1337 } }); - ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(player)); + ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithScores player) => w.Write(player)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayerWithScores()); decoded.Should().BeEquivalentTo(player); @@ -70,7 +70,7 @@ [new Tree("dog", [])]), new Tree("qux", [new Tree("cat", [])]) ]); - ReadOnlySpan rlp = Rlp.Write((ref RlpWriter w) => w.Write(tree)); + ReadOnlySpan rlp = Rlp.Write(tree, static (ref RlpWriter w, Tree tree) => w.Write(tree)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadTree()); decoded.Should().BeEquivalentTo(tree); From c42eb50efcb8f90ddcbeb877fda1e2556eb9c66e Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 10:46:18 -0300 Subject: [PATCH 057/106] Use `static` on `Read` --- .../Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs index 201198ec3e2..a29d4274716 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs @@ -86,7 +86,7 @@ public void ReadEmptyList() var actual = Rlp.Read(source, static (scoped ref RlpReader r) => { - return r.ReadSequence((scoped ref RlpReader _) => Array.Empty()); + return r.ReadSequence(static (scoped ref RlpReader _) => Array.Empty()); }); actual.Should().BeEmpty(); From ed0d88b609a0dfce632975a1431fb3bca972a654 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 10:49:03 -0300 Subject: [PATCH 058/106] Return `byte[]` instead of `ReadOnlySpan` in overload --- src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs index b825db9a86f..64b8075caa8 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs @@ -18,7 +18,7 @@ public static byte[] Write(RefRlpWriterAction action) return serialized; } - public static ReadOnlySpan Write(TContext ctx, RefRlpWriterAction action) + public static byte[] Write(TContext ctx, RefRlpWriterAction action) where TContext : allows ref struct { var lengthWriter = RlpWriter.LengthWriter(); From 88a8cc462967f82d93492c8d175fc0395a4a810f Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 10:50:42 -0300 Subject: [PATCH 059/106] Use `static` on `Write` --- .../Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs index 48164c084e3..abdeace1a93 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs @@ -45,7 +45,7 @@ public void WriteInteger_1Component() for (int i = 0; i < 0x80; i++) { var integer = i; - var serialized = Rlp.Write((ref RlpWriter w) => { w.Write(integer); }); + var serialized = Rlp.Write(integer, static (ref RlpWriter w, int integer) => { w.Write(integer); }); byte[] expected = [(byte)integer]; serialized.Should().BeEquivalentTo(expected); @@ -59,7 +59,7 @@ public void WriteInteger_2Components() for (int i = 0x80; i < 0x0100; i++) { var integer = i; - var serialized = Rlp.Write((ref RlpWriter w) => { w.Write(integer); }); + var serialized = Rlp.Write(integer, static (ref RlpWriter w, int integer) => { w.Write(integer); }); expected[1] = (byte)integer; serialized.Should().BeEquivalentTo(expected); @@ -73,7 +73,7 @@ public void WriteInteger_3Components() for (int i = 0x100; i < 0xFFFF; i++) { var integer = i; - var serialized = Rlp.Write((ref RlpWriter w) => { w.Write(integer); }); + var serialized = Rlp.Write(integer, static (ref RlpWriter w, int integer) => { w.Write(integer); }); expected[1] = (byte)((integer & 0xFF00) >> 8); expected[2] = (byte)((integer & 0x00FF) >> 0); From a596ab03674101dda0f2edc36e4340eafb6b00d3 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 11:04:14 -0300 Subject: [PATCH 060/106] Fix typo --- .../Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs index e638270ce4a..6c5bb2816ff 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs @@ -77,7 +77,7 @@ public void LongList() } [Test] - public void MutlipleLongList() + public void MultipleLongList() { var rlp = Rlp.Write(static (ref RlpWriter w) => { From df3ffc9f19e1d3b7b02d06c4225c2b1293896e99 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 11:20:22 -0300 Subject: [PATCH 061/106] Add overloads for `TContext` --- .../RlpWriter.cs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs index e445d78de5c..825c8827583 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs @@ -194,4 +194,56 @@ private void ContentWriteSequence(RefRlpWriterAction action) action(ref this); } + + // TODO: Figure out how to unify overloads + public void WriteSequence(TContext ctx, RefRlpWriterAction action) + { + switch (_mode) + { + case LengthMode: + LengthWriteSequence(ctx, action); + break; + case ContentMode: + ContentWriteSequence(ctx, action); + break; + } + } + + private void LengthWriteSequence(TContext ctx, RefRlpWriterAction action) + { + var inner = LengthWriter(); + action(ref inner, ctx); + if (inner.Length < 55) + { + Length += 1 + inner.Length; + } + else + { + Span binaryLength = stackalloc byte[sizeof(Int32)]; + BinaryPrimitives.WriteInt32BigEndian(binaryLength, inner.Length); + binaryLength = binaryLength.TrimStart((byte)0); + Length += 1 + inner.Length + binaryLength.Length; + } + } + + private void ContentWriteSequence(TContext ctx, RefRlpWriterAction action) + { + var lengthWriter = LengthWriter(); + action(ref lengthWriter, ctx); + if (lengthWriter.Length < 55) + { + _buffer[_position++] = (byte)(0xC0 + lengthWriter.Length); + } + else + { + Span binaryLength = stackalloc byte[sizeof(Int32)]; + BinaryPrimitives.WriteInt32BigEndian(binaryLength, lengthWriter.Length); + binaryLength = binaryLength.TrimStart((byte)0); + _buffer[_position++] = (byte)(0xF7 + binaryLength.Length); + binaryLength.CopyTo(_buffer.AsSpan()[_position..]); + _position += binaryLength.Length; + } + + action(ref this, ctx); + } } From 42b44018c2c9c3927e1fd27026be291dec3e0efb Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 11:20:43 -0300 Subject: [PATCH 062/106] Use `context` overload when possible --- .../RlpReadWriteTest.cs | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs index 6c5bb2816ff..97762e8e444 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs @@ -97,9 +97,9 @@ public void MultipleLongList() }); }); - var (dogs, cats) = Rlp.Read(rlp, (scoped ref RlpReader r) => + var (dogs, cats) = Rlp.Read(rlp, static (scoped ref RlpReader r) => { - var dogs = r.ReadSequence((scoped ref RlpReader r) => + var dogs = r.ReadSequence(static (scoped ref RlpReader r) => { List result = []; while (r.HasNext) @@ -109,7 +109,7 @@ public void MultipleLongList() return result; }); - var cats = r.ReadSequence((scoped ref RlpReader r) => + var cats = r.ReadSequence(static (scoped ref RlpReader r) => { List result = []; while (r.HasNext) @@ -133,9 +133,9 @@ public void MultipleLongList() [TestCase(2)] public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) { - var rlp = Rlp.Write((ref RlpWriter root) => + var rlp = Rlp.Write(length, static (ref RlpWriter root, int length) => { - root.WriteSequence((ref RlpWriter w) => + root.WriteSequence(length, static (ref RlpWriter w, int length) => { for (int i = 0; i < length; i++) { @@ -253,9 +253,9 @@ public void UserDefinedRecord() }), ]; - var rlp = Rlp.Write((ref RlpWriter w) => + var rlp = Rlp.Write(students, static (ref RlpWriter w, List students) => { - w.WriteSequence((ref RlpWriter w) => + w.WriteSequence(students, static (ref RlpWriter w, List students) => { foreach (var student in students) { @@ -286,7 +286,7 @@ public void ListCollection() { var list = new List { "cat", "dog" }; - var rlp = Rlp.Write((ref RlpWriter w) => w.Write(list, StringRlpConverter.Write)); + var rlp = Rlp.Write(list, static (ref RlpWriter w, List list) => w.Write(list, StringRlpConverter.Write)); var rlpExplicit = Rlp.Write(static (ref RlpWriter w) => { @@ -312,13 +312,13 @@ public void ListOfListCollection() [] ]; - var rlp = Rlp.Write((ref RlpWriter w) => - w.Write(list, (ref RlpWriter w, List v) => + var rlp = Rlp.Write(list, static (ref RlpWriter w, List> list) => + w.Write(list, static (ref RlpWriter w, List v) => w.Write(v, StringRlpConverter.Write))); var rlpExplicit = Rlp.Write(static (ref RlpWriter w) => { - w.WriteSequence((ref RlpWriter w) => + w.WriteSequence(static (ref RlpWriter w) => { w.WriteSequence(static (ref RlpWriter w) => { @@ -339,7 +339,7 @@ public void ListOfListCollection() rlpExplicit.Should().BeEquivalentTo(rlp); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => - r.ReadList((scoped ref RlpReader r) => + r.ReadList(static (scoped ref RlpReader r) => r.ReadList(StringRlpConverter.Read))); list.Should().BeEquivalentTo(decoded); @@ -354,19 +354,19 @@ public void DictionaryCollection() { 2, "cat" }, }; - var rlp = Rlp.Write((ref RlpWriter w) => + var rlp = Rlp.Write(dictionary, static (ref RlpWriter w, Dictionary dictionary) => w.Write(dictionary, Int32RlpConverter.Write, StringRlpConverter.Write)); - var rlpExplicit = Rlp.Write((ref RlpWriter w) => + var rlpExplicit = Rlp.Write(dictionary, static (ref RlpWriter w, Dictionary dictionary) => { - w.WriteSequence((ref RlpWriter w) => + w.WriteSequence(dictionary, static (ref RlpWriter w,Dictionary dictionary) => { - foreach (var (k, v) in dictionary) + foreach (var tuple in dictionary) { - w.WriteSequence((ref RlpWriter w) => + w.WriteSequence(tuple, static (ref RlpWriter w, KeyValuePair tuple) => { - w.Write(k); - w.Write(v); + w.Write(tuple.Key); + w.Write(tuple.Value); }); } }); From 917a7b8bcd542d39cc17beb42ecb381e29791a9e Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 11:50:37 -0300 Subject: [PATCH 063/106] Remove duplicated overloads --- .../Nethermind.Serialization.FastRlp/Rlp.cs | 10 +--- .../RlpWriter.cs | 51 +------------------ 2 files changed, 2 insertions(+), 59 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs index 64b8075caa8..d5369a54f1c 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs @@ -8,15 +8,7 @@ namespace Nethermind.Serialization.FastRlp; public static class Rlp { public static byte[] Write(RefRlpWriterAction action) - { - var lengthWriter = RlpWriter.LengthWriter(); - action(ref lengthWriter); - var serialized = new byte[lengthWriter.Length]; - var contentWriter = RlpWriter.ContentWriter(serialized); - action(ref contentWriter); - - return serialized; - } + => Write(action, static (ref RlpWriter w, RefRlpWriterAction action) => action(ref w)); public static byte[] Write(TContext ctx, RefRlpWriterAction action) where TContext : allows ref struct diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs index 825c8827583..118931ec097 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs @@ -145,57 +145,8 @@ private void ContentWrite(scoped ReadOnlySpan value) } public void WriteSequence(RefRlpWriterAction action) - { - switch (_mode) - { - case LengthMode: - LengthWriteSequence(action); - break; - case ContentMode: - ContentWriteSequence(action); - break; - } - } - - private void LengthWriteSequence(RefRlpWriterAction action) - { - var inner = LengthWriter(); - action(ref inner); - if (inner.Length < 55) - { - Length += 1 + inner.Length; - } - else - { - Span binaryLength = stackalloc byte[sizeof(Int32)]; - BinaryPrimitives.WriteInt32BigEndian(binaryLength, inner.Length); - binaryLength = binaryLength.TrimStart((byte)0); - Length += 1 + inner.Length + binaryLength.Length; - } - } - - private void ContentWriteSequence(RefRlpWriterAction action) - { - var lengthWriter = LengthWriter(); - action(ref lengthWriter); - if (lengthWriter.Length < 55) - { - _buffer[_position++] = (byte)(0xC0 + lengthWriter.Length); - } - else - { - Span binaryLength = stackalloc byte[sizeof(Int32)]; - BinaryPrimitives.WriteInt32BigEndian(binaryLength, lengthWriter.Length); - binaryLength = binaryLength.TrimStart((byte)0); - _buffer[_position++] = (byte)(0xF7 + binaryLength.Length); - binaryLength.CopyTo(_buffer.AsSpan()[_position..]); - _position += binaryLength.Length; - } - - action(ref this); - } + => WriteSequence(action, static (ref RlpWriter w, RefRlpWriterAction action) => action(ref w)); - // TODO: Figure out how to unify overloads public void WriteSequence(TContext ctx, RefRlpWriterAction action) { switch (_mode) From 88e7bacad76c0ad4c31fbf116643c92cb91b9658 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 11:51:26 -0300 Subject: [PATCH 064/106] Remove "context" TODO --- src/Nethermind/Nethermind.Serialization.FastRlp/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/README.md b/src/Nethermind/Nethermind.Serialization.FastRlp/README.md index c5e657290ec..c93d2adbeef 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/README.md +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/README.md @@ -4,7 +4,6 @@ Declarative RLP encoding a decoding with support for extensibility through manua ## TODO -- Avoid the need for closures through the usage of an extra "context" argument - Add support more instances for base types - Add support for generic Arrays (`X[]`) - Alternative API for writing based on `Async` and `Stream` From 1bc2d38671bcf4f00fac39dfab8fa83a5a68e150 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 11:57:58 -0300 Subject: [PATCH 065/106] Remove allocating overload - Not actually needed in practice --- .../RlpReadWriteTest.cs | 5 ++++- .../Nethermind.Serialization.FastRlp/RlpReader.cs | 9 --------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs index 97762e8e444..10d12516df8 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs @@ -165,7 +165,10 @@ public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) public void InvalidObjectReading() { var rlp = Rlp.Write(static (ref RlpWriter w) => { w.Write(42); }); - Action tryRead = () => Rlp.Read(rlp, (scoped ref RlpReader r) => { r.ReadSequence((scoped ref RlpReader _) => { }); }); + Action tryRead = () => Rlp.Read(rlp, static (scoped ref RlpReader r) => + { + return r.ReadSequence(static (scoped ref RlpReader _) => null as object); + }); tryRead.Should().Throw(); } diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs index 591ded438dc..abf48f343f0 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs @@ -102,15 +102,6 @@ public T ReadSequence(RefRlpReaderFunc func) return result; } - public void ReadSequence(RefRlpReaderAction func) - { - ReadSequence((scoped ref RlpReader r) => - { - func(ref r); - return null; - }); - } - public T Choice(params ReadOnlySpan> alternatives) { int startingPosition = _position; From fb254e9b6d97b863425f5faa7fb8bb75465fa317 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 12:00:18 -0300 Subject: [PATCH 066/106] More `static` usage --- .../RlpReadWriteTest.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs index 10d12516df8..5d0ec2d0092 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs @@ -58,9 +58,9 @@ public void LongList() }); }); - List decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => + List decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => { - return r.ReadSequence((scoped ref RlpReader r) => + return r.ReadSequence(static (scoped ref RlpReader r) => { List result = []; for (int i = 0; i < 100; i++) @@ -144,9 +144,9 @@ public void UnknownLengthList([Values(1, 3, 5, 10, 20)] int length) }); }); - List decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => + List decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => { - return r.ReadSequence((scoped ref RlpReader r) => + return r.ReadSequence(static (scoped ref RlpReader r) => { List result = []; while (r.HasNext) @@ -177,7 +177,7 @@ public void InvalidObjectReading() public void InvalidListReading() { var rlp = Rlp.Write(static (ref RlpWriter w) => { w.WriteSequence(static (ref RlpWriter _) => { }); }); - Func tryRead = () => Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadInt32()); + Func tryRead = () => Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadInt32()); tryRead.Should().Throw(); } @@ -185,7 +185,7 @@ public void InvalidListReading() [Test] public void Choice() { - RefRlpReaderFunc intReader = (scoped ref RlpReader r) => r.ReadInt32(); + RefRlpReaderFunc intReader = static (scoped ref RlpReader r) => r.ReadInt32(); RefRlpReaderFunc wrappedReader = (scoped ref RlpReader r) => r.ReadSequence(intReader); var intRlp = Rlp.Write(static (ref RlpWriter w) => { w.Write(42); }); From 21a18b9006f9d1268d18d0f3e68ade640f3adb8b Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 24 Dec 2024 12:17:35 -0300 Subject: [PATCH 067/106] Use "context" overload in Generator --- .../RlpSourceGenerator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs index 754e42c9026..63a721c3c87 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs @@ -123,7 +123,7 @@ private static string GenerateConverterClass( // `Write` method sb.AppendLine($"public static void Write(ref RlpWriter w, {fullTypeName} value)"); sb.AppendLine("{"); - sb.AppendLine("w.WriteSequence((ref RlpWriter w) => "); + sb.AppendLine($"w.WriteSequence(value, static (ref RlpWriter w, {fullTypeName} value) => "); sb.AppendLine("{"); foreach (var (name, typeName) in parameters) From 4740466acae9aa542b475ba3ce514ec7a6f5d075 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 10:17:42 -0300 Subject: [PATCH 068/106] Annotate as `scoped` --- src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs index 118931ec097..3640c02614e 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs @@ -94,7 +94,7 @@ private void ContentWrite(T value) where T : IBinaryInteger, ISignedNumber } } - public void Write(ReadOnlySpan value) + public void Write(scoped ReadOnlySpan value) { switch (_mode) { From 5705da18c3901b21ba1640c48aaaabb0a1efe331 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 10:26:26 -0300 Subject: [PATCH 069/106] Optimize `StringRlpConverter` --- .../RlpReadWriteTest.cs | 14 +++++++++++ .../Instances/StringRlpConverter.cs | 24 +++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs index 5d0ec2d0092..d4f8e70c845 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs @@ -8,6 +8,20 @@ namespace Nethermind.Serialization.FastRlp.Test; public class RlpReadWriteTest { + [Test] + public void LongString() + { + var rlp = Rlp.Write(static (ref RlpWriter w) => + { + var str = new string('A', 2000); + w.Write(str); + }); + + var decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.ReadString()); + + decoded.Should().Be(new string('A', 2000)); + } + [Test] public void HeterogeneousList() { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/StringRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/StringRlpConverter.cs index 0d5ecff6354..6cb51afd9ef 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/StringRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/StringRlpConverter.cs @@ -2,12 +2,16 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers; using System.Text; namespace Nethermind.Serialization.FastRlp.Instances; public abstract class StringRlpConverter : IRlpConverter { + private const int MaxStackSize = 256; + private static readonly Encoding Encoding = Encoding.UTF8; + public static string Read(ref RlpReader reader) { ReadOnlySpan obj = reader.ReadBytes(); @@ -16,8 +20,24 @@ public static string Read(ref RlpReader reader) public static void Write(ref RlpWriter writer, string value) { - ReadOnlySpan bytes = Encoding.UTF8.GetBytes(value); - writer.Write(bytes); + ReadOnlySpan charSpan = value.AsSpan(); + var valueByteLength = Encoding.GetMaxByteCount(charSpan.Length); + + byte[]? sharedBuffer = null; + try + { + Span buffer = valueByteLength <= MaxStackSize + ? stackalloc byte[valueByteLength] + : sharedBuffer = ArrayPool.Shared.Rent(valueByteLength); + + var bytes = Encoding.GetBytes(charSpan, buffer); + + writer.Write(buffer[..bytes]); + } + finally + { + if (sharedBuffer is not null) ArrayPool.Shared.Return(sharedBuffer); + } } } From cc5877647778904e0f6fe27cee4de2c7f7716d90 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 10:33:05 -0300 Subject: [PATCH 070/106] Rename `FastRlp` to `FluentRlp` - It's not that "fast" --- .../Nethermind.Serialization.FluentRlp.Generator.csproj} | 1 + .../RlpSourceGenerator.cs | 6 +++--- .../Nethermind.Serialization.FluentRlp.Test.csproj} | 4 ++-- .../RlpDerivedTest.cs | 4 ++-- .../RlpReadWriteTest.cs | 4 ++-- .../RlpReaderTest.cs | 4 ++-- .../RlpWriterTest.cs | 4 ++-- .../Student.cs | 6 +++--- .../IRlpConverter.cs | 2 +- .../Instances/DictionaryRlpConverter.cs | 2 +- .../Instances/IntegerRlpConverter.cs | 2 +- .../Instances/ListRlpConverter.cs | 2 +- .../Instances/StringRlpConverter.cs | 2 +- .../Int32Primitive.cs | 2 +- .../Nethermind.Serialization.FluentRlp.csproj} | 0 .../README.md | 2 +- .../Rlp.cs | 2 +- .../RlpReader.cs | 2 +- .../RlpWriter.cs | 2 +- src/Nethermind/Nethermind.sln | 6 +++--- 20 files changed, 30 insertions(+), 29 deletions(-) rename src/Nethermind/{Nethermind.Serialization.FastRlp.Generator/Nethermind.Serialization.FastRlp.Generator.csproj => Nethermind.Serialization.FluentRlp.Generator/Nethermind.Serialization.FluentRlp.Generator.csproj} (81%) rename src/Nethermind/{Nethermind.Serialization.FastRlp.Generator => Nethermind.Serialization.FluentRlp.Generator}/RlpSourceGenerator.cs (97%) rename src/Nethermind/{Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj => Nethermind.Serialization.FluentRlp.Test/Nethermind.Serialization.FluentRlp.Test.csproj} (73%) rename src/Nethermind/{Nethermind.Serialization.FastRlp.Test => Nethermind.Serialization.FluentRlp.Test}/RlpDerivedTest.cs (95%) rename src/Nethermind/{Nethermind.Serialization.FastRlp.Test => Nethermind.Serialization.FluentRlp.Test}/RlpReadWriteTest.cs (99%) rename src/Nethermind/{Nethermind.Serialization.FastRlp.Test => Nethermind.Serialization.FluentRlp.Test}/RlpReaderTest.cs (97%) rename src/Nethermind/{Nethermind.Serialization.FastRlp.Test => Nethermind.Serialization.FluentRlp.Test}/RlpWriterTest.cs (97%) rename src/Nethermind/{Nethermind.Serialization.FastRlp.Test => Nethermind.Serialization.FluentRlp.Test}/Student.cs (92%) rename src/Nethermind/{Nethermind.Serialization.FastRlp => Nethermind.Serialization.FluentRlp}/IRlpConverter.cs (86%) rename src/Nethermind/{Nethermind.Serialization.FastRlp => Nethermind.Serialization.FluentRlp}/Instances/DictionaryRlpConverter.cs (97%) rename src/Nethermind/{Nethermind.Serialization.FastRlp => Nethermind.Serialization.FluentRlp}/Instances/IntegerRlpConverter.cs (96%) rename src/Nethermind/{Nethermind.Serialization.FastRlp => Nethermind.Serialization.FluentRlp}/Instances/ListRlpConverter.cs (95%) rename src/Nethermind/{Nethermind.Serialization.FastRlp => Nethermind.Serialization.FluentRlp}/Instances/StringRlpConverter.cs (96%) rename src/Nethermind/{Nethermind.Serialization.FastRlp => Nethermind.Serialization.FluentRlp}/Int32Primitive.cs (94%) rename src/Nethermind/{Nethermind.Serialization.FastRlp/Nethermind.Serialization.FastRlp.csproj => Nethermind.Serialization.FluentRlp/Nethermind.Serialization.FluentRlp.csproj} (100%) rename src/Nethermind/{Nethermind.Serialization.FastRlp => Nethermind.Serialization.FluentRlp}/README.md (96%) rename src/Nethermind/{Nethermind.Serialization.FastRlp => Nethermind.Serialization.FluentRlp}/Rlp.cs (96%) rename src/Nethermind/{Nethermind.Serialization.FastRlp => Nethermind.Serialization.FluentRlp}/RlpReader.cs (98%) rename src/Nethermind/{Nethermind.Serialization.FastRlp => Nethermind.Serialization.FluentRlp}/RlpWriter.cs (99%) diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/Nethermind.Serialization.FastRlp.Generator.csproj b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/Nethermind.Serialization.FluentRlp.Generator.csproj similarity index 81% rename from src/Nethermind/Nethermind.Serialization.FastRlp.Generator/Nethermind.Serialization.FastRlp.Generator.csproj rename to src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/Nethermind.Serialization.FluentRlp.Generator.csproj index b47244e5b00..f8d13657465 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/Nethermind.Serialization.FastRlp.Generator.csproj +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/Nethermind.Serialization.FluentRlp.Generator.csproj @@ -4,6 +4,7 @@ netstandard2.1 enable true + Nethermind.Serialization.FluentRlp.Generator diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs similarity index 97% rename from src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index 63a721c3c87..f45fffb49da 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -10,7 +10,7 @@ using Microsoft.CodeAnalysis.Text; using System.Text; -namespace Nethermind.Serialization.FastRlp.Generator; +namespace Nethermind.Serialization.FluentRlp.Generator; [AttributeUsage(AttributeTargets.Class)] public sealed class RlpSerializable : Attribute; @@ -111,8 +111,8 @@ private static string GenerateConverterClass( sb.AppendLine("#nullable enable"); sb.AppendLine("using System;"); sb.AppendLine("using System.CodeDom.Compiler;"); - sb.AppendLine("using Nethermind.Serialization.FastRlp;"); - sb.AppendLine("using Nethermind.Serialization.FastRlp.Instances;"); + sb.AppendLine("using Nethermind.Serialization.FluentRlp;"); + sb.AppendLine("using Nethermind.Serialization.FluentRlp.Instances;"); sb.AppendLine(); if (@namespace is not null) sb.AppendLine($"namespace {@namespace};"); sb.AppendLine(""); diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Nethermind.Serialization.FluentRlp.Test.csproj similarity index 73% rename from src/Nethermind/Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj rename to src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Nethermind.Serialization.FluentRlp.Test.csproj index 4f49e9de0d2..131fbae2d89 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Nethermind.Serialization.FastRlp.Test.csproj +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Nethermind.Serialization.FluentRlp.Test.csproj @@ -22,8 +22,8 @@ - - + + diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs similarity index 95% rename from src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs index 8a22c1580f7..c2a3acd9c2b 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using FluentAssertions; -using Nethermind.Serialization.FastRlp.Generator; +using Nethermind.Serialization.FluentRlp.Generator; -namespace Nethermind.Serialization.FastRlp.Test; +namespace Nethermind.Serialization.FluentRlp.Test; [RlpSerializable] public record Player(int Id, string Username); diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs similarity index 99% rename from src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs index d4f8e70c845..cb728d08480 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using FluentAssertions; -using Nethermind.Serialization.FastRlp.Instances; +using Nethermind.Serialization.FluentRlp.Instances; -namespace Nethermind.Serialization.FastRlp.Test; +namespace Nethermind.Serialization.FluentRlp.Test; public class RlpReadWriteTest { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReaderTest.cs similarity index 97% rename from src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReaderTest.cs index a29d4274716..713ac40695f 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReaderTest.cs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using FluentAssertions; -using Nethermind.Serialization.FastRlp.Instances; +using Nethermind.Serialization.FluentRlp.Instances; -namespace Nethermind.Serialization.FastRlp.Test; +namespace Nethermind.Serialization.FluentRlp.Test; public class RlpReaderTest { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpWriterTest.cs similarity index 97% rename from src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpWriterTest.cs index abdeace1a93..85bd3c84b9f 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpWriterTest.cs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using FluentAssertions; -using Nethermind.Serialization.FastRlp.Instances; +using Nethermind.Serialization.FluentRlp.Instances; -namespace Nethermind.Serialization.FastRlp.Test; +namespace Nethermind.Serialization.FluentRlp.Test; [Parallelizable(ParallelScope.All)] public class RlpWriterTest diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Student.cs similarity index 92% rename from src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Student.cs index 50ccfeb9f29..07d08706e11 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp.Test/Student.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Student.cs @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using Nethermind.Serialization.FastRlp.Generator; -using Nethermind.Serialization.FastRlp.Instances; +using Nethermind.Serialization.FluentRlp.Generator; +using Nethermind.Serialization.FluentRlp.Instances; -namespace Nethermind.Serialization.FastRlp.Test; +namespace Nethermind.Serialization.FluentRlp.Test; public record Student(string Name, int Age, Dictionary Scores); diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/IRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/IRlpConverter.cs similarity index 86% rename from src/Nethermind/Nethermind.Serialization.FastRlp/IRlpConverter.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp/IRlpConverter.cs index 4a3a8b8a415..5fd7a14ed04 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/IRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/IRlpConverter.cs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -namespace Nethermind.Serialization.FastRlp; +namespace Nethermind.Serialization.FluentRlp; public interface IRlpConverter where T : allows ref struct { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/DictionaryRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/DictionaryRlpConverter.cs similarity index 97% rename from src/Nethermind/Nethermind.Serialization.FastRlp/Instances/DictionaryRlpConverter.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/DictionaryRlpConverter.cs index e83ea46d966..356a1970702 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/DictionaryRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/DictionaryRlpConverter.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; -namespace Nethermind.Serialization.FastRlp.Instances; +namespace Nethermind.Serialization.FluentRlp.Instances; public abstract class DictionaryRlpConverter where TKey : notnull { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/IntegerRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/IntegerRlpConverter.cs similarity index 96% rename from src/Nethermind/Nethermind.Serialization.FastRlp/Instances/IntegerRlpConverter.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/IntegerRlpConverter.cs index f1ebd0ae2da..75b004a07eb 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/IntegerRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/IntegerRlpConverter.cs @@ -3,7 +3,7 @@ using System; -namespace Nethermind.Serialization.FastRlp.Instances; +namespace Nethermind.Serialization.FluentRlp.Instances; public abstract class Int16RlpConverter : IRlpConverter { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/ListRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/ListRlpConverter.cs similarity index 95% rename from src/Nethermind/Nethermind.Serialization.FastRlp/Instances/ListRlpConverter.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/ListRlpConverter.cs index d6511be8f01..e72dd9b99de 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/ListRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/ListRlpConverter.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; -namespace Nethermind.Serialization.FastRlp.Instances; +namespace Nethermind.Serialization.FluentRlp.Instances; public abstract class ListRlpConverter { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/StringRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/StringRlpConverter.cs similarity index 96% rename from src/Nethermind/Nethermind.Serialization.FastRlp/Instances/StringRlpConverter.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/StringRlpConverter.cs index 6cb51afd9ef..b62ff1943c3 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Instances/StringRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/StringRlpConverter.cs @@ -5,7 +5,7 @@ using System.Buffers; using System.Text; -namespace Nethermind.Serialization.FastRlp.Instances; +namespace Nethermind.Serialization.FluentRlp.Instances; public abstract class StringRlpConverter : IRlpConverter { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Int32Primitive.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Int32Primitive.cs similarity index 94% rename from src/Nethermind/Nethermind.Serialization.FastRlp/Int32Primitive.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp/Int32Primitive.cs index a028914072f..f47c8700ce2 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Int32Primitive.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Int32Primitive.cs @@ -4,7 +4,7 @@ using System; using System.Buffers.Binary; -namespace Nethermind.Serialization.FastRlp; +namespace Nethermind.Serialization.FluentRlp; internal static class Int32Primitive { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Nethermind.Serialization.FastRlp.csproj b/src/Nethermind/Nethermind.Serialization.FluentRlp/Nethermind.Serialization.FluentRlp.csproj similarity index 100% rename from src/Nethermind/Nethermind.Serialization.FastRlp/Nethermind.Serialization.FastRlp.csproj rename to src/Nethermind/Nethermind.Serialization.FluentRlp/Nethermind.Serialization.FluentRlp.csproj diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/README.md b/src/Nethermind/Nethermind.Serialization.FluentRlp/README.md similarity index 96% rename from src/Nethermind/Nethermind.Serialization.FastRlp/README.md rename to src/Nethermind/Nethermind.Serialization.FluentRlp/README.md index c93d2adbeef..42894725a01 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/README.md +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/README.md @@ -1,4 +1,4 @@ -# FastRlp +# FluentRlp Declarative RLP encoding a decoding with support for extensibility through manually written `IRlpConverter`s and automatically generated through attributes. diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs similarity index 96% rename from src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs index d5369a54f1c..a4dfad2a1d7 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs @@ -3,7 +3,7 @@ using System; -namespace Nethermind.Serialization.FastRlp; +namespace Nethermind.Serialization.FluentRlp; public static class Rlp { diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs similarity index 98% rename from src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs index abf48f343f0..854014b9728 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs @@ -5,7 +5,7 @@ using System.Numerics; using System.Runtime.InteropServices; -namespace Nethermind.Serialization.FastRlp; +namespace Nethermind.Serialization.FluentRlp; public delegate TResult RefRlpReaderFunc(scoped ref RlpReader arg) where TResult : allows ref struct; diff --git a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs similarity index 99% rename from src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs rename to src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs index 3640c02614e..0014b85d19f 100644 --- a/src/Nethermind/Nethermind.Serialization.FastRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs @@ -6,7 +6,7 @@ using System.Numerics; using System.Runtime.InteropServices; -namespace Nethermind.Serialization.FastRlp; +namespace Nethermind.Serialization.FluentRlp; public delegate void RefRlpWriterAction(ref RlpWriter arg, T value) where T: allows ref struct; diff --git a/src/Nethermind/Nethermind.sln b/src/Nethermind/Nethermind.sln index 69cf44f6bf5..0b14cff7513 100644 --- a/src/Nethermind/Nethermind.sln +++ b/src/Nethermind/Nethermind.sln @@ -226,11 +226,11 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Shutter", "Nethe EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Shutter.Test", "Nethermind.Shutter.Test\Nethermind.Shutter.Test.csproj", "{CEA1C413-A96C-4339-AC1C-839B603DECC8}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.FastRlp.Test", "Nethermind.Serialization.FastRlp.Test\Nethermind.Serialization.FastRlp.Test.csproj", "{C8A91B54-F9CA-4211-BE16-F7A0B38223EB}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.FluentRlp.Test", "Nethermind.Serialization.FluentRlp.Test\Nethermind.Serialization.FluentRlp.Test.csproj", "{C8A91B54-F9CA-4211-BE16-F7A0B38223EB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.FastRlp", "Nethermind.Serialization.FastRlp\Nethermind.Serialization.FastRlp.csproj", "{3D358FD8-E047-4770-BCDF-8FBBBBDDE03A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.FluentRlp", "Nethermind.Serialization.FluentRlp\Nethermind.Serialization.FluentRlp.csproj", "{3D358FD8-E047-4770-BCDF-8FBBBBDDE03A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.FastRlp.Generator", "Nethermind.Serialization.FastRlp.Generator\Nethermind.Serialization.FastRlp.Generator.csproj", "{838687B6-9915-4BD2-98CC-79A4130B89B0}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.FluentRlp.Generator", "Nethermind.Serialization.FluentRlp.Generator\Nethermind.Serialization.FluentRlp.Generator.csproj", "{838687B6-9915-4BD2-98CC-79A4130B89B0}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution From 2456c4070f3932e8e584004f28b180772e0c0aaa Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 10:42:35 -0300 Subject: [PATCH 071/106] Remove allocations in Dictionary --- .../Instances/DictionaryRlpConverter.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/DictionaryRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/DictionaryRlpConverter.cs index 356a1970702..6e42de2dab3 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/DictionaryRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/DictionaryRlpConverter.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Collections.Generic; namespace Nethermind.Serialization.FluentRlp.Instances; @@ -31,12 +32,16 @@ public static Dictionary Read(ref RlpReader reader, RefRlpReaderFu public static void Write(ref RlpWriter writer, Dictionary value, RefRlpWriterAction writeKey, RefRlpWriterAction writeValue) { - writer.WriteSequence((ref RlpWriter w) => + var ctx = ValueTuple.Create(value, writeKey, writeValue); + writer.WriteSequence(ctx, static (ref RlpWriter w, (Dictionary, RefRlpWriterAction, RefRlpWriterAction) ctx) => { - foreach ((TKey k, TValue v) in value) + var (dictionary, writeKey, writeValue) = ctx; + foreach (var kp in dictionary) { - w.WriteSequence((ref RlpWriter w) => + var innerCtx = ValueTuple.Create(kp, writeKey, writeValue); + w.WriteSequence(innerCtx, static (ref RlpWriter w, (KeyValuePair, RefRlpWriterAction, RefRlpWriterAction) ctx) => { + var ((k, v), writeKey, writeValue) = ctx; writeKey(ref w, k); writeValue(ref w, v); }); From 868d272260b6583a2786d943a5df42e5719229d8 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 11:20:39 -0300 Subject: [PATCH 072/106] Remove unused method --- src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs | 6 ------ .../Nethermind.Serialization.FluentRlp/RlpReader.cs | 2 -- 2 files changed, 8 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs index a4dfad2a1d7..bfc6f3a153a 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs @@ -30,10 +30,4 @@ public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) // TODO: We might want to add an option to check for no trailing bytes. return result; } - - public static void Read(ReadOnlySpan source, RefRlpReaderAction func) - { - var reader = new RlpReader(source); - func(ref reader); - } } diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs index 854014b9728..7268ac4e67e 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs @@ -9,8 +9,6 @@ namespace Nethermind.Serialization.FluentRlp; public delegate TResult RefRlpReaderFunc(scoped ref RlpReader arg) where TResult : allows ref struct; -public delegate void RefRlpReaderAction(ref RlpReader arg); - public class RlpReaderException(string message) : Exception(message); public ref struct RlpReader From 69c8a92a5df84edf518d72a018757cc410a105d2 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 11:24:05 -0300 Subject: [PATCH 073/106] Introduce "context" overload for `Reader` --- .../RlpReader.cs | 14 ++++++++++---- .../RlpWriter.cs | 3 ++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs index 7268ac4e67e..95282701b8e 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs @@ -7,7 +7,11 @@ namespace Nethermind.Serialization.FluentRlp; -public delegate TResult RefRlpReaderFunc(scoped ref RlpReader arg) where TResult : allows ref struct; +public delegate TResult RefRlpReaderFunc(scoped ref RlpReader arg, TContext value) + where TResult : allows ref struct; + +public delegate TResult RefRlpReaderFunc(scoped ref RlpReader arg) + where TResult : allows ref struct; public class RlpReaderException(string message) : Exception(message); @@ -71,7 +75,9 @@ public ReadOnlySpan ReadBytes() } public T ReadSequence(RefRlpReaderFunc func) - { + => ReadSequence(func, static (scoped ref RlpReader reader, RefRlpReaderFunc func) => func(ref reader)); + + public T ReadSequence(TContext ctx, RefRlpReaderFunc func) { T result; var header = _buffer[_position++]; if (header < 0xC0) @@ -83,7 +89,7 @@ public T ReadSequence(RefRlpReaderFunc func) { var length = header - 0xC0; var reader = new RlpReader(_buffer.Slice(_position, length)); - result = func(ref reader); + result = func(ref reader, ctx); _position += length; } else @@ -93,7 +99,7 @@ public T ReadSequence(RefRlpReaderFunc func) _position += lengthOfLength; int length = Int32Primitive.Read(binaryLength); var reader = new RlpReader(_buffer.Slice(_position, length)); - result = func(ref reader); + result = func(ref reader, ctx); _position += length; } diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs index 0014b85d19f..71b2f0f97b1 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs @@ -8,7 +8,8 @@ namespace Nethermind.Serialization.FluentRlp; -public delegate void RefRlpWriterAction(ref RlpWriter arg, T value) where T: allows ref struct; +public delegate void RefRlpWriterAction(ref RlpWriter arg, TContext value) + where TContext: allows ref struct; public delegate void RefRlpWriterAction(ref RlpWriter arg); From f0191d1d5b50ca1d519b61c9dbda5dc64b17b35a Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 11:24:15 -0300 Subject: [PATCH 074/106] Remove allocations in `Dictionary` --- .../Instances/DictionaryRlpConverter.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/DictionaryRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/DictionaryRlpConverter.cs index 6e42de2dab3..d07b6892c09 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/DictionaryRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/DictionaryRlpConverter.cs @@ -10,13 +10,16 @@ public abstract class DictionaryRlpConverter where TKey : notnull { public static Dictionary Read(ref RlpReader reader, RefRlpReaderFunc readKey, RefRlpReaderFunc readValue) { - return reader.ReadSequence((scoped ref RlpReader r) => + var ctx = ValueTuple.Create(readKey, readValue); + return reader.ReadSequence(ctx, static (scoped ref RlpReader r, (RefRlpReaderFunc, RefRlpReaderFunc) ctx) => { Dictionary result = []; while (r.HasNext) { - (TKey key, TValue value) = r.ReadSequence((scoped ref RlpReader r) => + + (TKey key, TValue value) = r.ReadSequence(ctx, static (scoped ref RlpReader r, (RefRlpReaderFunc, RefRlpReaderFunc) ctx) => { + var (readKey, readValue) = ctx; TKey key = readKey(ref r); TValue value = readValue(ref r); From 1011df9c1e8668dbd7982491b50ace9c057c71f2 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 11:28:30 -0300 Subject: [PATCH 075/106] Remove allocations in `List` --- .../Instances/ListRlpConverter.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/ListRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/ListRlpConverter.cs index e72dd9b99de..5e6db5b22be 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/ListRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/ListRlpConverter.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Collections.Generic; namespace Nethermind.Serialization.FluentRlp.Instances; @@ -9,7 +10,7 @@ public abstract class ListRlpConverter { public static List Read(ref RlpReader reader, RefRlpReaderFunc func) { - return reader.ReadSequence((scoped ref RlpReader r) => + return reader.ReadSequence(func, static (scoped ref RlpReader r, RefRlpReaderFunc func) => { List result = []; while (r.HasNext) @@ -23,8 +24,10 @@ public static List Read(ref RlpReader reader, RefRlpReaderFunc func) public static void Write(ref RlpWriter writer, List value, RefRlpWriterAction action) { - writer.WriteSequence((ref RlpWriter w) => + var ctx = ValueTuple.Create(value, action); + writer.WriteSequence(ctx, static (ref RlpWriter w, (List, RefRlpWriterAction) ctx) => { + var (value, action) = ctx; foreach (T v in value) { action(ref w, v); From 63c3ed9701f786196b2f74dfb1f60c85416269d1 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 11:47:01 -0300 Subject: [PATCH 076/106] Use field Encoding --- .../Instances/StringRlpConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/StringRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/StringRlpConverter.cs index b62ff1943c3..dcc1158e6e3 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/StringRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/StringRlpConverter.cs @@ -15,7 +15,7 @@ public abstract class StringRlpConverter : IRlpConverter public static string Read(ref RlpReader reader) { ReadOnlySpan obj = reader.ReadBytes(); - return Encoding.UTF8.GetString(obj); + return Encoding.GetString(obj); } public static void Write(ref RlpWriter writer, string value) From 0c2807b246f83ecde78d9083504cb3551e797036 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 12:01:46 -0300 Subject: [PATCH 077/106] Add `ArrayRlpConverter` --- .../RlpSourceGenerator.cs | 21 +++++++++ .../RlpDerivedTest.cs | 13 +++++ .../Instances/ArrayRlpConverter.cs | 47 +++++++++++++++++++ .../README.md | 1 - 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/ArrayRlpConverter.cs diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index f45fffb49da..b24c80f4653 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -188,6 +188,13 @@ private static string MapTypeToReadCall(TypeSyntax syntax) return "ReadBytes()"; } + // Arrays + if (syntax is ArrayTypeSyntax array) + { + var elementType = array.ElementType; + return $"ReadArray(static (scoped ref RlpReader r) => r.{MapTypeToReadCall(elementType)})"; + } + // Generics if (syntax is GenericNameSyntax generic) { @@ -222,6 +229,20 @@ private static string MapTypeToReadCall(TypeSyntax syntax) /// private static string MapTypeToWriteCall(string name, TypeSyntax syntax) { + // Arrays + if (syntax is ArrayTypeSyntax array) + { + var elementType = array.ElementType; + + switch (elementType.ToString()) + { + case "byte" or "Byte" or "System.Byte": + return $"Write(value.{name})"; + default: + return $"Write(value.{name}, static (ref RlpWriter w, {elementType.ToString()} value) => w.Write(value))"; + } + } + // Generics if (syntax is GenericNameSyntax generic) { diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs index c2a3acd9c2b..c3d55da3c59 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs @@ -15,6 +15,9 @@ public record PlayerWithFriends(int Id, string Username, List Friends); [RlpSerializable] public record PlayerWithScores(int Id, string Username, Dictionary Scores); +[RlpSerializable] +public record PlayerWithCodes(int Id, string Username, int[] Codes); + [RlpSerializable] public record Tree(string Value, List Children); @@ -46,6 +49,16 @@ public void RecordWithList() decoded.Should().BeEquivalentTo(player); } + [Test] + public void RecordWithArray() + { + var player = new PlayerWithCodes(Id: 42, Username: "SuperUser", Codes: [2, 4, 8, 16, 32, 64]); + ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithCodes player) => w.Write(player)); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayerWithCodes()); + decoded.Should().BeEquivalentTo(player); + } + [Test] public void RecordWithDictionary() { diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/ArrayRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/ArrayRlpConverter.cs new file mode 100644 index 00000000000..eaa0ed50236 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/ArrayRlpConverter.cs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; + +namespace Nethermind.Serialization.FluentRlp.Instances; + +public abstract class ArrayRlpConverter +{ + public static T[] Read(ref RlpReader reader, RefRlpReaderFunc func) + { + return reader.ReadSequence(func, static (scoped ref RlpReader r, RefRlpReaderFunc func) => + { + List result = []; + while (r.HasNext) + { + result.Add(func(ref r)); + } + + // TODO: Avoid copying + return result.ToArray(); + }); + } + + public static void Write(ref RlpWriter writer, T[] value, RefRlpWriterAction action) + { + var ctx = ValueTuple.Create(value, action); + writer.WriteSequence(ctx, static (ref RlpWriter w, (T[], RefRlpWriterAction) ctx) => + { + var (value, action) = ctx; + foreach (T v in value) + { + action(ref w, v); + } + }); + } +} + +public static class ArrayRlpConverterExt +{ + public static T[] ReadArray(this ref RlpReader reader, RefRlpReaderFunc func) + => ArrayRlpConverter.Read(ref reader, func); + + public static void Write(this ref RlpWriter writer, T[] value, RefRlpWriterAction action) + => ArrayRlpConverter.Write(ref writer, value, action); +} diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/README.md b/src/Nethermind/Nethermind.Serialization.FluentRlp/README.md index 42894725a01..2a5a75aff0f 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/README.md +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/README.md @@ -5,6 +5,5 @@ Declarative RLP encoding a decoding with support for extensibility through manua ## TODO - Add support more instances for base types -- Add support for generic Arrays (`X[]`) - Alternative API for writing based on `Async` and `Stream` - Support for parameterizable names when using attributes From efc25bddfcec9fa384725f3609640d24a52e6621 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 15:23:24 -0300 Subject: [PATCH 078/106] Remove allocations --- .../Nethermind.Serialization.FluentRlp.Test/Student.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Student.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Student.cs index 07d08706e11..a8e2b76f1fb 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Student.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Student.cs @@ -39,11 +39,11 @@ public static Student Read(ref RlpReader reader) public static void Write(ref RlpWriter writer, Student value) { - writer.WriteSequence((ref RlpWriter w) => + writer.WriteSequence(value, static (ref RlpWriter w, Student value) => { w.Write(value.Name); w.Write(value.Age); - w.WriteSequence((ref RlpWriter w) => + w.WriteSequence(value, (ref RlpWriter w, Student value) => { foreach (var (subject, score) in value.Scores) { From aae0100b523b6e22d6b37571e00b6641bbe53dd1 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 15:36:21 -0300 Subject: [PATCH 079/106] Add `TupleRlpConverter` - Only 2-element tuples for now --- .../RlpReadWriteTest.cs | 22 +++++++++++ .../Instances/TupleRlpConverter.cs | 37 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/TupleRlpConverter.cs diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs index cb728d08480..fa9e493fb98 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs @@ -395,4 +395,26 @@ public void DictionaryCollection() decoded.Should().BeEquivalentTo(dictionary); } + + [Test] + public void TupleCollection() + { + var tuple = (42, 1337); + + var rlp = Rlp.Write(tuple, static (ref RlpWriter w, (int, int) tuple) + => w.Write(tuple, Int32RlpConverter.Write, Int32RlpConverter.Write)); + + var rlpExplicit = Rlp.Write(tuple, static (ref RlpWriter w, (int, int) tuple) => + { + w.Write(tuple.Item1); + w.Write(tuple.Item2); + }); + + rlp.Should().BeEquivalentTo(rlpExplicit); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => + r.ReadTuple(Int32RlpConverter.Read, Int32RlpConverter.Read)); + + decoded.Should().BeEquivalentTo(tuple); + } } diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/TupleRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/TupleRlpConverter.cs new file mode 100644 index 00000000000..e0a20154092 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/TupleRlpConverter.cs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Serialization.FluentRlp.Instances; + +public abstract class TupleRlpConverter +{ + public static (T1, T2) Read(ref RlpReader reader, RefRlpReaderFunc read1, RefRlpReaderFunc read2) + { + T1 _1 = read1(ref reader); + T2 _2 = read2(ref reader); + + return (_1, _2); + } + + public static void Write(ref RlpWriter writer, (T1, T2) value, RefRlpWriterAction write1, RefRlpWriterAction write2) + { + write1(ref writer, value.Item1); + write2(ref writer, value.Item2); + } +} + +public static class TupleRlpConverterExt +{ + public static (T1, T2) ReadTuple( + this ref RlpReader reader, + RefRlpReaderFunc read1, + RefRlpReaderFunc read2 + ) => TupleRlpConverter.Read(ref reader, read1, read2); + + public static void Write( + this ref RlpWriter writer, + (T1, T2) value, + RefRlpWriterAction write1, + RefRlpWriterAction write2 + ) => TupleRlpConverter.Write(ref writer, value, write1, write2); +} From c7a32d1fc6e29a8d62fae453a36f29b38fc3f29f Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 15:46:44 -0300 Subject: [PATCH 080/106] Support `Tuple` in source generators --- .../RlpSourceGenerator.cs | 31 ++++++++++++++----- .../RlpDerivedTest.cs | 13 ++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index b24c80f4653..c75b666050a 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -196,10 +196,21 @@ private static string MapTypeToReadCall(TypeSyntax syntax) } // Generics - if (syntax is GenericNameSyntax generic) + if (syntax is GenericNameSyntax or TupleTypeSyntax) { - var typeConstructor = generic.Identifier.ToString(); - var typeParameters = generic.TypeArgumentList.Arguments; + var typeConstructor = syntax switch + { + GenericNameSyntax generic => generic.Identifier.ToString(), + TupleTypeSyntax _ => "Tuple", + _ => throw new ArgumentOutOfRangeException(nameof(syntax)) + }; + + var typeParameters = syntax switch + { + GenericNameSyntax generic => generic.TypeArgumentList.Arguments, + TupleTypeSyntax tuple => tuple.Elements.Select(e => e.Type), + _ => throw new ArgumentOutOfRangeException(nameof(syntax)) + }; var sb = new StringBuilder("Read"); sb.Append(typeConstructor.Capitalize()); @@ -239,14 +250,20 @@ private static string MapTypeToWriteCall(string name, TypeSyntax syntax) case "byte" or "Byte" or "System.Byte": return $"Write(value.{name})"; default: - return $"Write(value.{name}, static (ref RlpWriter w, {elementType.ToString()} value) => w.Write(value))"; + return + $"Write(value.{name}, static (ref RlpWriter w, {elementType.ToString()} value) => w.Write(value))"; } } - // Generics - if (syntax is GenericNameSyntax generic) + // Generics and Tuples + if (syntax is GenericNameSyntax or TupleTypeSyntax) { - var typeParameters = generic.TypeArgumentList.Arguments; + var typeParameters = syntax switch + { + GenericNameSyntax generic => generic.TypeArgumentList.Arguments, + TupleTypeSyntax tuple => tuple.Elements.Select(e => e.Type), + _ => throw new ArgumentOutOfRangeException(nameof(syntax)) + }; var sb = new StringBuilder("Write"); sb.AppendLine("("); diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs index c3d55da3c59..d9768b9e9a2 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs @@ -27,6 +27,9 @@ public record RawData(int Tag, byte[] Data); [RlpSerializable] public record Integers(short A, int B, long C, Int128 D); +[RlpSerializable] +public record IntegerTuple((int, long) Values); + public class RlpDerivedTest { [Test] @@ -73,6 +76,16 @@ public void RecordWithDictionary() decoded.Should().BeEquivalentTo(player); } + [Test] + public void RecordWithTuple() + { + var integerTuple = new IntegerTuple((42, 1337)); + ReadOnlySpan rlp = Rlp.Write(integerTuple, static (ref RlpWriter w, IntegerTuple tuple) => w.Write(tuple)); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadIntegerTuple()); + decoded.Should().BeEquivalentTo(integerTuple); + } + [Test] public void RecursiveRecord() { From 3dc5b6592d35046048f36d06bf42bb0d113304d0 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 16:38:22 -0300 Subject: [PATCH 081/106] Add support for `Representation` --- .../RlpSourceGenerator.cs | 78 +++++++++++++++---- .../RlpDerivedTest.cs | 22 ++++++ 2 files changed, 84 insertions(+), 16 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index c75b666050a..4a438710d1a 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -12,8 +12,24 @@ namespace Nethermind.Serialization.FluentRlp.Generator; +public enum RlpRepresentation : byte +{ + /// + /// The RLP encoding will be a sequence of RLP objects for each property. + /// + Record = 0, + + /// + /// The RLP encoding will be equivalent to the only underlying property. + /// + Newtype = 1, +} + [AttributeUsage(AttributeTargets.Class)] -public sealed class RlpSerializable : Attribute; +public sealed class RlpSerializable(RlpRepresentation representation = RlpRepresentation.Record) : Attribute +{ + public RlpRepresentation Representation { get; } = representation; +} /// /// A source generator that finds all records with [RlpSerializable] and @@ -49,14 +65,14 @@ private void Execute( ISymbol? symbol = semanticModel.GetDeclaredSymbol(recordDecl); if (symbol is null) continue; - // If not, skip the record - var attributes = symbol.GetAttributes(); - if (!attributes.Any(a => + AttributeData? rlpSerializableAttribute = symbol + .GetAttributes() + .FirstOrDefault(a => a.AttributeClass?.Name == nameof(RlpSerializable) || - a.AttributeClass?.ToDisplayString() == nameof(RlpSerializable))) - { - continue; - } + a.AttributeClass?.ToDisplayString() == nameof(RlpSerializable)); + + // If not, skip the record + if (rlpSerializableAttribute is null) continue; // Extract the fully qualified record name with its namespace var recordName = symbol.Name; @@ -64,12 +80,27 @@ private void Execute( // TODO: Deal with missing and nested namespaces var @namespace = symbol.ContainingNamespace?.ToDisplayString(); + var representation = (RlpRepresentation)(rlpSerializableAttribute.ConstructorArguments[0].Value ?? 0); + // Gather recursively all members that are fields or primary constructor parameters // so we can read them in the same order they are declared. var parameters = GetRecordParameters(recordDecl); + // Ensure `Newtype` is only used in single-property records + if (representation == RlpRepresentation.Newtype && parameters.Count != 1) + { + var descriptor = new DiagnosticDescriptor( + "RLP0001", + $"Invalid {nameof(RlpRepresentation)}", + $"'{nameof(RlpRepresentation.Newtype)}' representation is only allowed for records with a single property", + "", DiagnosticSeverity.Error, true); + context.ReportDiagnostic(Diagnostic.Create(descriptor: descriptor, recordDecl.GetLocation())); + + return; + } + // Build the converter class source - var generatedCode = GenerateConverterClass(@namespace, fullTypeName, recordName, parameters); + var generatedCode = GenerateConverterClass(@namespace, fullTypeName, recordName, parameters, representation); // Add to the compilation context.AddSource($"{recordName}RlpConverter.g.cs", SourceText.From(generatedCode, Encoding.UTF8)); @@ -103,7 +134,8 @@ private static string GenerateConverterClass( string? @namespace, string fullTypeName, string recordName, - List<(string Name, TypeSyntax TypeName)> parameters) + List<(string Name, TypeSyntax TypeName)> parameters, + RlpRepresentation representation) { var sb = new StringBuilder(); @@ -123,8 +155,11 @@ private static string GenerateConverterClass( // `Write` method sb.AppendLine($"public static void Write(ref RlpWriter w, {fullTypeName} value)"); sb.AppendLine("{"); - sb.AppendLine($"w.WriteSequence(value, static (ref RlpWriter w, {fullTypeName} value) => "); - sb.AppendLine("{"); + if (representation == RlpRepresentation.Record) + { + sb.AppendLine($"w.WriteSequence(value, static (ref RlpWriter w, {fullTypeName} value) => "); + sb.AppendLine("{"); + } foreach (var (name, typeName) in parameters) { @@ -132,14 +167,21 @@ private static string GenerateConverterClass( sb.AppendLine($"w.{writeCall};"); } - sb.AppendLine("});"); + if (representation == RlpRepresentation.Record) + { + sb.AppendLine("});"); + } + sb.AppendLine("}"); // `Read` method sb.AppendLine($"public static {fullTypeName} Read(ref RlpReader r)"); sb.AppendLine("{"); - sb.AppendLine("return r.ReadSequence(static (scoped ref RlpReader r) =>"); - sb.AppendLine("{"); + if (representation == RlpRepresentation.Record) + { + sb.AppendLine("return r.ReadSequence(static (scoped ref RlpReader r) =>"); + sb.AppendLine("{"); + } foreach (var (name, typeName) in parameters) { @@ -156,7 +198,11 @@ private static string GenerateConverterClass( sb.AppendLine(");"); - sb.AppendLine("});"); + if (representation == RlpRepresentation.Record) + { + sb.AppendLine("});"); + } + sb.AppendLine("}"); sb.AppendLine("}"); diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs index d9768b9e9a2..617e71daed3 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs @@ -3,6 +3,7 @@ using FluentAssertions; using Nethermind.Serialization.FluentRlp.Generator; +using Nethermind.Serialization.FluentRlp.Instances; namespace Nethermind.Serialization.FluentRlp.Test; @@ -30,6 +31,9 @@ public record Integers(short A, int B, long C, Int128 D); [RlpSerializable] public record IntegerTuple((int, long) Values); +[RlpSerializable(RlpRepresentation.Newtype)] +public record Address(string HexString); + public class RlpDerivedTest { [Test] @@ -101,4 +105,22 @@ [new Tree("cat", [])]) var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadTree()); decoded.Should().BeEquivalentTo(tree); } + + [Test] + public void NewtypeRecords() + { + var address = new Address("0x1234567890ABCDEF"); + + var rlp = Rlp.Write(address, static (ref RlpWriter writer, Address address) + => writer.Write(address)); + + var rlpExplicit = Rlp.Write(address, (ref RlpWriter writer, Address value) + => writer.Write(value.HexString)); + + rlp.Should().BeEquivalentTo(rlpExplicit); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadAddress()); + + decoded.Should().BeEquivalentTo(address); + } } From fd9fc30603b39f92d87eabf0f937c7ec895183d7 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 26 Dec 2024 17:01:54 -0300 Subject: [PATCH 082/106] Formatting --- .../RlpSourceGenerator.cs | 2 +- .../RlpReadWriteTest.cs | 4 ++-- .../RlpReaderTest.cs | 4 ++-- .../RlpWriterTest.cs | 4 ++-- .../Nethermind.Serialization.FluentRlp/RlpReader.cs | 3 ++- .../Nethermind.Serialization.FluentRlp/RlpWriter.cs | 8 +++++--- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index 4a438710d1a..d014afa6cc9 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System; diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs index fa9e493fb98..1bb57f1d2ec 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs @@ -205,7 +205,7 @@ public void Choice() var intRlp = Rlp.Write(static (ref RlpWriter w) => { w.Write(42); }); var wrappedIntRlp = Rlp.Write(static (ref RlpWriter w) => w.WriteSequence(static (ref RlpWriter w) => { w.Write(42); })); - foreach (var rlp in (byte[][]) [intRlp, wrappedIntRlp]) + foreach (var rlp in (byte[][])[intRlp, wrappedIntRlp]) { int decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.Choice(wrappedReader, intReader)); @@ -376,7 +376,7 @@ public void DictionaryCollection() var rlpExplicit = Rlp.Write(dictionary, static (ref RlpWriter w, Dictionary dictionary) => { - w.WriteSequence(dictionary, static (ref RlpWriter w,Dictionary dictionary) => + w.WriteSequence(dictionary, static (ref RlpWriter w, Dictionary dictionary) => { foreach (var tuple in dictionary) { diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReaderTest.cs index 713ac40695f..1b21dab93a2 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReaderTest.cs @@ -29,7 +29,7 @@ public void ReadEmptyString() [Test] public void ReadLongString() { - byte[] source = [0xb8, 0x38, .."Lorem ipsum dolor sit amet, consectetur adipisicing elit"u8]; + byte[] source = [0xb8, 0x38, .. "Lorem ipsum dolor sit amet, consectetur adipisicing elit"u8]; string actual = Rlp.Read(source, static (scoped ref RlpReader r) => r.ReadString()); actual.Should().Be("Lorem ipsum dolor sit amet, consectetur adipisicing elit"); @@ -64,7 +64,7 @@ public void ReadLongInteger() [Test] public void ReadStringList() { - byte[] source = [0xc8, 0x83, .."cat"u8, 0x83, .."dog"u8]; + byte[] source = [0xc8, 0x83, .. "cat"u8, 0x83, .. "dog"u8]; var actual = Rlp.Read(source, static (scoped ref RlpReader r) => { return r.ReadSequence(static (scoped ref RlpReader r) => diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpWriterTest.cs index 85bd3c84b9f..0a26e2c8e61 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpWriterTest.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using FluentAssertions; @@ -35,7 +35,7 @@ public void WriteLongString() w.Write("Lorem ipsum dolor sit amet, consectetur adipisicing elit"); }); - byte[] expected = [0xb8, 0x38, .."Lorem ipsum dolor sit amet, consectetur adipisicing elit"u8]; + byte[] expected = [0xb8, 0x38, .. "Lorem ipsum dolor sit amet, consectetur adipisicing elit"u8]; serialized.Should().BeEquivalentTo(expected); } diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs index 95282701b8e..2242e567c79 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs @@ -77,7 +77,8 @@ public ReadOnlySpan ReadBytes() public T ReadSequence(RefRlpReaderFunc func) => ReadSequence(func, static (scoped ref RlpReader reader, RefRlpReaderFunc func) => func(ref reader)); - public T ReadSequence(TContext ctx, RefRlpReaderFunc func) { + public T ReadSequence(TContext ctx, RefRlpReaderFunc func) + { T result; var header = _buffer[_position++]; if (header < 0xC0) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs index 71b2f0f97b1..2d00680488b 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs @@ -9,7 +9,7 @@ namespace Nethermind.Serialization.FluentRlp; public delegate void RefRlpWriterAction(ref RlpWriter arg, TContext value) - where TContext: allows ref struct; + where TContext : allows ref struct; public delegate void RefRlpWriterAction(ref RlpWriter arg); @@ -65,7 +65,8 @@ private void LengthWrite(T value) where T : IBinaryInteger, ISignedNumber< if (bigEndian.Length == 0) { Length++; - } else if (bigEndian.Length == 1 && bigEndian[0] < 0x80) + } + else if (bigEndian.Length == 1 && bigEndian[0] < 0x80) { Length++; } @@ -85,7 +86,8 @@ private void ContentWrite(T value) where T : IBinaryInteger, ISignedNumber if (bigEndian.Length == 0) { _buffer[_position++] = 0; - } else if (bigEndian.Length == 1 && bigEndian[0] < 0x80) + } + else if (bigEndian.Length == 1 && bigEndian[0] < 0x80) { _buffer[_position++] = bigEndian[0]; } From 8341a0e956b0c7995a2e50f3c1b8e4f9a5bf4310 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 27 Dec 2024 11:26:11 -0300 Subject: [PATCH 083/106] Update docs --- .../RlpSourceGenerator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index d014afa6cc9..f10eff65902 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -32,8 +32,8 @@ public sealed class RlpSerializable(RlpRepresentation representation = RlpRepres } /// -/// A source generator that finds all records with [RlpSerializable] and -/// generates an abstract RlpConverter class with a Read method. +/// A source generator that finds all records with [RlpSerializable] attribute and +/// generates an abstract `IRlpConverter` class with `Read` and `Write` methods. /// [Generator] public sealed class RlpSourceGenerator : IIncrementalGenerator From 3e1330d529fc24a28c52c717bbbd6c3159a1c8f4 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 30 Dec 2024 12:11:53 -0300 Subject: [PATCH 084/106] Throw on trailing bytes --- src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs index bfc6f3a153a..3cfe43eb5ff 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs @@ -27,7 +27,7 @@ public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) { var reader = new RlpReader(source); T result = func(ref reader); - // TODO: We might want to add an option to check for no trailing bytes. + if (reader.HasNext) throw new RlpReaderException("RLP has trailing bytes"); return result; } } From 4f68cb303ae4c8b3d05dd99fa46853ee6af85f9e Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 30 Dec 2024 12:41:47 -0300 Subject: [PATCH 085/106] Reduce usage of `StringBuilder` --- .../RlpSourceGenerator.cs | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index f10eff65902..08a3d10a082 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -258,19 +258,13 @@ private static string MapTypeToReadCall(TypeSyntax syntax) _ => throw new ArgumentOutOfRangeException(nameof(syntax)) }; - var sb = new StringBuilder("Read"); - sb.Append(typeConstructor.Capitalize()); - sb.AppendLine("("); - + var sb = new StringBuilder(); + sb.AppendLine($"Read{typeConstructor.Capitalize()}("); foreach (var typeParameter in typeParameters) { - sb.Append("static (scoped ref RlpReader r) =>"); - sb.Append("{"); - sb.Append($"return r.{MapTypeToReadCall(typeParameter)};"); - sb.Append("},"); + sb.AppendLine($$"""static (scoped ref RlpReader r) => { return r.{{MapTypeToReadCall(typeParameter)}}; },"""); } - - sb.Length -= 1; // Remove the trailing `,` + sb.Length -= 2; // Remove the trailing `,\n` sb.Append(")"); return sb.ToString(); @@ -297,7 +291,7 @@ private static string MapTypeToWriteCall(string name, TypeSyntax syntax) return $"Write(value.{name})"; default: return - $"Write(value.{name}, static (ref RlpWriter w, {elementType.ToString()} value) => w.Write(value))"; + $"Write(value.{name}, static (ref RlpWriter w, {elementType} value) => w.Write(value))"; } } @@ -311,19 +305,13 @@ private static string MapTypeToWriteCall(string name, TypeSyntax syntax) _ => throw new ArgumentOutOfRangeException(nameof(syntax)) }; - var sb = new StringBuilder("Write"); - sb.AppendLine("("); - - sb.AppendLine($"value.{name},"); + var sb = new StringBuilder(); + sb.AppendLine($"Write(value.{name},"); foreach (var typeParameter in typeParameters) { - sb.Append($"static (ref RlpWriter w, {typeParameter.ToString()} value) =>"); - sb.Append("{"); - sb.Append("w.Write(value);"); - sb.Append("},"); + sb.AppendLine($$"""static (ref RlpWriter w, {{typeParameter}} value) => { w.Write(value); },"""); } - - sb.Length -= 1; // Remove the trailing `,` + sb.Length -= 2; // Remove the trailing `,\n` sb.Append(")"); return sb.ToString(); @@ -361,4 +349,30 @@ public static IEnumerable Intersperse(this IEnumerable source, T elemen first = false; } } + +// public static string Foo = +// $$""" +// // +// #nullable enable +// +// using System; +// using System.CodeDom.Compiler; +// using Nethermind.Serialization.FluentRlp; +// using Nethermind.Serialization.FluentRlp.Instances; +// +// {{@namespace ?? ""}} +// +// {{GeneratedCodeAttribute}} +// public abstract class {{recordName}}RlpConverter : IRlpConverter<{{fullTypeName}}> +// { +// public static void Write(ref RlpWriter w, {{fullTypeName}} value) +// { +// w.WriteSequence(value, static (ref RlpWriter w, {{fullTypeName}} value) => +// { +// +// +// } +// } +// } +// """; } From ac98b21f3ed8d93dc6478fca428af22c2e5f1cfb Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 30 Dec 2024 12:42:23 -0300 Subject: [PATCH 086/106] Remove unused code --- .../RlpSourceGenerator.cs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index 08a3d10a082..1c24b5f2c8e 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -337,21 +337,6 @@ public static class StringExt public static string Capitalize(this string str) => str[0].ToString().ToUpper() + str[1..]; } -public static class EnumerableExt -{ - public static IEnumerable Intersperse(this IEnumerable source, T element) - { - bool first = true; - foreach (T value in source) - { - if (!first) yield return element; - yield return value; - first = false; - } - } - -// public static string Foo = -// $$""" // // // #nullable enable // @@ -375,4 +360,3 @@ public static IEnumerable Intersperse(this IEnumerable source, T elemen // } // } // """; -} From 79dc13fc74fdf46df52479fdc7d8f1eb165c237b Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 30 Dec 2024 13:17:42 -0300 Subject: [PATCH 087/106] Use format string --- .../RlpSourceGenerator.cs | 158 ++++++++---------- 1 file changed, 66 insertions(+), 92 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index 1c24b5f2c8e..90cbe81be64 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -39,7 +39,7 @@ public sealed class RlpSerializable(RlpRepresentation representation = RlpRepres public sealed class RlpSourceGenerator : IIncrementalGenerator { private const string Version = "0.1"; - private const string GeneratedCodeAttribute = $"""[GeneratedCode("{nameof(RlpSourceGenerator)}", "{Version}")]"""; + public const string GeneratedCodeAttribute = $"""[GeneratedCode("{nameof(RlpSourceGenerator)}", "{Version}")]"""; public void Initialize(IncrementalGeneratorInitializationContext context) { @@ -137,86 +137,84 @@ private static string GenerateConverterClass( List<(string Name, TypeSyntax TypeName)> parameters, RlpRepresentation representation) { - var sb = new StringBuilder(); - - sb.AppendLine("// "); - sb.AppendLine("#nullable enable"); - sb.AppendLine("using System;"); - sb.AppendLine("using System.CodeDom.Compiler;"); - sb.AppendLine("using Nethermind.Serialization.FluentRlp;"); - sb.AppendLine("using Nethermind.Serialization.FluentRlp.Instances;"); - sb.AppendLine(); - if (@namespace is not null) sb.AppendLine($"namespace {@namespace};"); - sb.AppendLine(""); - sb.AppendLine(GeneratedCodeAttribute); - sb.AppendLine($"public abstract class {recordName}RlpConverter : IRlpConverter<{fullTypeName}>"); - sb.AppendLine("{"); - - // `Write` method - sb.AppendLine($"public static void Write(ref RlpWriter w, {fullTypeName} value)"); - sb.AppendLine("{"); - if (representation == RlpRepresentation.Record) - { - sb.AppendLine($"w.WriteSequence(value, static (ref RlpWriter w, {fullTypeName} value) => "); - sb.AppendLine("{"); - } - + var writeCalls = new StringBuilder(); foreach (var (name, typeName) in parameters) { var writeCall = MapTypeToWriteCall(name, typeName); - sb.AppendLine($"w.{writeCall};"); - } - - if (representation == RlpRepresentation.Record) - { - sb.AppendLine("});"); - } - - sb.AppendLine("}"); - - // `Read` method - sb.AppendLine($"public static {fullTypeName} Read(ref RlpReader r)"); - sb.AppendLine("{"); - if (representation == RlpRepresentation.Record) - { - sb.AppendLine("return r.ReadSequence(static (scoped ref RlpReader r) =>"); - sb.AppendLine("{"); + writeCalls.AppendLine($"w.{writeCall};"); } + var readCalls = new StringBuilder(); foreach (var (name, typeName) in parameters) { var readCall = MapTypeToReadCall(typeName); - sb.AppendLine($"var {name} = r.{readCall};"); + readCalls.AppendLine($"var {name} = r.{readCall};"); } - sb.Append($"return new {fullTypeName}("); + var constructorCall = new StringBuilder($"{fullTypeName}("); for (int i = 0; i < parameters.Count; i++) { - sb.Append(parameters[i].Name); - if (i < parameters.Count - 1) sb.Append(", "); - } - - sb.AppendLine(");"); - - if (representation == RlpRepresentation.Record) - { - sb.AppendLine("});"); + constructorCall.Append(parameters[i].Name); + if (i < parameters.Count - 1) constructorCall.Append(", "); } - - sb.AppendLine("}"); - sb.AppendLine("}"); - - sb.AppendLine(); - sb.AppendLine(GeneratedCodeAttribute); - sb.AppendLine($"public static class {recordName}Ext"); - sb.AppendLine("{"); - sb.AppendLine( - $"public static {fullTypeName} Read{recordName}(this ref RlpReader reader) => {recordName}RlpConverter.Read(ref reader);"); - sb.AppendLine( - $"public static void Write(this ref RlpWriter writer, {fullTypeName} value) => {recordName}RlpConverter.Write(ref writer, value);"); - sb.AppendLine("}"); - - return sb.ToString(); + constructorCall.Append(");"); + + return + $$""" + // + #nullable enable + using System; + using System.CodeDom.Compiler; + using Nethermind.Serialization.FluentRlp; + using Nethermind.Serialization.FluentRlp.Instances; + + {{(@namespace is null ? "" : $"namespace {@namespace};")}} + + {{GeneratedCodeAttribute}} + public abstract class {{recordName}}RlpConverter : IRlpConverter<{{fullTypeName}}> + { + public static void Write(ref RlpWriter w, {{fullTypeName}} value) + { + {{ + (representation == RlpRepresentation.Record + ? $$""" + w.WriteSequence(value, static (ref RlpWriter w, {{fullTypeName}} value) => + { + {{writeCalls}} + }); + """ + : writeCalls) + }} + } + + public static {{fullTypeName}} Read(ref RlpReader r) + { + {{ + (representation == RlpRepresentation.Record + ? $$""" + return r.ReadSequence(static (scoped ref RlpReader r) => + { + {{readCalls}} + + return new {{constructorCall}} + }); + """ + : $""" + {readCalls} + + return new {constructorCall} + """) + }} + } + } + + {{GeneratedCodeAttribute}} + public static class {{recordName}}Ext + { + public static {{fullTypeName}} Read{{recordName}}(this ref RlpReader reader) => {{recordName}}RlpConverter.Read(ref reader); + public static void Write(this ref RlpWriter writer, {{fullTypeName}} value) => {{recordName}}RlpConverter.Write(ref writer, value); + } + """; } /// @@ -336,27 +334,3 @@ public static class StringExt { public static string Capitalize(this string str) => str[0].ToString().ToUpper() + str[1..]; } - -// // -// #nullable enable -// -// using System; -// using System.CodeDom.Compiler; -// using Nethermind.Serialization.FluentRlp; -// using Nethermind.Serialization.FluentRlp.Instances; -// -// {{@namespace ?? ""}} -// -// {{GeneratedCodeAttribute}} -// public abstract class {{recordName}}RlpConverter : IRlpConverter<{{fullTypeName}}> -// { -// public static void Write(ref RlpWriter w, {{fullTypeName}} value) -// { -// w.WriteSequence(value, static (ref RlpWriter w, {{fullTypeName}} value) => -// { -// -// -// } -// } -// } -// """; From c9ba2eb236c0c5f3458882bbcc1dcde288335541 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 30 Dec 2024 13:23:29 -0300 Subject: [PATCH 088/106] Merge `Array` with generics --- .../RlpSourceGenerator.cs | 34 ++++++------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index 90cbe81be64..863a49989d8 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -39,7 +39,7 @@ public sealed class RlpSerializable(RlpRepresentation representation = RlpRepres public sealed class RlpSourceGenerator : IIncrementalGenerator { private const string Version = "0.1"; - public const string GeneratedCodeAttribute = $"""[GeneratedCode("{nameof(RlpSourceGenerator)}", "{Version}")]"""; + private const string GeneratedCodeAttribute = $"""[GeneratedCode("{nameof(RlpSourceGenerator)}", "{Version}")]"""; public void Initialize(IncrementalGeneratorInitializationContext context) { @@ -232,20 +232,14 @@ private static string MapTypeToReadCall(TypeSyntax syntax) return "ReadBytes()"; } - // Arrays - if (syntax is ArrayTypeSyntax array) - { - var elementType = array.ElementType; - return $"ReadArray(static (scoped ref RlpReader r) => r.{MapTypeToReadCall(elementType)})"; - } - // Generics - if (syntax is GenericNameSyntax or TupleTypeSyntax) + if (syntax is GenericNameSyntax or TupleTypeSyntax or ArrayTypeSyntax) { var typeConstructor = syntax switch { GenericNameSyntax generic => generic.Identifier.ToString(), TupleTypeSyntax _ => "Tuple", + ArrayTypeSyntax _ => "Array", _ => throw new ArgumentOutOfRangeException(nameof(syntax)) }; @@ -253,6 +247,7 @@ private static string MapTypeToReadCall(TypeSyntax syntax) { GenericNameSyntax generic => generic.TypeArgumentList.Arguments, TupleTypeSyntax tuple => tuple.Elements.Select(e => e.Type), + ArrayTypeSyntax array => [array.ElementType], _ => throw new ArgumentOutOfRangeException(nameof(syntax)) }; @@ -278,28 +273,21 @@ private static string MapTypeToReadCall(TypeSyntax syntax) /// private static string MapTypeToWriteCall(string name, TypeSyntax syntax) { - // Arrays - if (syntax is ArrayTypeSyntax array) + // Hard-coded cases + switch (syntax.ToString()) { - var elementType = array.ElementType; - - switch (elementType.ToString()) - { - case "byte" or "Byte" or "System.Byte": - return $"Write(value.{name})"; - default: - return - $"Write(value.{name}, static (ref RlpWriter w, {elementType} value) => w.Write(value))"; - } + case "byte[]" or "Byte[]" or "System.Byte[]" or "Span" or "System.Span" or "ReadOnlySpan" or "System.ReadOnlySpan": + return $"Write(value.{name})"; } - // Generics and Tuples - if (syntax is GenericNameSyntax or TupleTypeSyntax) + // Generics + if (syntax is GenericNameSyntax or TupleTypeSyntax or ArrayTypeSyntax) { var typeParameters = syntax switch { GenericNameSyntax generic => generic.TypeArgumentList.Arguments, TupleTypeSyntax tuple => tuple.Elements.Select(e => e.Type), + ArrayTypeSyntax array => [array.ElementType], _ => throw new ArgumentOutOfRangeException(nameof(syntax)) }; From db5408ad82323134d935c2b0c5874e1dff48d32b Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 30 Dec 2024 13:32:18 -0300 Subject: [PATCH 089/106] Formatting --- .../RlpSourceGenerator.cs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index 863a49989d8..495f5e91696 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -175,22 +175,19 @@ public abstract class {{recordName}}RlpConverter : IRlpConverter<{{fullTypeName} { public static void Write(ref RlpWriter w, {{fullTypeName}} value) { - {{ - (representation == RlpRepresentation.Record + {{(representation == RlpRepresentation.Record ? $$""" w.WriteSequence(value, static (ref RlpWriter w, {{fullTypeName}} value) => { {{writeCalls}} }); """ - : writeCalls) - }} + : writeCalls)}} } public static {{fullTypeName}} Read(ref RlpReader r) { - {{ - (representation == RlpRepresentation.Record + {{(representation == RlpRepresentation.Record ? $$""" return r.ReadSequence(static (scoped ref RlpReader r) => { @@ -203,8 +200,7 @@ public static void Write(ref RlpWriter w, {{fullTypeName}} value) {readCalls} return new {constructorCall} - """) - }} + """)}} } } From 31726b6deae885456e501d0a1e8d3d372f49d8b6 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 2 Jan 2025 10:35:17 -0300 Subject: [PATCH 090/106] Fix `0` --- .../RlpWriterTest.cs | 11 ++++++++++- .../Nethermind.Serialization.FluentRlp/RlpWriter.cs | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpWriterTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpWriterTest.cs index 0a26e2c8e61..f601981bc1e 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpWriterTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpWriterTest.cs @@ -39,10 +39,19 @@ public void WriteLongString() serialized.Should().BeEquivalentTo(expected); } + [Test] + public void WriteZero() + { + var serialized = Rlp.Write(static (ref RlpWriter w) => { w.Write(0); }); + + byte[] expected = [0x80]; + serialized.Should().BeEquivalentTo(expected); + } + [Test] public void WriteInteger_1Component() { - for (int i = 0; i < 0x80; i++) + for (int i = 1; i < 0x80; i++) { var integer = i; var serialized = Rlp.Write(integer, static (ref RlpWriter w, int integer) => { w.Write(integer); }); diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs index 2d00680488b..7d99aab6235 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs @@ -85,7 +85,7 @@ private void ContentWrite(T value) where T : IBinaryInteger, ISignedNumber if (bigEndian.Length == 0) { - _buffer[_position++] = 0; + _buffer[_position++] = 0x80; } else if (bigEndian.Length == 1 && bigEndian[0] < 0x80) { From 149145b064eb62f3ece6f4f5639ac04afbd01b6a Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 2 Jan 2025 10:44:00 -0300 Subject: [PATCH 091/106] Remove signed integer requirement --- .../Instances/IntegerRlpConverter.cs | 32 +++++++++++++++++++ .../RlpReader.cs | 2 +- .../RlpWriter.cs | 6 ++-- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/IntegerRlpConverter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/IntegerRlpConverter.cs index 75b004a07eb..ee67bc98074 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/IntegerRlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Instances/IntegerRlpConverter.cs @@ -33,6 +33,34 @@ public abstract class Int128RlpConverter : IRlpConverter public static void Write(ref RlpWriter writer, Int128 value) => writer.Write(value); } +public abstract class UInt16RlpConverter : IRlpConverter +{ + public static UInt16 Read(ref RlpReader reader) => reader.ReadInteger(); + + public static void Write(ref RlpWriter writer, UInt16 value) => writer.Write(value); +} + +public abstract class UInt32RlpConverter : IRlpConverter +{ + public static UInt32 Read(ref RlpReader reader) => reader.ReadInteger(); + + public static void Write(ref RlpWriter writer, UInt32 value) => writer.Write(value); +} + +public abstract class UInt64RlpConverter : IRlpConverter +{ + public static UInt64 Read(ref RlpReader reader) => reader.ReadInteger(); + + public static void Write(ref RlpWriter writer, UInt64 value) => writer.Write(value); +} + +public abstract class UInt128RlpConverter : IRlpConverter +{ + public static UInt128 Read(ref RlpReader reader) => reader.ReadInteger(); + + public static void Write(ref RlpWriter writer, UInt128 value) => writer.Write(value); +} + // NOTE: No need for `Write` overloads since they're covered by generic primitives // `Read` methods are provided for a consistent API (instead of using generics primitives) public static class IntegerRlpConverterExt @@ -41,4 +69,8 @@ public static class IntegerRlpConverterExt public static Int32 ReadInt32(this ref RlpReader reader) => reader.ReadInteger(); public static Int64 ReadInt64(this ref RlpReader reader) => reader.ReadInteger(); public static Int128 ReadInt128(this ref RlpReader reader) => reader.ReadInteger(); + public static UInt16 ReadUInt16(this ref RlpReader reader) => reader.ReadInteger(); + public static UInt64 UInt64(this ref RlpReader reader) => reader.ReadInteger(); + public static UInt64 ReadUInt64(this ref RlpReader reader) => reader.ReadInteger(); + public static UInt128 UReadInt128(this ref RlpReader reader) => reader.ReadInteger(); } diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs index 2242e567c79..0788a7814ce 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs @@ -28,7 +28,7 @@ public RlpReader(ReadOnlySpan buffer) public bool HasNext => _position < _buffer.Length; - public T ReadInteger() where T : IBinaryInteger, ISignedNumber + public T ReadInteger() where T : IBinaryInteger { ReadOnlySpan bigEndian; var header = _buffer[_position]; diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs index 7d99aab6235..626b057b20f 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs @@ -42,7 +42,7 @@ public static RlpWriter ContentWriter(byte[] buffer) }; } - public void Write(T value) where T : IBinaryInteger, ISignedNumber + public void Write(T value) where T : IBinaryInteger { switch (_mode) { @@ -55,7 +55,7 @@ public void Write(T value) where T : IBinaryInteger, ISignedNumber } } - private void LengthWrite(T value) where T : IBinaryInteger, ISignedNumber + private void LengthWrite(T value) where T : IBinaryInteger { var size = Marshal.SizeOf(); Span bigEndian = stackalloc byte[size]; @@ -76,7 +76,7 @@ private void LengthWrite(T value) where T : IBinaryInteger, ISignedNumber< } } - private void ContentWrite(T value) where T : IBinaryInteger, ISignedNumber + private void ContentWrite(T value) where T : IBinaryInteger { var size = Marshal.SizeOf(); Span bigEndian = stackalloc byte[size]; From dd57f803482ff2223465de8d40f64d942b180864 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 2 Jan 2025 11:28:38 -0300 Subject: [PATCH 092/106] Add `BytesRead` --- .../RlpReaderTest.cs | 12 ++++++++++++ .../Nethermind.Serialization.FluentRlp/RlpReader.cs | 1 + 2 files changed, 13 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReaderTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReaderTest.cs index 1b21dab93a2..b39ef0e895a 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReaderTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReaderTest.cs @@ -141,4 +141,16 @@ public void ReadSetTheoreticalRepresentation() new object[] { new object[] { }, new object[] { new object[] { } } }, }); } + + [Test] + public void ReadTrailingBytes() + { + byte[] source = [0x83, (byte)'d', (byte)'o', (byte)'g']; + + var reader = new RlpReader(source); + _ = reader.ReadString(); + + reader.HasNext.Should().BeFalse(); + reader.BytesRead.Should().Be(source.Length); + } } diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs index 0788a7814ce..f3af28e2bf1 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs @@ -27,6 +27,7 @@ public RlpReader(ReadOnlySpan buffer) } public bool HasNext => _position < _buffer.Length; + public int BytesRead => _position; public T ReadInteger() where T : IBinaryInteger { From 40cfd25ded81ca90b30c2e744d6e1c860d30b1d5 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 2 Jan 2025 13:27:46 -0300 Subject: [PATCH 093/106] Include user `using` statements --- .../RlpSourceGenerator.cs | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index 495f5e91696..6941ec2b28d 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -6,6 +6,7 @@ using System.Collections.Immutable; using System.Linq; using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.CodeAnalysis.Text; using System.Text; @@ -80,6 +81,13 @@ private void Execute( // TODO: Deal with missing and nested namespaces var @namespace = symbol.ContainingNamespace?.ToDisplayString(); + // Extract all `using` directives + var usingDirectives = semanticModel.SyntaxTree.GetCompilationUnitRoot() + .Usings + .Select(u => u.ToString()) + .ToList(); + + // Get the `RlpRepresentation` mode var representation = (RlpRepresentation)(rlpSerializableAttribute.ConstructorArguments[0].Value ?? 0); // Gather recursively all members that are fields or primary constructor parameters @@ -100,7 +108,7 @@ private void Execute( } // Build the converter class source - var generatedCode = GenerateConverterClass(@namespace, fullTypeName, recordName, parameters, representation); + var generatedCode = GenerateConverterClass(@namespace, usingDirectives, fullTypeName, recordName, parameters, representation); // Add to the compilation context.AddSource($"{recordName}RlpConverter.g.cs", SourceText.From(generatedCode, Encoding.UTF8)); @@ -132,11 +140,26 @@ private void Execute( private static string GenerateConverterClass( string? @namespace, + List usingDirectives, string fullTypeName, string recordName, List<(string Name, TypeSyntax TypeName)> parameters, RlpRepresentation representation) { + List defaultUsingDirectives = + [ + "using System;", + "using System.CodeDom.Compiler;", + "using Nethermind.Serialization.FluentRlp;", + "using Nethermind.Serialization.FluentRlp.Instances;" + ]; + IEnumerable directives = defaultUsingDirectives.Concat(usingDirectives).Distinct(); + var usingStatements = new StringBuilder(); + foreach (var usingDirective in directives) + { + usingStatements.AppendLine(usingDirective); + } + var writeCalls = new StringBuilder(); foreach (var (name, typeName) in parameters) { @@ -163,10 +186,7 @@ private static string GenerateConverterClass( $$""" // #nullable enable - using System; - using System.CodeDom.Compiler; - using Nethermind.Serialization.FluentRlp; - using Nethermind.Serialization.FluentRlp.Instances; + {{usingStatements}} {{(@namespace is null ? "" : $"namespace {@namespace};")}} From f419316fd9c0082ee9642706f8d84e9e43d2023f Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 2 Jan 2025 15:14:19 -0300 Subject: [PATCH 094/106] Fix nested Generics support --- .../RlpSourceGenerator.cs | 11 +++++------ .../RlpDerivedTest.cs | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs index 6941ec2b28d..e3c86607115 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Generator/RlpSourceGenerator.cs @@ -187,7 +187,6 @@ private static string GenerateConverterClass( // #nullable enable {{usingStatements}} - {{(@namespace is null ? "" : $"namespace {@namespace};")}} {{GeneratedCodeAttribute}} @@ -287,13 +286,13 @@ private static string MapTypeToReadCall(TypeSyntax syntax) /// Map the type name to the appropriate Write method on the `RlpWriter` /// Extend this mapping for more types as needed. /// - private static string MapTypeToWriteCall(string name, TypeSyntax syntax) + private static string MapTypeToWriteCall(string? propertyName, TypeSyntax syntax) { // Hard-coded cases switch (syntax.ToString()) { case "byte[]" or "Byte[]" or "System.Byte[]" or "Span" or "System.Span" or "ReadOnlySpan" or "System.ReadOnlySpan": - return $"Write(value.{name})"; + return propertyName is null ? "Write(value)" : $"Write(value.{propertyName})"; } // Generics @@ -308,10 +307,10 @@ private static string MapTypeToWriteCall(string name, TypeSyntax syntax) }; var sb = new StringBuilder(); - sb.AppendLine($"Write(value.{name},"); + sb.AppendLine(propertyName is null ? "Write(value," : $"Write(value.{propertyName},"); foreach (var typeParameter in typeParameters) { - sb.AppendLine($$"""static (ref RlpWriter w, {{typeParameter}} value) => { w.Write(value); },"""); + sb.AppendLine($$"""static (ref RlpWriter w, {{typeParameter}} value) => { w.{{MapTypeToWriteCall(null, typeParameter)}}; },"""); } sb.Length -= 2; // Remove the trailing `,\n` sb.Append(")"); @@ -320,7 +319,7 @@ private static string MapTypeToWriteCall(string name, TypeSyntax syntax) } // Default - return $"Write(value.{name})"; + return propertyName is not null ? $"Write(value.{propertyName})" : "Write(value)"; } private static string MapTypeAlias(string alias) => diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs index 617e71daed3..ca20b987441 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs @@ -34,6 +34,9 @@ public record IntegerTuple((int, long) Values); [RlpSerializable(RlpRepresentation.Newtype)] public record Address(string HexString); +[RlpSerializable] +public record AccessList(List<(Address, List)> Entries); + public class RlpDerivedTest { [Test] @@ -123,4 +126,18 @@ public void NewtypeRecords() decoded.Should().BeEquivalentTo(address); } + + [Test] + public void RecordWithNestedGenerics() + { + var accessList = new AccessList([ + (new Address("0x1234567890ABCDEF"), [1, 1, 3, 5, 8, 13]), + (new Address("0xFEDCBA0987654321"), [2, 4, 6, 8, 10]) + ]); + + ReadOnlySpan rlp = Rlp.Write(accessList, (ref RlpWriter writer, AccessList value) => writer.Write(value)); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadAccessList()); + decoded.Should().BeEquivalentTo(accessList); + } } From d3592d8bd9ef259cb2b486e0329383ae37efc1eb Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 2 Jan 2025 15:20:46 -0300 Subject: [PATCH 095/106] Initial benchmark --- ...hermind.Serialization.Rlp.Benchmark.csproj | 20 +++ .../Program.cs | 116 +++++++++++++++ .../UInt256RlpConverter.cs | 132 ++++++++++++++++++ src/Nethermind/Nethermind.sln | 6 + 4 files changed, 274 insertions(+) create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Nethermind.Serialization.Rlp.Benchmark.csproj create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Nethermind.Serialization.Rlp.Benchmark.csproj b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Nethermind.Serialization.Rlp.Benchmark.csproj new file mode 100644 index 00000000000..74025581ed1 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Nethermind.Serialization.Rlp.Benchmark.csproj @@ -0,0 +1,20 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + + + + + + + diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs new file mode 100644 index 00000000000..d4d2dd0c066 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs @@ -0,0 +1,116 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Running; +using Nethermind.Int256; +using Nethermind.Serialization.FluentRlp; +using Nethermind.Serialization.FluentRlp.Generator; + +namespace Nethermind.Serialization.Rlp.Benchmark; + +[RlpSerializable(RlpRepresentation.Newtype)] +public record Address(byte[] Bytes) +{ + public const int Size = 20; +} + +[RlpSerializable] +public record AccessList(List<(Address, List)> Addresses); + +public class CurrentFluentBenchmark +{ + private readonly Nethermind.Core.Eip2930.AccessList _current; + private readonly AccessList _fluent; + + public CurrentFluentBenchmark() + { + _current = Benchmark.Current.BuildAccessList(new Random(42)); + _fluent = Benchmark.Fluent.BuildAccessList(new Random(42)); + } + + [Benchmark(Baseline = true)] + public Nethermind.Core.Eip2930.AccessList Current() + { + var decoder = Eip2930.AccessListDecoder.Instance; + + var length = decoder.GetLength(_current, RlpBehaviors.None); + var stream = new RlpStream(length); + decoder.Encode(stream, _current); + + stream.Reset(); + + var decoded = decoder.Decode(stream); + return decoded!; + } + + [Benchmark] + public AccessList Fluent() + { + var decoded = FluentRlp.Rlp.Write(_fluent, (ref RlpWriter writer, AccessList value) => writer.Write(value)); + var encoded = FluentRlp.Rlp.Read(decoded, (scoped ref RlpReader reader) => reader.ReadAccessList()); + + return encoded; + } +} + +public static class Current +{ + private static Nethermind.Core.Address BuildAddress(Random rnd) + { + var bytes = new byte[Core.Address.Size]; + rnd.NextBytes(bytes); + return new Nethermind.Core.Address(bytes); + } + + public static Nethermind.Core.Eip2930.AccessList BuildAccessList(Random rnd) + { + var builder = new Nethermind.Core.Eip2930.AccessList.Builder(); + for (int i = 0; i < 1_000; i++) + { + builder.AddAddress(BuildAddress(rnd)); + var keyCount = rnd.Next(10); + for (int j = 0; j < keyCount; j++) + { + var bytes = new byte[32]; + rnd.NextBytes(bytes); + builder.AddStorage(new UInt256(bytes)); + } + } + + return builder.Build(); + } +} + +public static class Fluent +{ + private static Address BuildAddress(Random rnd) + { + var bytes = new byte[Address.Size]; + rnd.NextBytes(bytes); + return new Address(bytes); + } + + public static AccessList BuildAccessList(Random rnd) + { + var result = new List<(Address, List)>(1_000); + for (int i = 0; i < 1_000; i++) + { + Address address = BuildAddress(rnd); + List keys = []; + var keyCount = rnd.Next(10); + for (int j = 0; j < keyCount; j++) + { + var bytes = new byte[32]; + rnd.NextBytes(bytes); + keys.Add(new UInt256(bytes)); + } + + result.Add((address, keys)); + } + + return new AccessList(result); + } +} + +public static class Program +{ + public static void Main() => BenchmarkRunner.Run(typeof(Program).Assembly); +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs new file mode 100644 index 00000000000..3678bdfbf2d --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Numerics; +using Nethermind.Int256; +using Nethermind.Serialization.FluentRlp; + +namespace Nethermind.Serialization.Rlp.Benchmark; + +// NOTE: This converter is required since `UInt256` does not implement `IBinaryInteger` (which it should) +public abstract class UInt256RlpConverter : IRlpConverter +{ + private readonly struct Wrap(UInt256 value) : IBinaryInteger + { + public UInt256 Unwrap => value; + + public static bool TryReadBigEndian(ReadOnlySpan source, bool isUnsigned, out Wrap value) + { + var uint256 = new UInt256(source, isBigEndian: true); + value = new Wrap(uint256); + return true; + } + + public bool TryWriteBigEndian(Span destination, out int bytesWritten) + { + var uint256 = Unwrap; + uint256.ToBigEndian(destination); + bytesWritten = 32; + return true; + } + + // NOTE: None of the following are required + public override bool Equals(object? obj) => obj is Wrap other && Equals(other); + public override int GetHashCode() => throw new NotImplementedException(); + public int CompareTo(object? obj) => throw new NotImplementedException(); + public int CompareTo(Wrap other) => throw new NotImplementedException(); + public bool Equals(Wrap other) => throw new NotImplementedException(); + public string ToString(string? format, IFormatProvider? formatProvider) => throw new NotImplementedException(); + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) => throw new NotImplementedException(); + public static Wrap Parse(string s, IFormatProvider? provider) => throw new NotImplementedException(); + public static bool TryParse([NotNullWhen(true)] string? s, IFormatProvider? provider, out Wrap result) => throw new NotImplementedException(); + public static Wrap Parse(ReadOnlySpan s, IFormatProvider? provider) => throw new NotImplementedException(); + public static bool TryParse(ReadOnlySpan s, IFormatProvider? provider, out Wrap result) => throw new NotImplementedException(); + public static Wrap operator +(Wrap left, Wrap right) => throw new NotImplementedException(); + public static Wrap AdditiveIdentity { get; } + public static Wrap operator &(Wrap left, Wrap right) => throw new NotImplementedException(); + public static Wrap operator |(Wrap left, Wrap right) => throw new NotImplementedException(); + public static Wrap operator ^(Wrap left, Wrap right) => throw new NotImplementedException(); + public static Wrap operator ~(Wrap value) => throw new NotImplementedException(); + public static bool operator ==(Wrap left, Wrap right) => throw new NotImplementedException(); + public static bool operator !=(Wrap left, Wrap right) => throw new NotImplementedException(); + public static bool operator >(Wrap left, Wrap right) => throw new NotImplementedException(); + public static bool operator >=(Wrap left, Wrap right) => throw new NotImplementedException(); + public static bool operator <(Wrap left, Wrap right) => throw new NotImplementedException(); + public static bool operator <=(Wrap left, Wrap right) => throw new NotImplementedException(); + public static Wrap operator --(Wrap value) => throw new NotImplementedException(); + public static Wrap operator /(Wrap left, Wrap right) => throw new NotImplementedException(); + public static Wrap operator ++(Wrap value) => throw new NotImplementedException(); + public static Wrap operator %(Wrap left, Wrap right) => throw new NotImplementedException(); + public static Wrap MultiplicativeIdentity { get; } + public static Wrap operator *(Wrap left, Wrap right) => throw new NotImplementedException(); + public static Wrap operator -(Wrap left, Wrap right) => throw new NotImplementedException(); + public static Wrap operator -(Wrap value) => throw new NotImplementedException(); + public static Wrap operator +(Wrap value) => throw new NotImplementedException(); + public static Wrap Abs(Wrap value) => throw new NotImplementedException(); + public static bool IsCanonical(Wrap value) => throw new NotImplementedException(); + public static bool IsComplexNumber(Wrap value) => throw new NotImplementedException(); + public static bool IsEvenInteger(Wrap value) => throw new NotImplementedException(); + public static bool IsFinite(Wrap value) => throw new NotImplementedException(); + public static bool IsImaginaryNumber(Wrap value) => throw new NotImplementedException(); + public static bool IsInfinity(Wrap value) => throw new NotImplementedException(); + public static bool IsInteger(Wrap value) => throw new NotImplementedException(); + public static bool IsNaN(Wrap value) => throw new NotImplementedException(); + public static bool IsNegative(Wrap value) => throw new NotImplementedException(); + public static bool IsNegativeInfinity(Wrap value) => throw new NotImplementedException(); + public static bool IsNormal(Wrap value) => throw new NotImplementedException(); + public static bool IsOddInteger(Wrap value) => throw new NotImplementedException(); + public static bool IsPositive(Wrap value) => throw new NotImplementedException(); + public static bool IsPositiveInfinity(Wrap value) => throw new NotImplementedException(); + public static bool IsRealNumber(Wrap value) => throw new NotImplementedException(); + public static bool IsSubnormal(Wrap value) => throw new NotImplementedException(); + public static bool IsZero(Wrap value) => throw new NotImplementedException(); + public static Wrap MaxMagnitude(Wrap x, Wrap y) => throw new NotImplementedException(); + public static Wrap MaxMagnitudeNumber(Wrap x, Wrap y) => throw new NotImplementedException(); + public static Wrap MinMagnitude(Wrap x, Wrap y) => throw new NotImplementedException(); + public static Wrap MinMagnitudeNumber(Wrap x, Wrap y) => throw new NotImplementedException(); + public static Wrap Parse(ReadOnlySpan s, NumberStyles style, IFormatProvider? provider) => throw new NotImplementedException(); + public static Wrap Parse(string s, NumberStyles style, IFormatProvider? provider) => throw new NotImplementedException(); + public static bool TryConvertFromChecked(TOther value, out Wrap result) where TOther : INumberBase => throw new NotImplementedException(); + public static bool TryConvertFromSaturating(TOther value, out Wrap result) where TOther : INumberBase => throw new NotImplementedException(); + public static bool TryConvertFromTruncating(TOther value, out Wrap result) where TOther : INumberBase => throw new NotImplementedException(); + public static bool TryConvertToChecked(Wrap value, [MaybeNullWhen(false)] out TOther result) where TOther : INumberBase => throw new NotImplementedException(); + public static bool TryConvertToSaturating(Wrap value, [MaybeNullWhen(false)] out TOther result) where TOther : INumberBase => throw new NotImplementedException(); + public static bool TryConvertToTruncating(Wrap value, [MaybeNullWhen(false)] out TOther result) where TOther : INumberBase => throw new NotImplementedException(); + public static bool TryParse(ReadOnlySpan s, NumberStyles style, IFormatProvider? provider, out Wrap result) => throw new NotImplementedException(); + public static bool TryParse([NotNullWhen(true)] string? s, NumberStyles style, IFormatProvider? provider, out Wrap result) => throw new NotImplementedException(); + public static Wrap One { get; } + public static int Radix { get; } + public static Wrap Zero { get; } + public static bool IsPow2(Wrap value) => throw new NotImplementedException(); + public static Wrap Log2(Wrap value) => throw new NotImplementedException(); + public static Wrap operator <<(Wrap value, int shiftAmount) => throw new NotImplementedException(); + public static Wrap operator >> (Wrap value, int shiftAmount) => throw new NotImplementedException(); + public static Wrap operator >>> (Wrap value, int shiftAmount) => throw new NotImplementedException(); + public int GetByteCount() => throw new NotImplementedException(); + public int GetShortestBitLength() => throw new NotImplementedException(); + public static Wrap PopCount(Wrap value) => throw new NotImplementedException(); + public static Wrap TrailingZeroCount(Wrap value) => throw new NotImplementedException(); + public static bool TryReadLittleEndian(ReadOnlySpan source, bool isUnsigned, out Wrap value) => throw new NotImplementedException(); + public bool TryWriteLittleEndian(Span destination, out int bytesWritten) => throw new NotImplementedException(); + } + + public static UInt256 Read(ref RlpReader reader) + { + var wrap = reader.ReadInteger(); + return wrap.Unwrap; + } + + public static void Write(ref RlpWriter writer, UInt256 value) + { + var wrap = new Wrap(value); + writer.Write(wrap); + } +} + +public static class UInt256RlpConverterExt +{ + public static UInt256 ReadUInt256(this ref RlpReader reader) => UInt256RlpConverter.Read(ref reader); + public static void Write(this ref RlpWriter writer, UInt256 value) => UInt256RlpConverter.Write(ref writer, value); +} diff --git a/src/Nethermind/Nethermind.sln b/src/Nethermind/Nethermind.sln index 0b14cff7513..6c84eec649d 100644 --- a/src/Nethermind/Nethermind.sln +++ b/src/Nethermind/Nethermind.sln @@ -232,6 +232,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.Fl EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.FluentRlp.Generator", "Nethermind.Serialization.FluentRlp.Generator\Nethermind.Serialization.FluentRlp.Generator.csproj", "{838687B6-9915-4BD2-98CC-79A4130B89B0}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Serialization.Rlp.Benchmark", "Nethermind.Serialization.Rlp.Benchmark\Nethermind.Serialization.Rlp.Benchmark.csproj", "{2D355B4C-AE53-4F93-9E3B-91C027CC5075}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -642,6 +644,10 @@ Global {838687B6-9915-4BD2-98CC-79A4130B89B0}.Debug|Any CPU.Build.0 = Debug|Any CPU {838687B6-9915-4BD2-98CC-79A4130B89B0}.Release|Any CPU.ActiveCfg = Release|Any CPU {838687B6-9915-4BD2-98CC-79A4130B89B0}.Release|Any CPU.Build.0 = Release|Any CPU + {2D355B4C-AE53-4F93-9E3B-91C027CC5075}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2D355B4C-AE53-4F93-9E3B-91C027CC5075}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2D355B4C-AE53-4F93-9E3B-91C027CC5075}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2D355B4C-AE53-4F93-9E3B-91C027CC5075}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From a52d92337b9cdea08546e9c6171968e6e0403538 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 2 Jan 2025 15:45:53 -0300 Subject: [PATCH 096/106] Prefer `sizeof(T)` over `Marshal.SizeOf` - Nice speedup --- .../Nethermind.Serialization.FluentRlp.csproj | 1 + .../Nethermind.Serialization.FluentRlp/RlpWriter.cs | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Nethermind.Serialization.FluentRlp.csproj b/src/Nethermind/Nethermind.Serialization.FluentRlp/Nethermind.Serialization.FluentRlp.csproj index f1e6730cc6e..f3ba37c6fa3 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/Nethermind.Serialization.FluentRlp.csproj +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Nethermind.Serialization.FluentRlp.csproj @@ -2,6 +2,7 @@ enable + true diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs index 626b057b20f..e188fb59574 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs @@ -42,7 +42,7 @@ public static RlpWriter ContentWriter(byte[] buffer) }; } - public void Write(T value) where T : IBinaryInteger + public void Write(T value) where T : unmanaged, IBinaryInteger { switch (_mode) { @@ -55,9 +55,9 @@ public void Write(T value) where T : IBinaryInteger } } - private void LengthWrite(T value) where T : IBinaryInteger + private unsafe void LengthWrite(T value) where T : unmanaged, IBinaryInteger { - var size = Marshal.SizeOf(); + var size = sizeof(T); Span bigEndian = stackalloc byte[size]; value.WriteBigEndian(bigEndian); bigEndian = bigEndian.TrimStart((byte)0); @@ -76,9 +76,9 @@ private void LengthWrite(T value) where T : IBinaryInteger } } - private void ContentWrite(T value) where T : IBinaryInteger + private unsafe void ContentWrite(T value) where T : unmanaged, IBinaryInteger { - var size = Marshal.SizeOf(); + var size = sizeof(T); Span bigEndian = stackalloc byte[size]; value.WriteBigEndian(bigEndian); bigEndian = bigEndian.TrimStart((byte)0); From 606a260e3ccee2ec7225affca601ebdfbacab610 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 2 Jan 2025 15:47:42 -0300 Subject: [PATCH 097/106] Formatting --- .../UInt256RlpConverter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs index 3678bdfbf2d..94f61c9fa93 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs @@ -102,8 +102,8 @@ public bool TryWriteBigEndian(Span destination, out int bytesWritten) public static bool IsPow2(Wrap value) => throw new NotImplementedException(); public static Wrap Log2(Wrap value) => throw new NotImplementedException(); public static Wrap operator <<(Wrap value, int shiftAmount) => throw new NotImplementedException(); - public static Wrap operator >> (Wrap value, int shiftAmount) => throw new NotImplementedException(); - public static Wrap operator >>> (Wrap value, int shiftAmount) => throw new NotImplementedException(); + public static Wrap operator >>(Wrap value, int shiftAmount) => throw new NotImplementedException(); + public static Wrap operator >>>(Wrap value, int shiftAmount) => throw new NotImplementedException(); public int GetByteCount() => throw new NotImplementedException(); public int GetShortestBitLength() => throw new NotImplementedException(); public static Wrap PopCount(Wrap value) => throw new NotImplementedException(); From 2a9bee7269adf803be25fc585dc169a9ecbf39c0 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 2 Jan 2025 15:51:07 -0300 Subject: [PATCH 098/106] File encoding --- .../Nethermind.Serialization.Rlp.Benchmark/Program.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs index d4d2dd0c066..b2fffd81ad3 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs @@ -1,4 +1,4 @@ -using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Running; using Nethermind.Int256; using Nethermind.Serialization.FluentRlp; @@ -112,5 +112,8 @@ public static AccessList BuildAccessList(Random rnd) public static class Program { - public static void Main() => BenchmarkRunner.Run(typeof(Program).Assembly); + public static void Main() + { + BenchmarkRunner.Run(typeof(Program).Assembly); + } } From bcaa83f6d359775ab276d29d6d9a7219eb5e21c4 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 2 Jan 2025 16:15:05 -0300 Subject: [PATCH 099/106] Remove unused import --- src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs index e188fb59574..5e982f14de6 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs @@ -4,7 +4,6 @@ using System; using System.Buffers.Binary; using System.Numerics; -using System.Runtime.InteropServices; namespace Nethermind.Serialization.FluentRlp; From 783acc6d98dc0104a9e173b5274d1c333b5741a8 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 3 Jan 2025 12:43:11 -0300 Subject: [PATCH 100/106] Avoid boxing due to interface default bodies --- .../UInt256RlpConverter.cs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs index 94f61c9fa93..95a421e0c59 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/UInt256RlpConverter.cs @@ -31,6 +31,27 @@ public bool TryWriteBigEndian(Span destination, out int bytesWritten) return true; } + public int WriteBigEndian(Span destination) + { + var uint256 = Unwrap; + uint256.ToBigEndian(destination); + return 32; + } + + public int WriteBigEndian(byte[] destination, int startIndex) + { + var uint256 = Unwrap; + uint256.ToBigEndian(destination.AsSpan()[startIndex..]); + return 32; + } + + public int WriteBigEndian(byte[] destination) + { + var uint256 = Unwrap; + uint256.ToBigEndian(destination); + return 32; + } + // NOTE: None of the following are required public override bool Equals(object? obj) => obj is Wrap other && Equals(other); public override int GetHashCode() => throw new NotImplementedException(); @@ -110,6 +131,9 @@ public bool TryWriteBigEndian(Span destination, out int bytesWritten) public static Wrap TrailingZeroCount(Wrap value) => throw new NotImplementedException(); public static bool TryReadLittleEndian(ReadOnlySpan source, bool isUnsigned, out Wrap value) => throw new NotImplementedException(); public bool TryWriteLittleEndian(Span destination, out int bytesWritten) => throw new NotImplementedException(); + public int WriteLittleEndian(Span destination) => throw new NotImplementedException(); + public int WriteLittleEndian(byte[] destination) => throw new NotImplementedException(); + public int WriteLittleEndian(byte[] destination, int startIndex) => throw new NotImplementedException(); } public static UInt256 Read(ref RlpReader reader) From 6dc21f4a1535c21f35fbbec8aed6ea83564f8cc6 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 3 Jan 2025 12:59:31 -0300 Subject: [PATCH 101/106] Add `MemoryDiagnoser` --- src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs index b2fffd81ad3..32c290c3480 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp.Benchmark/Program.cs @@ -15,6 +15,7 @@ public record Address(byte[] Bytes) [RlpSerializable] public record AccessList(List<(Address, List)> Addresses); +[MemoryDiagnoser] public class CurrentFluentBenchmark { private readonly Nethermind.Core.Eip2930.AccessList _current; From 1c8f472ecc15679ca07080bcc2e3da2b93c21a81 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 6 Jan 2025 15:38:58 -0300 Subject: [PATCH 102/106] Use `IBufferWriter` instead of `byte[]` - Return value is now `ReadOnlyMemory` - Add overloads for reading `ReadOnlyMemory` - Add `FluentAssertions` extensions --- .../Extensions.cs | 32 +++++++++++++++++++ .../RlpDerivedTest.cs | 14 ++++---- .../RlpReadWriteTest.cs | 3 +- .../Nethermind.Serialization.FluentRlp/Rlp.cs | 16 +++++++--- .../RlpWriter.cs | 29 ++++++++--------- 5 files changed, 65 insertions(+), 29 deletions(-) create mode 100644 src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Extensions.cs diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Extensions.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Extensions.cs new file mode 100644 index 00000000000..f2c491248ca --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/Extensions.cs @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using FluentAssertions.Collections; + +namespace Nethermind.Serialization.FluentRlp.Test; + +// NOTE: `FluentAssertions` currently does not support `(ReadOnly)Span` or `(ReadOnly)Memory` assertions. +public static class Extensions +{ + public static GenericCollectionAssertions Should(this ReadOnlySpan span) => span.ToArray().Should(); + public static GenericCollectionAssertions Should(this ReadOnlyMemory memory) => memory.ToArray().Should(); + + public static AndConstraint> BeEquivalentTo( + this GenericCollectionAssertions @this, + ReadOnlySpan expectation, + string because = "", + params object[] becauseArgs) + { + return @this.BeEquivalentTo(expectation.ToArray(), config => config, because, becauseArgs); + } + + public static AndConstraint> BeEquivalentTo( + this GenericCollectionAssertions @this, + ReadOnlyMemory expectation, + string because = "", + params object[] becauseArgs) + { + return @this.BeEquivalentTo(expectation.ToArray(), config => config, because, becauseArgs); + } +} diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs index ca20b987441..98558ca22de 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs @@ -43,7 +43,7 @@ public class RlpDerivedTest public void FlatRecord() { var player = new Player(Id: 42, Username: "SuperUser"); - ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, Player player) => w.Write(player)); + var rlp = Rlp.Write(player, static (ref RlpWriter w, Player player) => w.Write(player)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayer()); decoded.Should().BeEquivalentTo(player); @@ -53,7 +53,7 @@ public void FlatRecord() public void RecordWithList() { var player = new PlayerWithFriends(Id: 42, Username: "SuperUser", Friends: ["ana", "bob"]); - ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithFriends player) => w.Write(player)); + var rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithFriends player) => w.Write(player)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayerWithFriends()); decoded.Should().BeEquivalentTo(player); @@ -63,7 +63,7 @@ public void RecordWithList() public void RecordWithArray() { var player = new PlayerWithCodes(Id: 42, Username: "SuperUser", Codes: [2, 4, 8, 16, 32, 64]); - ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithCodes player) => w.Write(player)); + var rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithCodes player) => w.Write(player)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayerWithCodes()); decoded.Should().BeEquivalentTo(player); @@ -77,7 +77,7 @@ public void RecordWithDictionary() { "foo", 42 }, { "bar", 1337 } }); - ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithScores player) => w.Write(player)); + var rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithScores player) => w.Write(player)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayerWithScores()); decoded.Should().BeEquivalentTo(player); @@ -87,7 +87,7 @@ public void RecordWithDictionary() public void RecordWithTuple() { var integerTuple = new IntegerTuple((42, 1337)); - ReadOnlySpan rlp = Rlp.Write(integerTuple, static (ref RlpWriter w, IntegerTuple tuple) => w.Write(tuple)); + var rlp = Rlp.Write(integerTuple, static (ref RlpWriter w, IntegerTuple tuple) => w.Write(tuple)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadIntegerTuple()); decoded.Should().BeEquivalentTo(integerTuple); @@ -103,7 +103,7 @@ [new Tree("dog", [])]), new Tree("qux", [new Tree("cat", [])]) ]); - ReadOnlySpan rlp = Rlp.Write(tree, static (ref RlpWriter w, Tree tree) => w.Write(tree)); + var rlp = Rlp.Write(tree, static (ref RlpWriter w, Tree tree) => w.Write(tree)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadTree()); decoded.Should().BeEquivalentTo(tree); @@ -135,7 +135,7 @@ public void RecordWithNestedGenerics() (new Address("0xFEDCBA0987654321"), [2, 4, 6, 8, 10]) ]); - ReadOnlySpan rlp = Rlp.Write(accessList, (ref RlpWriter writer, AccessList value) => writer.Write(value)); + var rlp = Rlp.Write(accessList, (ref RlpWriter writer, AccessList value) => writer.Write(value)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadAccessList()); decoded.Should().BeEquivalentTo(accessList); diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs index 1bb57f1d2ec..46eacf22f4f 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs @@ -201,11 +201,10 @@ public void Choice() { RefRlpReaderFunc intReader = static (scoped ref RlpReader r) => r.ReadInt32(); RefRlpReaderFunc wrappedReader = (scoped ref RlpReader r) => r.ReadSequence(intReader); - var intRlp = Rlp.Write(static (ref RlpWriter w) => { w.Write(42); }); var wrappedIntRlp = Rlp.Write(static (ref RlpWriter w) => w.WriteSequence(static (ref RlpWriter w) => { w.Write(42); })); - foreach (var rlp in (byte[][])[intRlp, wrappedIntRlp]) + foreach (var rlp in (ReadOnlyMemory[])[intRlp, wrappedIntRlp]) { int decoded = Rlp.Read(rlp, (scoped ref RlpReader r) => r.Choice(wrappedReader, intReader)); diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs index 3cfe43eb5ff..0768079d1e9 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs @@ -2,26 +2,32 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers; +using System.Runtime.CompilerServices; namespace Nethermind.Serialization.FluentRlp; public static class Rlp { - public static byte[] Write(RefRlpWriterAction action) + public static ReadOnlyMemory Write(RefRlpWriterAction action) => Write(action, static (ref RlpWriter w, RefRlpWriterAction action) => action(ref w)); - public static byte[] Write(TContext ctx, RefRlpWriterAction action) + public static ReadOnlyMemory Write(TContext ctx, RefRlpWriterAction action) where TContext : allows ref struct { var lengthWriter = RlpWriter.LengthWriter(); action(ref lengthWriter, ctx); - var serialized = new byte[lengthWriter.Length]; - var contentWriter = RlpWriter.ContentWriter(serialized); + var buffer = new ArrayBufferWriter(lengthWriter.Length); + var contentWriter = RlpWriter.ContentWriter(buffer); action(ref contentWriter, ctx); - return serialized; + return buffer.WrittenMemory; } + public static T Read(ReadOnlyMemory source, RefRlpReaderFunc func) where T : allows ref struct + => Read(source.Span, func); + + [OverloadResolutionPriority(1)] public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) where T : allows ref struct { diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs index 5e982f14de6..bcc144b28b5 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers; using System.Buffers.Binary; using System.Numerics; @@ -21,8 +22,7 @@ public ref struct RlpWriter public int Length { get; private set; } - private byte[] _buffer; - private int _position; + private IBufferWriter _buffer; public static RlpWriter LengthWriter() { @@ -32,7 +32,7 @@ public static RlpWriter LengthWriter() }; } - public static RlpWriter ContentWriter(byte[] buffer) + public static RlpWriter ContentWriter(IBufferWriter buffer) { return new RlpWriter { @@ -84,11 +84,11 @@ private unsafe void ContentWrite(T value) where T : unmanaged, IBinaryInteger if (bigEndian.Length == 0) { - _buffer[_position++] = 0x80; + _buffer.Write([(byte)0x80]); } else if (bigEndian.Length == 1 && bigEndian[0] < 0x80) { - _buffer[_position++] = bigEndian[0]; + _buffer.Write(bigEndian[..1]); } else { @@ -130,20 +130,19 @@ private void ContentWrite(scoped ReadOnlySpan value) { if (value.Length < 55) { - _buffer[_position++] = (byte)(0x80 + value.Length); + _buffer.Write([(byte)(0x80 + value.Length)]); } else { Span binaryLength = stackalloc byte[sizeof(int)]; BinaryPrimitives.WriteInt32BigEndian(binaryLength, value.Length); binaryLength = binaryLength.TrimStart((byte)0); - _buffer[_position++] = (byte)(0xB7 + binaryLength.Length); - binaryLength.CopyTo(_buffer.AsSpan()[_position..]); - _position += binaryLength.Length; + + _buffer.Write([(byte)(0xB7 + binaryLength.Length)]); + _buffer.Write(binaryLength); } - value.CopyTo(_buffer.AsSpan()[_position..]); - _position += value.Length; + _buffer.Write(value); } public void WriteSequence(RefRlpWriterAction action) @@ -185,16 +184,16 @@ private void ContentWriteSequence(TContext ctx, RefRlpWriterAction binaryLength = stackalloc byte[sizeof(Int32)]; BinaryPrimitives.WriteInt32BigEndian(binaryLength, lengthWriter.Length); binaryLength = binaryLength.TrimStart((byte)0); - _buffer[_position++] = (byte)(0xF7 + binaryLength.Length); - binaryLength.CopyTo(_buffer.AsSpan()[_position..]); - _position += binaryLength.Length; + + _buffer.Write([(byte)(0xF7 + binaryLength.Length)]); + _buffer.Write(binaryLength); } action(ref this, ctx); From b2e59512b9928117cdb8d08a4c8d5c460d6f3079 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 6 Jan 2025 15:38:58 -0300 Subject: [PATCH 103/106] Use `IBufferWriter` instead of `byte[]` --- .../RlpDerivedTest.cs | 14 +++--- .../RlpReadWriteTest.cs | 1 - .../Nethermind.Serialization.FluentRlp/Rlp.cs | 48 +++++++++++++++++-- .../RlpWriter.cs | 29 ++++++----- 4 files changed, 66 insertions(+), 26 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs index ca20b987441..98558ca22de 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpDerivedTest.cs @@ -43,7 +43,7 @@ public class RlpDerivedTest public void FlatRecord() { var player = new Player(Id: 42, Username: "SuperUser"); - ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, Player player) => w.Write(player)); + var rlp = Rlp.Write(player, static (ref RlpWriter w, Player player) => w.Write(player)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayer()); decoded.Should().BeEquivalentTo(player); @@ -53,7 +53,7 @@ public void FlatRecord() public void RecordWithList() { var player = new PlayerWithFriends(Id: 42, Username: "SuperUser", Friends: ["ana", "bob"]); - ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithFriends player) => w.Write(player)); + var rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithFriends player) => w.Write(player)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayerWithFriends()); decoded.Should().BeEquivalentTo(player); @@ -63,7 +63,7 @@ public void RecordWithList() public void RecordWithArray() { var player = new PlayerWithCodes(Id: 42, Username: "SuperUser", Codes: [2, 4, 8, 16, 32, 64]); - ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithCodes player) => w.Write(player)); + var rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithCodes player) => w.Write(player)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayerWithCodes()); decoded.Should().BeEquivalentTo(player); @@ -77,7 +77,7 @@ public void RecordWithDictionary() { "foo", 42 }, { "bar", 1337 } }); - ReadOnlySpan rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithScores player) => w.Write(player)); + var rlp = Rlp.Write(player, static (ref RlpWriter w, PlayerWithScores player) => w.Write(player)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadPlayerWithScores()); decoded.Should().BeEquivalentTo(player); @@ -87,7 +87,7 @@ public void RecordWithDictionary() public void RecordWithTuple() { var integerTuple = new IntegerTuple((42, 1337)); - ReadOnlySpan rlp = Rlp.Write(integerTuple, static (ref RlpWriter w, IntegerTuple tuple) => w.Write(tuple)); + var rlp = Rlp.Write(integerTuple, static (ref RlpWriter w, IntegerTuple tuple) => w.Write(tuple)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadIntegerTuple()); decoded.Should().BeEquivalentTo(integerTuple); @@ -103,7 +103,7 @@ [new Tree("dog", [])]), new Tree("qux", [new Tree("cat", [])]) ]); - ReadOnlySpan rlp = Rlp.Write(tree, static (ref RlpWriter w, Tree tree) => w.Write(tree)); + var rlp = Rlp.Write(tree, static (ref RlpWriter w, Tree tree) => w.Write(tree)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadTree()); decoded.Should().BeEquivalentTo(tree); @@ -135,7 +135,7 @@ public void RecordWithNestedGenerics() (new Address("0xFEDCBA0987654321"), [2, 4, 6, 8, 10]) ]); - ReadOnlySpan rlp = Rlp.Write(accessList, (ref RlpWriter writer, AccessList value) => writer.Write(value)); + var rlp = Rlp.Write(accessList, (ref RlpWriter writer, AccessList value) => writer.Write(value)); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => r.ReadAccessList()); decoded.Should().BeEquivalentTo(accessList); diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs index 1bb57f1d2ec..cd153890de9 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs @@ -201,7 +201,6 @@ public void Choice() { RefRlpReaderFunc intReader = static (scoped ref RlpReader r) => r.ReadInt32(); RefRlpReaderFunc wrappedReader = (scoped ref RlpReader r) => r.ReadSequence(intReader); - var intRlp = Rlp.Write(static (ref RlpWriter w) => { w.Write(42); }); var wrappedIntRlp = Rlp.Write(static (ref RlpWriter w) => w.WriteSequence(static (ref RlpWriter w) => { w.Write(42); })); diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs index 3cfe43eb5ff..e918ebdbd33 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/Rlp.cs @@ -2,6 +2,8 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers; +using System.Diagnostics; namespace Nethermind.Serialization.FluentRlp; @@ -15,11 +17,11 @@ public static byte[] Write(TContext ctx, RefRlpWriterAction { var lengthWriter = RlpWriter.LengthWriter(); action(ref lengthWriter, ctx); - var serialized = new byte[lengthWriter.Length]; - var contentWriter = RlpWriter.ContentWriter(serialized); + var bufferWriter = new FixedArrayBufferWriter(lengthWriter.Length); + var contentWriter = RlpWriter.ContentWriter(bufferWriter); action(ref contentWriter, ctx); - return serialized; + return bufferWriter.Buffer; } public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) @@ -31,3 +33,43 @@ public static T Read(ReadOnlySpan source, RefRlpReaderFunc func) return result; } } + +/// +/// The existing performs various bound checks and supports resizing buffers +/// which we don't need for our use case. +/// +internal class FixedArrayBufferWriter : IBufferWriter +{ + private readonly T[] _buffer; + private int _index; + + public T[] Buffer => _buffer; + + /// + /// Creates an instance of an , in which data can be written to, + /// with a fixed capacity specified. + /// + /// The capacity of the underlying buffer. + public FixedArrayBufferWriter(int capacity) + { + _buffer = new T[capacity]; + _index = 0; + } + + public void Advance(int count) + { + _index += count; + } + + public Memory GetMemory(int sizeHint = 0) + { + Debug.Assert(_buffer.Length > _index); + return _buffer.AsMemory(_index); + } + + public Span GetSpan(int sizeHint = 0) + { + Debug.Assert(_buffer.Length > _index); + return _buffer.AsSpan(_index); + } +} diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs index 5e982f14de6..bcc144b28b5 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpWriter.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers; using System.Buffers.Binary; using System.Numerics; @@ -21,8 +22,7 @@ public ref struct RlpWriter public int Length { get; private set; } - private byte[] _buffer; - private int _position; + private IBufferWriter _buffer; public static RlpWriter LengthWriter() { @@ -32,7 +32,7 @@ public static RlpWriter LengthWriter() }; } - public static RlpWriter ContentWriter(byte[] buffer) + public static RlpWriter ContentWriter(IBufferWriter buffer) { return new RlpWriter { @@ -84,11 +84,11 @@ private unsafe void ContentWrite(T value) where T : unmanaged, IBinaryInteger if (bigEndian.Length == 0) { - _buffer[_position++] = 0x80; + _buffer.Write([(byte)0x80]); } else if (bigEndian.Length == 1 && bigEndian[0] < 0x80) { - _buffer[_position++] = bigEndian[0]; + _buffer.Write(bigEndian[..1]); } else { @@ -130,20 +130,19 @@ private void ContentWrite(scoped ReadOnlySpan value) { if (value.Length < 55) { - _buffer[_position++] = (byte)(0x80 + value.Length); + _buffer.Write([(byte)(0x80 + value.Length)]); } else { Span binaryLength = stackalloc byte[sizeof(int)]; BinaryPrimitives.WriteInt32BigEndian(binaryLength, value.Length); binaryLength = binaryLength.TrimStart((byte)0); - _buffer[_position++] = (byte)(0xB7 + binaryLength.Length); - binaryLength.CopyTo(_buffer.AsSpan()[_position..]); - _position += binaryLength.Length; + + _buffer.Write([(byte)(0xB7 + binaryLength.Length)]); + _buffer.Write(binaryLength); } - value.CopyTo(_buffer.AsSpan()[_position..]); - _position += value.Length; + _buffer.Write(value); } public void WriteSequence(RefRlpWriterAction action) @@ -185,16 +184,16 @@ private void ContentWriteSequence(TContext ctx, RefRlpWriterAction binaryLength = stackalloc byte[sizeof(Int32)]; BinaryPrimitives.WriteInt32BigEndian(binaryLength, lengthWriter.Length); binaryLength = binaryLength.TrimStart((byte)0); - _buffer[_position++] = (byte)(0xF7 + binaryLength.Length); - binaryLength.CopyTo(_buffer.AsSpan()[_position..]); - _position += binaryLength.Length; + + _buffer.Write([(byte)(0xF7 + binaryLength.Length)]); + _buffer.Write(binaryLength); } action(ref this, ctx); From 7e03ac3960370bb0a174ee0896be67fe385da82a Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 10 Jan 2025 14:11:22 -0300 Subject: [PATCH 104/106] Add support for `Optional` --- .../RlpReadWriteTest.cs | 47 +++++++++++++++++++ .../RlpReader.cs | 28 +++++++++++ 2 files changed, 75 insertions(+) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs index cd153890de9..673fa1bb45b 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs @@ -252,6 +252,53 @@ public void ChoiceDeep() decoded.Should().Be(("dog", "cat", "42")); } + [Test] + public void OptionalStruct() + { + (int, int?) tuple = (42, null); + + var rlp = Rlp.Write(tuple, static (ref RlpWriter w, (int _1, int? _2) value) => + { + w.Write(value._1); + if (value._2.HasValue) + { + w.Write(value._2.Value); + } + }); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => + { + var _1 = r.ReadInt32(); + var _2 = r.Optional(static (scoped ref RlpReader r) => r.ReadInt32()); + + return (_1, _2); + }); + + decoded.Should().Be(tuple); + } + + + [Test] + public void OptionalReference() + { + string? value = null; + + var rlp = Rlp.Write(value, static (ref RlpWriter w, string? value) => + { + if (value is not null) + { + w.Write(value); + } + }); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => + { + return r.Optional(static (scoped ref RlpReader r) => r.ReadString()); + }); + + decoded.Should().Be(value); + } + [Test] public void UserDefinedRecord() { diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs index f3af28e2bf1..786f4c11859 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs @@ -124,4 +124,32 @@ public T Choice(params ReadOnlySpan> alternatives) } throw new RlpReaderException("RLP does not correspond to any alternative"); } + + public T? Optional(RefRlpReaderFunc f, T? _ = null) where T : class + { + int startingPosition = _position; + try + { + return f(ref this); + } + catch (Exception) + { + _position = startingPosition; + return null; + } + } + + public T? Optional(RefRlpReaderFunc f, T? _ = null) where T : struct + { + int startingPosition = _position; + try + { + return f(ref this); + } + catch (Exception) + { + _position = startingPosition; + return null; + } + } } From 04ac6b210b3a44cc573e92049e0a8278bb71e4f0 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 10 Jan 2025 15:17:05 -0300 Subject: [PATCH 105/106] Use `sizeof` instead of `Marshal.SizeOf` in `RlpReader` --- .../Nethermind.Serialization.FluentRlp/RlpReader.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs index 786f4c11859..6d7048de3bd 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp/RlpReader.cs @@ -3,7 +3,6 @@ using System; using System.Numerics; -using System.Runtime.InteropServices; namespace Nethermind.Serialization.FluentRlp; @@ -29,7 +28,7 @@ public RlpReader(ReadOnlySpan buffer) public bool HasNext => _position < _buffer.Length; public int BytesRead => _position; - public T ReadInteger() where T : IBinaryInteger + public unsafe T ReadInteger() where T : unmanaged, IBinaryInteger { ReadOnlySpan bigEndian; var header = _buffer[_position]; @@ -42,7 +41,7 @@ public T ReadInteger() where T : IBinaryInteger bigEndian = ReadBytes(); } - Span buffer = stackalloc byte[Marshal.SizeOf()]; + Span buffer = stackalloc byte[sizeof(T)]; bigEndian.CopyTo(buffer[^bigEndian.Length..]); return T.ReadBigEndian(buffer, false); } From 3250b158f8adfb9c5401fbc4e67a2c820c432d13 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 10 Jan 2025 15:25:19 -0300 Subject: [PATCH 106/106] More tests for `Optional` --- .../RlpReadWriteTest.cs | 48 +++++++++++++++---- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs index 673fa1bb45b..ac08499fe2a 100644 --- a/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs +++ b/src/Nethermind/Nethermind.Serialization.FluentRlp.Test/RlpReadWriteTest.cs @@ -255,26 +255,22 @@ public void ChoiceDeep() [Test] public void OptionalStruct() { - (int, int?) tuple = (42, null); + int? value = null; - var rlp = Rlp.Write(tuple, static (ref RlpWriter w, (int _1, int? _2) value) => + var rlp = Rlp.Write(value, static (ref RlpWriter w, int? value) => { - w.Write(value._1); - if (value._2.HasValue) + if (value.HasValue) { - w.Write(value._2.Value); + w.Write(value.Value); } }); var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => { - var _1 = r.ReadInt32(); - var _2 = r.Optional(static (scoped ref RlpReader r) => r.ReadInt32()); - - return (_1, _2); + return r.Optional(static (scoped ref RlpReader r) => r.ReadInt32()); }); - decoded.Should().Be(tuple); + decoded.Should().Be(value); } @@ -299,6 +295,38 @@ public void OptionalReference() decoded.Should().Be(value); } + [Test] + public void OptionalDeep() + { + (string, string?, int, int?) tuple = ("dog", null, 42, null); + + var rlp = Rlp.Write(tuple, static (ref RlpWriter w, (string _1, string? _2, int _3, int? _4) tuple) => + { + w.Write(tuple._1); + if (tuple._2 is not null) + { + w.Write(tuple._2); + } + w.Write(tuple._3); + if (tuple._4.HasValue) + { + w.Write(tuple._4.Value); + } + }); + + var decoded = Rlp.Read(rlp, static (scoped ref RlpReader r) => + { + var _1 = r.ReadString(); + var _2 = r.Optional(static (scoped ref RlpReader r) => r.ReadString()); + var _3 = r.ReadInt32(); + var _4 = r.Optional(static (scoped ref RlpReader r) => r.ReadInt32()); + + return (_1, _2, _3, _4); + }); + + decoded.Should().Be(tuple); + } + [Test] public void UserDefinedRecord() {