Feature/Era1 import/export

src/Nethermind/Directory.Packages.props

@@ -13,6 +13,7 @@
     <PackageVersion Include="BouncyCastle.Cryptography" Version="2.4.0" />
     <PackageVersion Include="Ckzg.Bindings" Version="1.0.2.855" />
     <PackageVersion Include="Colorful.Console" Version="1.2.15" />
+    <PackageVersion Include="CommunityToolkit.HighPerformance" Version="8.3.2" />
     <PackageVersion Include="ConcurrentHashSet" Version="1.3.0" />
     <PackageVersion Include="coverlet.collector" Version="6.0.2" />
     <PackageVersion Include="Crc32.NET" Version="1.2.0" />

src/Nethermind/Nethermind.Api/IBasicApi.cs

@@ -15,6 +15,7 @@
 using Nethermind.Core.Timers;
 using Nethermind.Crypto;
 using Nethermind.Db;
+using Nethermind.Era1;
 using Nethermind.KeyStore;
 using Nethermind.Logging;
 using Nethermind.Serialization.Json;
@@ -63,7 +64,8 @@ public ContainerBuilder ConfigureContainerBuilderFromBasicApi(ContainerBuilder b
         {
             builder
                 .AddPropertiesFrom<IBasicApi>(this)
-                .AddSingleton(ConfigProvider.GetConfig<ISyncConfig>());
+                .AddSingleton(ConfigProvider.GetConfig<ISyncConfig>())
+                .AddSingleton(ConfigProvider.GetConfig<IEraConfig>());
 
             DbProvider!.ConfigureServiceCollection(builder);
 

src/Nethermind/Nethermind.Blockchain/BlockTreeSuggestPacer.cs

@@ -19,7 +19,7 @@ public class BlockTreeSuggestPacer : IDisposable
     private readonly long _resumeBatchSize;
     private readonly IBlockTree _blockTree;
 
-    public BlockTreeSuggestPacer(IBlockTree blockTree, long stopBatchSize, long resumeBatchSize)
+    public BlockTreeSuggestPacer(IBlockTree blockTree, long stopBatchSize = 4096, long resumeBatchSize = 2048)
     {
         blockTree.NewHeadBlock += BlockTreeOnNewHeadBlock;
         _blockTree = blockTree;

src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Linq;
+using Nethermind.Blockchain;
 using Nethermind.Core.ConsensusRequests;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Specs;
@@ -193,6 +194,7 @@ public BlockBuilder WithUncles(params BlockHeader[] uncles)
         {
             TestObjectInternal = TestObjectInternal.WithReplacedBody(
                 TestObjectInternal.Body.WithChangedUncles(uncles));
+            TestObjectInternal.Header.UnclesHash = UnclesHash.Calculate(uncles);
             return this;
         }
 

src/Nethermind/Nethermind.Core.Test/Builders/TestItem.cs

@@ -199,5 +199,12 @@ public static byte[] GenerateIndexedAccountRlp(int index, AccountDecoder? accoun
             byte[] value = accountDecoder.Encode(account).Bytes;
             return value;
         }
+
+        public static UInt256 GetRandomAmount(Random? random = null)
+        {
+            Span<byte> buffer = stackalloc byte[32];
+            (random ?? Random).NextBytes(buffer);
+            return new UInt256(buffer);
+        }
     }
 }

src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs

@@ -34,7 +34,9 @@ public static ContainerBuilder AddPropertiesFrom<T>(this ContainerBuilder config
             object? val = propertyInfo.GetValue(source);
             if (val != null)
             {
-                configuration.RegisterInstance(val).As(propertyInfo.PropertyType);
+                configuration.RegisterInstance(val)
+                    .As(propertyInfo.PropertyType)
+                    .ExternallyOwned();
             }
         }
 
@@ -55,7 +57,8 @@ public static ContainerBuilder AddSingleton<T>(this ContainerBuilder builder, T
     {
         builder.RegisterInstance(instance)
             .As<T>()
-            .SingleInstance();
+            .SingleInstance()
+            .ExternallyOwned();
 
         return builder;
     }
@@ -71,11 +74,12 @@ public static ContainerBuilder AddSingleton<T, TImpl>(this ContainerBuilder buil
         return builder;
     }
 
-    public static ContainerBuilder AddKeyedSingleton<T>(this ContainerBuilder builder, string key, T instance) where T : class
+    public static ContainerBuilder AddKeyedSingleton<T>(this ContainerBuilder builder, object key, T instance) where T : class
     {
         builder.RegisterInstance(instance)
-            .Named<T>(key)
-            .SingleInstance();
+            .Keyed<T>(key)
+            .SingleInstance()
+            .ExternallyOwned();
 
         return builder;
     }

src/Nethermind/Nethermind.Era.Test/AccumulatorCalculatorTests.cs

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Runtime.Intrinsics.Arm;
+using System.Text;
+using System.Threading.Tasks;
+using Nethermind.Core.Crypto;
+
+namespace Nethermind.Era1.Test;
+public class AccumulatorCalculatorTests
+{
+    [Test]
+    public void Add_AddOneHashAndUInt256_DoesNotThrow()
+    {
+        using var sut = new AccumulatorCalculator();
+
+        Assert.That(() => sut.Add(Keccak.Zero, 0), Throws.Nothing);
+    }
+    [Test]
+    public void ComputeRoot_AddValues_ReturnsExpectedResult()
+    {
+        using var sut = new AccumulatorCalculator();
+        sut.Add(Keccak.Zero, 1);
+        sut.Add(Keccak.MaxValue, 2);
+
+        var result = sut.ComputeRoot().ToByteArray();
+
+        Assert.That(result, Is.EquivalentTo(new[] { 0x3E, 0xD6, 0x26, 0x52, 0xDF, 0xB7, 0xE1, 0x07, 0x2D, 0x0F, 0x04, 0x0F, 0xEB, 0x6D, 0x00, 0x2A, 0x9F, 0x7C, 0xE3, 0x7C, 0xF8, 0xDD, 0xB1, 0x65, 0x49, 0xA7, 0xAC, 0x5C, 0xF8, 0xE3, 0xB7, 0x91 }));
+    }
+
+    [Test]
+    public void ComputeRoot_AddOneHashAndUInt256_DoesNotThrow()
+    {
+        using var sut = new AccumulatorCalculator();
+        sut.Add(Keccak.Zero, 1);
+        sut.Add(Keccak.MaxValue, 2);
+
+        var result = sut.ComputeRoot().ToByteArray();
+
+        Assert.That(result, Is.EquivalentTo(new[] { 0x3E, 0xD6, 0x26, 0x52, 0xDF, 0xB7, 0xE1, 0x07, 0x2D, 0x0F, 0x04, 0x0F, 0xEB, 0x6D, 0x00, 0x2A, 0x9F, 0x7C, 0xE3, 0x7C, 0xF8, 0xDD, 0xB1, 0x65, 0x49, 0xA7, 0xAC, 0x5C, 0xF8, 0xE3, 0xB7, 0x91 }));
+    }
+}

src/Nethermind/Nethermind.Era.Test/AdminEraServiceTests.cs

@@ -0,0 +1,83 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using Nethermind.Config;
+using Nethermind.Logging;
+using NSubstitute;
+
+namespace Nethermind.Era1.Test;
+
+public class AdminEraServiceTests
+{
+    [Test]
+    public void CanCallcExport()
+    {
+        IEraImporter importer = Substitute.For<IEraImporter>();
+        AdminEraService adminEraService = new AdminEraService(
+            importer,
+            Substitute.For<IEraExporter>(),
+            Substitute.For<IProcessExitSource>(),
+            LimboLogs.Instance);
+
+        adminEraService.ImportHistory("somewhere", 99, 999, null);
+        importer.Received().Import("somewhere", 99, 999, null);
+    }
+
+    [Test]
+    public void ThrowsWhenExistingImportIsRunning()
+    {
+        IEraImporter importer = Substitute.For<IEraImporter>();
+        TaskCompletionSource importTcs = new TaskCompletionSource();
+        importer.Import("somewhere", 99, 999, null).Returns(importTcs.Task);
+        AdminEraService adminEraService = new AdminEraService(
+            importer,
+            Substitute.For<IEraExporter>(),
+            Substitute.For<IProcessExitSource>(),
+            LimboLogs.Instance);
+
+        adminEraService.ImportHistory("somewhere", 99, 999, null);
+
+        Assert.That(() => adminEraService.ImportHistory("somewhere", 99, 999, null), Throws.Exception);
+
+        importTcs.TrySetResult();
+
+        // Not throw
+        adminEraService.ImportHistory("somewhere", 99, 999, null);
+    }
+
+    [Test]
+    public void CanCallExport()
+    {
+        IEraExporter exporter = Substitute.For<IEraExporter>();
+        AdminEraService adminEraService = new AdminEraService(
+            Substitute.For<IEraImporter>(),
+            exporter,
+            Substitute.For<IProcessExitSource>(),
+            LimboLogs.Instance);
+
+        adminEraService.ExportHistory("somewhere", 99, 999);
+        exporter.Received().Export("somewhere", 99, 999);
+    }
+
+    [Test]
+    public void ThrowsWhenExistingExportIsRunning()
+    {
+        IEraExporter exporter = Substitute.For<IEraExporter>();
+        TaskCompletionSource importTcs = new TaskCompletionSource();
+        exporter.Export("somewhere", 99, 999).Returns(importTcs.Task);
+        AdminEraService adminEraService = new AdminEraService(
+            Substitute.For<IEraImporter>(),
+            exporter,
+            Substitute.For<IProcessExitSource>(),
+            LimboLogs.Instance);
+
+        adminEraService.ExportHistory("somewhere", 99, 999);
+
+        Assert.That(() => adminEraService.ExportHistory("somewhere", 99, 999), Throws.Exception);
+
+        importTcs.TrySetResult();
+
+        // Not throw
+        adminEraService.ExportHistory("somewhere", 99, 999);
+    }
+}

src/Nethermind/Nethermind.Era.Test/E2StoreWriterTests.cs

@@ -0,0 +1,142 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Buffers.Binary;
+using System.IO.MemoryMappedFiles;
+using DotNetty.Buffers;
+using FluentAssertions;
+using Nethermind.Core.Extensions;
+using Nethermind.Serialization.Rlp;
+using Snappier;
+
+namespace Nethermind.Era1.Test;
+internal class E2StoreWriterTests
+{
+    byte[] TestBytes = new byte[] { 0x0f, 0xf0, 0xff, 0xff };
+
+    [TestCase(EntryTypes.Version)]
+    [TestCase(EntryTypes.CompressedHeader)]
+    [TestCase(EntryTypes.CompressedBody)]
+    [TestCase(EntryTypes.CompressedReceipts)]
+    [TestCase(EntryTypes.Accumulator)]
+    [TestCase(EntryTypes.BlockIndex)]
+    public async Task WriteEntry_WritingAnEntry_WritesCorrectHeaderType(ushort type)
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2StoreWriter sut = new E2StoreWriter(stream);
+
+        await sut.WriteEntry(type, Array.Empty<byte>());
+
+        Assert.That(BinaryPrimitives.ReadInt16LittleEndian(stream.ToArray()), Is.EqualTo(type));
+    }
+
+    [TestCase(6)]
+    [TestCase(20)]
+    [TestCase(32)]
+    public async Task WriteEntry_WritingAnEntry_WritesCorrectLengthInHeader(int length)
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2StoreWriter sut = new E2StoreWriter(stream);
+
+        await sut.WriteEntry(EntryTypes.CompressedHeader, new byte[length]);
+
+        Assert.That(BinaryPrimitives.ReadInt32LittleEndian(stream.ToArray().Slice(2)), Is.EqualTo(length));
+    }
+
+    [TestCase(1)]
+    [TestCase(5)]
+    [TestCase(12)]
+    public async Task WriteEntry_WritingAnEntry_ReturnCorrectNumberofBytesWritten(int length)
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2StoreWriter sut = new E2StoreWriter(stream);
+
+        int result = await sut.WriteEntry(EntryTypes.CompressedHeader, new byte[length]);
+
+        Assert.That(result, Is.EqualTo(length + E2StoreWriter.HeaderSize));
+    }
+
+
+    [Test]
+    public async Task WriteEntry_WritingAnEntry_ZeroesAtCorrectIndexesInHeader()
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2StoreWriter sut = new E2StoreWriter(stream);
+
+        await sut.WriteEntry(EntryTypes.CompressedHeader, TestBytes);
+        byte[] bytes = stream.ToArray();
+
+        Assert.That(bytes[6], Is.EqualTo(0));
+        Assert.That(bytes[7], Is.EqualTo(0));
+    }
+
+    [Test]
+    public async Task WriteEntry_WritingEntryValue_BytesAreWrittenToStream()
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2StoreWriter sut = new E2StoreWriter(stream);
+
+        await sut.WriteEntry(EntryTypes.CompressedHeader, TestBytes);
+        byte[] result = stream.ToArray();
+
+        Assert.That(new ArraySegment<byte>(result, E2StoreWriter.HeaderSize, TestBytes.Length), Is.EquivalentTo(TestBytes));
+    }
+
+    [Test]
+    public async Task WriteEntryAsSnappy_WritingEntryValue_WritesEncodedBytesToStream()
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2StoreWriter sut = new E2StoreWriter(stream);
+
+        await sut.WriteEntryAsSnappy(EntryTypes.CompressedHeader, TestBytes);
+        stream.Position = E2StoreWriter.HeaderSize;
+        using var snappy = new SnappyStream(stream, System.IO.Compression.CompressionMode.Decompress);
+        byte[] buffer = new byte[32];
+
+        Assert.That(() => snappy.Read(buffer), Throws.Nothing);
+    }
+
+    [Test]
+    public async Task WriteEntryAsSnappy_WritingEntryValue_ReturnsCompressedSize()
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2StoreWriter sut = new E2StoreWriter(stream);
+
+        int result = await sut.WriteEntryAsSnappy(EntryTypes.CompressedHeader, TestBytes);
+
+        Assert.That(result, Is.EqualTo(stream.Length));
+    }
+
+    [Test]
+    public async Task ReadEntryValue_ReadingValueBytesOfEntry_ReturnsBytesRead()
+    {
+        string tmpFile = Path.GetTempFileName();
+        E2StoreWriter sut = new E2StoreWriter(File.OpenWrite(tmpFile));
+        await sut.WriteEntry(EntryTypes.Accumulator, TestBytes);
+        sut.Dispose();
+
+        using E2StoreReader reader = new E2StoreReader(tmpFile);
+        (var readBytes, _) = reader.ReadEntryAndDecode(0, buf => buf.ReadAllBytesAsArray(), EntryTypes.Accumulator);
+        Assert.That(readBytes, Is.EquivalentTo(TestBytes));
+        Assert.That(readBytes.Length, Is.EqualTo(TestBytes.Length));
+    }
+
+    [Test]
+    public async Task ReadEntryValueAsSnappy_ReadingValueBytesOfEntry_ReturnsDecompressedBytes()
+    {
+        string tmpFile = Path.GetTempFileName();
+        using E2StoreWriter sut = new E2StoreWriter(File.OpenWrite(tmpFile));
+        MemoryStream compressed = new();
+        using SnappyStream snappy = new SnappyStream(compressed, System.IO.Compression.CompressionMode.Compress);
+        snappy.Write(TestBytes);
+        snappy.Flush();
+        long position = sut.Position;
+        await sut.WriteEntry(EntryTypes.CompressedHeader, compressed.ToArray());
+        sut.Dispose();
+
+        using E2StoreReader reader = new E2StoreReader(tmpFile);
+        (var readBytes, _) = await reader.ReadSnappyCompressedEntryAndDecode(position, buf => buf.ReadAllBytesAsArray(), EntryTypes.CompressedHeader, default);
+        Assert.That(readBytes, Is.EquivalentTo(TestBytes));
+    }
+}