Merge branch 'master' into feature/chiseled

NethermindEth · Jan 13, 2024 · aaa833b · aaa833b
2 parents 720bc5b + a3a8cc2
commit aaa833b
Show file tree

Hide file tree

Showing 6 changed files with 99 additions and 38 deletions.
diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props
@@ -30,8 +30,6 @@
     <PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="8.0.0" />
     <PackageVersion Include="Microsoft.AspNetCore.DataProtection.Abstractions" Version="8.0.0" />
     <PackageVersion Include="Microsoft.AspNetCore.DataProtection.Extensions" Version="8.0.0" />
-    <PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
-    <PackageVersion Include="Microsoft.AspNetCore.WebSockets" Version="2.2.1" />
     <PackageVersion Include="Microsoft.ClearScript.V8" Version="7.4.4" />
     <PackageVersion Include="Microsoft.ClearScript.V8.Native.linux-arm64" Version="7.4.4" />
     <PackageVersion Include="Microsoft.ClearScript.V8.Native.linux-x64" Version="7.4.4" />

diff --git a/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs b/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs
@@ -43,55 +43,76 @@ public override async Task Execute(CancellationToken cancellationToken)
 
     private async Task InitDbFromSnapshot(CancellationToken cancellationToken)
     {
-        string dbPath = _api.Config<IInitConfig>().BaseDbPath;
-        if (Path.Exists(dbPath))
-        {
-            if (_logger.IsInfo)
-                _logger.Info($"Database already exists at {dbPath}. Skipping snapshot initialization.");
-            return;
-        }
 
         ISnapshotConfig snapshotConfig = _api.Config<ISnapshotConfig>();
+        string dbPath = _api.Config<IInitConfig>().BaseDbPath;
         string snapshotUrl = snapshotConfig.DownloadUrl ??
                              throw new InvalidOperationException("Snapshot download URL is not configured");
-
         string snapshotFileName = Path.Combine(snapshotConfig.SnapshotDirectory, snapshotConfig.SnapshotFileName);
-        Directory.CreateDirectory(snapshotConfig.SnapshotDirectory);
 
-        while (true)
+        if (Path.Exists(dbPath))
         {
-            try
+            if (GetCheckpoint(snapshotConfig) < Stage.Extracted)
             {
-                await DownloadSnapshotTo(snapshotUrl, snapshotFileName, cancellationToken);
-                break;
+                if (_logger.IsInfo)
+                    _logger.Info($"Extracting wasn't finished last time, restarting it. To interrupt press Ctrl^C");
+                // Wait few seconds if user wants to stop reinitialization
+                await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
+                Directory.Delete(dbPath, true);
             }
-            catch (IOException e)
+            else
             {
-                if (_logger.IsError)
-                    _logger.Error($"Snapshot download failed. Retrying in 5 seconds. Error: {e}");
-                await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
+                if (_logger.IsInfo)
+                    _logger.Info($"Database already exists at {dbPath}. Interrupting");
+
+                return;
             }
-            cancellationToken.ThrowIfCancellationRequested();
         }
 
-        if (snapshotConfig.Checksum is not null)
+        Directory.CreateDirectory(snapshotConfig.SnapshotDirectory);
+
+        if (GetCheckpoint(snapshotConfig) < Stage.Downloaded)
         {
-            bool isChecksumValid = await VerifyChecksum(snapshotFileName, snapshotConfig.Checksum, cancellationToken);
-            if (!isChecksumValid)
+            while (true)
             {
-                if (_logger.IsError)
-                    _logger.Error("Snapshot checksum verification failed. Aborting, but will continue running.");
-                return;
+                try
+                {
+                    await DownloadSnapshotTo(snapshotUrl, snapshotFileName, cancellationToken);
+                    break;
+                }
+                catch (IOException e)
+                {
+                    if (_logger.IsError)
+                        _logger.Error($"Snapshot download failed. Retrying in 5 seconds. Error: {e}");
+                    await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
+                }
+                cancellationToken.ThrowIfCancellationRequested();
             }
-
-            if (_logger.IsInfo)
-                _logger.Info("Snapshot checksum verified.");
+            SetCheckpoint(snapshotConfig, Stage.Downloaded);
         }
-        else if (_logger.IsWarn)
-            _logger.Warn("Snapshot checksum is not configured");
 
+        if (GetCheckpoint(snapshotConfig) < Stage.Verified)
+        {
+            if (snapshotConfig.Checksum is not null)
+            {
+                bool isChecksumValid = await VerifyChecksum(snapshotFileName, snapshotConfig.Checksum, cancellationToken);
+                if (!isChecksumValid)
+                {
+                    if (_logger.IsError)
+                        _logger.Error("Snapshot checksum verification failed. Aborting, but will continue running.");
+                    return;
+                }
+
+                if (_logger.IsInfo)
+                    _logger.Info("Snapshot checksum verified.");
+            }
+            else if (_logger.IsWarn)
+                _logger.Warn("Snapshot checksum is not configured");
+            SetCheckpoint(snapshotConfig, Stage.Verified);
+        }
 
         await ExtractSnapshotTo(snapshotFileName, dbPath, cancellationToken);
+        SetCheckpoint(snapshotConfig, Stage.Extracted);
 
         if (_logger.IsInfo)
         {
@@ -100,6 +121,8 @@ private async Task InitDbFromSnapshot(CancellationToken cancellationToken)
         }
 
         File.Delete(snapshotFileName);
+
+        SetCheckpoint(snapshotConfig, Stage.End);
     }
 
     private async Task DownloadSnapshotTo(
@@ -187,4 +210,33 @@ private Task ExtractSnapshotTo(string snapshotPath, string dbPath, CancellationT
 
             ZipFile.ExtractToDirectory(snapshotPath, dbPath);
         }, cancellationToken);
+
+    private enum Stage
+    {
+        Start,
+        Downloaded,
+        Verified,
+        Extracted,
+        End,
+    }
+
+    private static void SetCheckpoint(ISnapshotConfig snapshotConfig, Stage stage)
+    {
+        string checkpointPath = Path.Combine(snapshotConfig.SnapshotDirectory, "checkpoint" + "_" + snapshotConfig.SnapshotFileName);
+        File.WriteAllText(checkpointPath, stage.ToString());
+    }
+
+    private static Stage GetCheckpoint(ISnapshotConfig snapshotConfig)
+    {
+        string checkpointPath = Path.Combine(snapshotConfig.SnapshotDirectory, "checkpoint" + "_" + snapshotConfig.SnapshotFileName);
+        if (File.Exists(checkpointPath))
+        {
+            string stringStage = File.ReadAllText(checkpointPath);
+            return (Stage)Enum.Parse(typeof(Stage), stringStage);
+        }
+        else
+        {
+            return Stage.Start;
+        }
+    }
 }
diff --git a/src/Nethermind/Nethermind.Monitoring/Nethermind.Monitoring.csproj b/src/Nethermind/Nethermind.Monitoring/Nethermind.Monitoring.csproj
@@ -1,7 +1,11 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
+  <PropertyGroup>
+    <OutputType>Library</OutputType>
+    <NoDefaultLaunchSettingsFile>true</NoDefaultLaunchSettingsFile>
+  </PropertyGroup>
+
   <ItemGroup>
-    <PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
     <PackageReference Include="prometheus-net" />
     <PackageReference Include="prometheus-net.AspNetCore" />

diff --git a/src/Nethermind/Nethermind.Sockets/Extensions.cs b/src/Nethermind/Nethermind.Sockets/Extensions.cs
@@ -4,7 +4,9 @@
 using System;
 using System.Linq;
 using System.Net.WebSockets;
+using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Primitives;
 using Nethermind.Logging;
@@ -23,7 +25,7 @@ public static void UseWebSocketsModules(this IApplicationBuilder app)
                 logger = scope.ServiceProvider.GetService<ILogManager>()?.GetClassLogger();
             }
 
-            app.Use(async (context, next) =>
+            app.Use(async (HttpContext context, Func<Task> next) =>
             {
                 string id = string.Empty;
                 string clientName = string.Empty;
@@ -50,8 +52,8 @@ public static void UseWebSocketsModules(this IApplicationBuilder app)
 
                     if (logger?.IsDebug == true) logger.Info($"Initializing WebSockets for client: '{clientName}'.");
 
-                    var webSocket = await context.WebSockets.AcceptWebSocketAsync();
-                    var socketsClient =
+                    using WebSocket webSocket = await context.WebSockets.AcceptWebSocketAsync();
+                    using ISocketsClient socketsClient =
                         module.CreateClient(webSocket, clientName, context);
                     id = socketsClient.Id;
                     await socketsClient.ReceiveAsync();

diff --git a/src/Nethermind/Nethermind.Sockets/Nethermind.Sockets.csproj b/src/Nethermind/Nethermind.Sockets/Nethermind.Sockets.csproj
@@ -1,7 +1,9 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
     <Nullable>enable</Nullable>
+    <OutputType>Library</OutputType>
+    <NoDefaultLaunchSettingsFile>true</NoDefaultLaunchSettingsFile>
   </PropertyGroup>
 
   <ItemGroup>
@@ -11,7 +13,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.AspNetCore.WebSockets" />
     <PackageReference Include="System.Text.Encodings.Web" />
   </ItemGroup>
 

diff --git a/src/Nethermind/Nethermind.sln b/src/Nethermind/Nethermind.sln
@@ -206,6 +206,10 @@ EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nethermind.UPnP.Plugin", "Nethermind.UPnP.Plugin\Nethermind.UPnP.Plugin.csproj", "{48E50409-26FE-4FD8-AF6E-2A0E79F794CE}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{0E0F75A7-A03F-4D66-910B-17DB659A33E3}"
+	ProjectSection(SolutionItems) = preProject
+		Directory.Build.props = Directory.Build.props
+		Directory.Packages.props = Directory.Packages.props
+	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nethermind.Serialization.Ssz.Test", "Nethermind.Serialization.Ssz.Test\Nethermind.Serialization.Ssz.Test.csproj", "{E1E7BEFC-52C0-49ED-B0A7-CB8C3250D120}"
 EndProject