From cde33bd8d0710bdc4464358e49d6864768887ce1 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Fri, 12 Jan 2024 14:50:53 +0000 Subject: [PATCH 1/3] Dispose websocket (#6520) --- src/Nethermind/Nethermind.Sockets/Extensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Sockets/Extensions.cs b/src/Nethermind/Nethermind.Sockets/Extensions.cs index b4d066cb652..daa725f35e2 100644 --- a/src/Nethermind/Nethermind.Sockets/Extensions.cs +++ b/src/Nethermind/Nethermind.Sockets/Extensions.cs @@ -50,8 +50,8 @@ public static void UseWebSocketsModules(this IApplicationBuilder app) if (logger?.IsDebug == true) logger.Info($"Initializing WebSockets for client: '{clientName}'."); - var webSocket = await context.WebSockets.AcceptWebSocketAsync(); - var socketsClient = + using WebSocket webSocket = await context.WebSockets.AcceptWebSocketAsync(); + using ISocketsClient socketsClient = module.CreateClient(webSocket, clientName, context); id = socketsClient.Id; await socketsClient.ReceiveAsync(); From a7bc612bc1cd4fad47400f7a5ab9701f1ca62a0e Mon Sep 17 00:00:00 2001 From: Ahmad Bitar <33181301+smartprogrammer93@users.noreply.github.com> Date: Fri, 12 Jan 2024 19:15:02 +0300 Subject: [PATCH 2/3] [Cleanup] Remove deprecated aspnet core nugets (#6513) --- src/Nethermind/Directory.Packages.props | 2 -- .../Nethermind.Monitoring/Nethermind.Monitoring.csproj | 8 ++++++-- src/Nethermind/Nethermind.Sockets/Extensions.cs | 4 +++- .../Nethermind.Sockets/Nethermind.Sockets.csproj | 5 +++-- src/Nethermind/Nethermind.sln | 4 ++++ 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props index 35a1721c39a..1cee10e2369 100644 --- a/src/Nethermind/Directory.Packages.props +++ b/src/Nethermind/Directory.Packages.props @@ -30,8 +30,6 @@ - - diff --git a/src/Nethermind/Nethermind.Monitoring/Nethermind.Monitoring.csproj b/src/Nethermind/Nethermind.Monitoring/Nethermind.Monitoring.csproj index 015cce13ca3..d8c8a5b1b50 100644 --- a/src/Nethermind/Nethermind.Monitoring/Nethermind.Monitoring.csproj +++ b/src/Nethermind/Nethermind.Monitoring/Nethermind.Monitoring.csproj @@ -1,7 +1,11 @@ - + + + Library + true + + - diff --git a/src/Nethermind/Nethermind.Sockets/Extensions.cs b/src/Nethermind/Nethermind.Sockets/Extensions.cs index daa725f35e2..615963e7c8c 100644 --- a/src/Nethermind/Nethermind.Sockets/Extensions.cs +++ b/src/Nethermind/Nethermind.Sockets/Extensions.cs @@ -4,7 +4,9 @@ using System; using System.Linq; using System.Net.WebSockets; +using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Primitives; using Nethermind.Logging; @@ -23,7 +25,7 @@ public static void UseWebSocketsModules(this IApplicationBuilder app) logger = scope.ServiceProvider.GetService()?.GetClassLogger(); } - app.Use(async (context, next) => + app.Use(async (HttpContext context, Func next) => { string id = string.Empty; string clientName = string.Empty; diff --git a/src/Nethermind/Nethermind.Sockets/Nethermind.Sockets.csproj b/src/Nethermind/Nethermind.Sockets/Nethermind.Sockets.csproj index c81c056fa1c..bbcf655c05a 100644 --- a/src/Nethermind/Nethermind.Sockets/Nethermind.Sockets.csproj +++ b/src/Nethermind/Nethermind.Sockets/Nethermind.Sockets.csproj @@ -1,7 +1,9 @@ - + enable + Library + true @@ -11,7 +13,6 @@ - diff --git a/src/Nethermind/Nethermind.sln b/src/Nethermind/Nethermind.sln index f97c5206a4f..395531b2b64 100644 --- a/src/Nethermind/Nethermind.sln +++ b/src/Nethermind/Nethermind.sln @@ -206,6 +206,10 @@ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nethermind.UPnP.Plugin", "Nethermind.UPnP.Plugin\Nethermind.UPnP.Plugin.csproj", "{48E50409-26FE-4FD8-AF6E-2A0E79F794CE}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{0E0F75A7-A03F-4D66-910B-17DB659A33E3}" + ProjectSection(SolutionItems) = preProject + Directory.Build.props = Directory.Build.props + Directory.Packages.props = Directory.Packages.props + EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nethermind.Serialization.Ssz.Test", "Nethermind.Serialization.Ssz.Test\Nethermind.Serialization.Ssz.Test.csproj", "{E1E7BEFC-52C0-49ED-B0A7-CB8C3250D120}" EndProject From a3a8cc2ac28b6f88afe9bbbbdd4cab5d904d921c Mon Sep 17 00:00:00 2001 From: Arimionim <31865930+Arimionim@users.noreply.github.com> Date: Fri, 12 Jan 2024 20:44:00 +0000 Subject: [PATCH 3/3] Do not throw IOException when file is already downloaded (#6346) Co-authored-by: Ben Adams --- .../InitDatabaseSnapshot.cs | 110 +++++++++++++----- 1 file changed, 81 insertions(+), 29 deletions(-) diff --git a/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs b/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs index 59427d94c00..4322f7554ed 100644 --- a/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs +++ b/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs @@ -43,55 +43,76 @@ public override async Task Execute(CancellationToken cancellationToken) private async Task InitDbFromSnapshot(CancellationToken cancellationToken) { - string dbPath = _api.Config().BaseDbPath; - if (Path.Exists(dbPath)) - { - if (_logger.IsInfo) - _logger.Info($"Database already exists at {dbPath}. Skipping snapshot initialization."); - return; - } ISnapshotConfig snapshotConfig = _api.Config(); + string dbPath = _api.Config().BaseDbPath; string snapshotUrl = snapshotConfig.DownloadUrl ?? throw new InvalidOperationException("Snapshot download URL is not configured"); - string snapshotFileName = Path.Combine(snapshotConfig.SnapshotDirectory, snapshotConfig.SnapshotFileName); - Directory.CreateDirectory(snapshotConfig.SnapshotDirectory); - while (true) + if (Path.Exists(dbPath)) { - try + if (GetCheckpoint(snapshotConfig) < Stage.Extracted) { - await DownloadSnapshotTo(snapshotUrl, snapshotFileName, cancellationToken); - break; + if (_logger.IsInfo) + _logger.Info($"Extracting wasn't finished last time, restarting it. To interrupt press Ctrl^C"); + // Wait few seconds if user wants to stop reinitialization + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + Directory.Delete(dbPath, true); } - catch (IOException e) + else { - if (_logger.IsError) - _logger.Error($"Snapshot download failed. Retrying in 5 seconds. Error: {e}"); - await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + if (_logger.IsInfo) + _logger.Info($"Database already exists at {dbPath}. Interrupting"); + + return; } - cancellationToken.ThrowIfCancellationRequested(); } - if (snapshotConfig.Checksum is not null) + Directory.CreateDirectory(snapshotConfig.SnapshotDirectory); + + if (GetCheckpoint(snapshotConfig) < Stage.Downloaded) { - bool isChecksumValid = await VerifyChecksum(snapshotFileName, snapshotConfig.Checksum, cancellationToken); - if (!isChecksumValid) + while (true) { - if (_logger.IsError) - _logger.Error("Snapshot checksum verification failed. Aborting, but will continue running."); - return; + try + { + await DownloadSnapshotTo(snapshotUrl, snapshotFileName, cancellationToken); + break; + } + catch (IOException e) + { + if (_logger.IsError) + _logger.Error($"Snapshot download failed. Retrying in 5 seconds. Error: {e}"); + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + } + cancellationToken.ThrowIfCancellationRequested(); } - - if (_logger.IsInfo) - _logger.Info("Snapshot checksum verified."); + SetCheckpoint(snapshotConfig, Stage.Downloaded); } - else if (_logger.IsWarn) - _logger.Warn("Snapshot checksum is not configured"); + if (GetCheckpoint(snapshotConfig) < Stage.Verified) + { + if (snapshotConfig.Checksum is not null) + { + bool isChecksumValid = await VerifyChecksum(snapshotFileName, snapshotConfig.Checksum, cancellationToken); + if (!isChecksumValid) + { + if (_logger.IsError) + _logger.Error("Snapshot checksum verification failed. Aborting, but will continue running."); + return; + } + + if (_logger.IsInfo) + _logger.Info("Snapshot checksum verified."); + } + else if (_logger.IsWarn) + _logger.Warn("Snapshot checksum is not configured"); + SetCheckpoint(snapshotConfig, Stage.Verified); + } await ExtractSnapshotTo(snapshotFileName, dbPath, cancellationToken); + SetCheckpoint(snapshotConfig, Stage.Extracted); if (_logger.IsInfo) { @@ -100,6 +121,8 @@ private async Task InitDbFromSnapshot(CancellationToken cancellationToken) } File.Delete(snapshotFileName); + + SetCheckpoint(snapshotConfig, Stage.End); } private async Task DownloadSnapshotTo( @@ -187,4 +210,33 @@ private Task ExtractSnapshotTo(string snapshotPath, string dbPath, CancellationT ZipFile.ExtractToDirectory(snapshotPath, dbPath); }, cancellationToken); + + private enum Stage + { + Start, + Downloaded, + Verified, + Extracted, + End, + } + + private static void SetCheckpoint(ISnapshotConfig snapshotConfig, Stage stage) + { + string checkpointPath = Path.Combine(snapshotConfig.SnapshotDirectory, "checkpoint" + "_" + snapshotConfig.SnapshotFileName); + File.WriteAllText(checkpointPath, stage.ToString()); + } + + private static Stage GetCheckpoint(ISnapshotConfig snapshotConfig) + { + string checkpointPath = Path.Combine(snapshotConfig.SnapshotDirectory, "checkpoint" + "_" + snapshotConfig.SnapshotFileName); + if (File.Exists(checkpointPath)) + { + string stringStage = File.ReadAllText(checkpointPath); + return (Stage)Enum.Parse(typeof(Stage), stringStage); + } + else + { + return Stage.Start; + } + } }