Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce RAM consumption for ForceProcessing requests #5090

Merged
merged 7 commits into from
Jan 12, 2023
Merged

Reduce RAM consumption for ForceProcessing requests #5090

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5c01eb9
iterative to fixed test change
OlegJakushkin Jan 3, 2023
5ed60ab
minor fixes
OlegJakushkin Jan 3, 2023
986f378
Merge remote-tracking branch 'origin/master' into fix/issue-4557
OlegJakushkin Jan 4, 2023
651dfe1
Clean up
OlegJakushkin Jan 4, 2023
98b20c6
Merge remote-tracking branch 'origin/master' into fix/issue-4557
OlegJakushkin Jan 4, 2023
eec3173
Cleanup complience
OlegJakushkin Jan 4, 2023
3dfc556
Automated cleanup compliance
OlegJakushkin Jan 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,9 @@ private void FireProcessingQueueEmpty()

if (!shouldProcess)
{
if (_logger.IsDebug) _logger.Debug($"Skipped processing of {suggestedBlock.ToString(Block.Format.FullHashAndNumber)}, Head = {_blockTree.Head?.Header?.ToString(BlockHeader.Format.Short)}, total diff = {totalDifficulty}, head total diff = {_blockTree.Head?.TotalDifficulty}");
if (_logger.IsDebug)
_logger.Debug(
$"Skipped processing of {suggestedBlock.ToString(Block.Format.FullHashAndNumber)}, Head = {_blockTree.Head?.Header?.ToString(BlockHeader.Format.Short)}, total diff = {totalDifficulty}, head total diff = {_blockTree.Head?.TotalDifficulty}");
return null;
}

Expand Down Expand Up @@ -499,7 +501,7 @@ private void PrepareBlocksToProcess(Block suggestedBlock, ProcessingOptions opti
List<Block> blocksToProcess = processingBranch.BlocksToProcess;
if (options.ContainsFlag(ProcessingOptions.ForceProcessing))
{
processingBranch.Blocks.Clear();
processingBranch.Blocks.Clear(); // TODO: investigate why if we clear it all we need to collect and iterate on all the blocks in PrepareProcessingBranch?
OlegJakushkin marked this conversation as resolved.
Show resolved Hide resolved
blocksToProcess.Add(suggestedBlock);
}
else
Expand Down Expand Up @@ -537,9 +539,15 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin
bool suggestedBlockIsPostMerge = suggestedBlock.IsPostMerge;

Block toBeProcessed = suggestedBlock;
long iterations = 0;
do
{
blocksToBeAddedToMain.Add(toBeProcessed);
iterations++;
if (!options.ContainsFlag(ProcessingOptions.ForceProcessing))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we create a variable for that?

{
blocksToBeAddedToMain.Add(toBeProcessed);
}

if (_logger.IsTrace)
_logger.Trace(
$"To be processed (of {suggestedBlock.ToString(Block.Format.Short)}) is {toBeProcessed?.ToString(Block.Format.Short)}");
Expand Down Expand Up @@ -586,7 +594,7 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin
// If we hit this condition, it means that something is wrong in MultiSyncModeSelector.
// MultiSyncModeSelector switched to full sync when it shouldn't
// In this case, it is better to stop searching for more blocks and failed during the processing than trying to build a branch up to the genesis point
if (blocksToBeAddedToMain.Count > MaxBlocksDuringFastSyncTransition)
if (iterations > MaxBlocksDuringFastSyncTransition)
{
if (_logger.IsWarn) _logger.Warn($"Too long branch to be processed during fast sync transition. Current block to be processed {toBeProcessed}, StateRoot: {toBeProcessed?.StateRoot}");
break;
Expand All @@ -610,11 +618,13 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin
// we need to dig deeper to go all the way to the false (reorg boundary) head
// otherwise some nodes would be missing
bool notFoundTheBranchingPointYet = !_blockTree.IsMainChain(branchingPoint.Hash!);
bool notReachedTheReorgBoundary = branchingPoint.Number > (_blockTree.Head?.Header.Number ?? 0);
bool notReachedTheReorgBoundary = (options & ProcessingOptions.Trace) == ProcessingOptions.Trace
OlegJakushkin marked this conversation as resolved.
Show resolved Hide resolved
&& (branchingPoint.Number > (_blockTree.Head?.Header.Number ?? 0));
Comment on lines +621 to +622
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

better naming needed, now notReachedTheReorgBoundary is actually trace Or notReachedTheReorgBoundary

preMergeFinishBranchingCondition = (notFoundTheBranchingPointYet || notReachedTheReorgBoundary);
if (_logger.IsTrace)
_logger.Trace(
$" Current branching point: {branchingPoint.Number}, {branchingPoint.Hash} TD: {branchingPoint.TotalDifficulty} Processing conditions notFoundTheBranchingPointYet {notFoundTheBranchingPointYet}, notReachedTheReorgBoundary: {notReachedTheReorgBoundary}, suggestedBlockIsPostMerge {suggestedBlockIsPostMerge}");

} while (preMergeFinishBranchingCondition);

if (branchingPoint is not null && branchingPoint.Hash != _blockTree.Head?.Hash)
Expand Down