stagedsync: add dbg.SaveHeapProfileNearOOM to headers stage (erigonte…

…ch#11549)

relates to:
erigontech#10734
erigontech#11387

restart Erigon with `SAVE_HEAP_PROFILE = true` env variable
wait until we reach 45% or more alloc in stage_headers when
"noProgressCounter >= 5" or "Rejected header marked as bad"

erigon-lib/common/dbg/experiments.go

@@ -18,16 +18,20 @@ package dbg
 
 import (
 	"os"
+	"path/filepath"
 	"runtime"
+	"runtime/pprof"
 	"strconv"
 	"sync"
 	"time"
 
 	"github.com/erigontech/erigon-lib/log/v3"
+	"github.com/erigontech/erigon-lib/mmap"
 )
 
 var (
 	doMemstat        = EnvBool("NO_MEMSTAT", true)
+	saveHeapProfile  = EnvBool("SAVE_HEAP_PROFILE", false)
 	writeMap         = EnvBool("WRITE_MAP", false)
 	noSync           = EnvBool("NO_SYNC", false)
 	mdbxReadahead    = EnvBool("MDBX_READAHEAD", false)
@@ -241,3 +245,27 @@ func LogHashMismatchReason() bool {
 	})
 	return logHashMismatchReason
 }
+
+func SaveHeapProfileNearOOM() {
+	if !saveHeapProfile {
+		return
+	}
+
+	var m runtime.MemStats
+	ReadMemStats(&m)
+	if m.Alloc < (mmap.TotalMemory()/100)*45 {
+		return
+	}
+
+	// above 45%
+	filePath := filepath.Join(os.TempDir(), "erigon-mem.prof")
+	log.Info("[Experiment] saving heap profile as near OOM", "alloc", m.Alloc, "filePath", filePath)
+
+	f, _ := os.Create(filePath)
+	defer func() {
+		_ = f.Close()
+	}()
+
+	runtime.GC()
+	_ = pprof.WriteHeapProfile(f)
+}

eth/stagedsync/stage_headers.go

@@ -296,6 +296,7 @@ Loop:
 				logger.Info("Req/resp stats", "req", stats.Requests, "reqMin", stats.ReqMinBlock, "reqMax", stats.ReqMaxBlock,
 					"skel", stats.SkeletonRequests, "skelMin", stats.SkeletonReqMinBlock, "skelMax", stats.SkeletonReqMaxBlock,
 					"resp", stats.Responses, "respMin", stats.RespMinBlock, "respMax", stats.RespMaxBlock, "dups", stats.Duplicates, "alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
+				dbg.SaveHeapProfileNearOOM()
 				cfg.hd.LogAnchorState()
 				if wasProgress {
 					logger.Warn("Looks like chain is not progressing, moving to the next stage")

turbo/stages/headerdownload/header_algos.go

@@ -122,7 +122,8 @@ func (hd *HeaderDownload) SingleHeaderAsSegment(headerRaw []byte, header *types.
 	headerHash := types.RawRlpHash(headerRaw)
 	if _, bad := hd.badHeaders[headerHash]; bad {
 		hd.stats.RejectedBadHeaders++
-		hd.logger.Warn("[downloader] Rejected header marked as bad", "hash", headerHash, "height", header.Number.Uint64())
+		dbg.SaveHeapProfileNearOOM()
+		hd.logger.Warn("[downloader] SingleHeaderAsSegment: Rejected header marked as bad", "hash", headerHash, "height", header.Number.Uint64())
 		return nil, BadBlockPenalty, nil
 	}
 	if penalizePoSBlocks && header.Difficulty.Sign() == 0 {
@@ -538,7 +539,8 @@ func (hd *HeaderDownload) InsertHeader(hf FeedHeaderFunc, terminalTotalDifficult
 			hd.removeUpwards(link)
 			dataflow.HeaderDownloadStates.AddChange(link.blockHeight, dataflow.HeaderBad)
 			hd.stats.RejectedBadHeaders++
-			hd.logger.Warn("[downloader] Rejected header marked as bad", "hash", link.hash, "height", link.blockHeight)
+			dbg.SaveHeapProfileNearOOM()
+			hd.logger.Warn("[downloader] InsertHeader: Rejected header marked as bad", "hash", link.hash, "height", link.blockHeight)
 			return true, false, 0, lastTime, nil
 		}
 		if !link.verified {