From b8d919bee43ecb147b821d24f8434dfe772dc0c0 Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 22:04:00 +0200 Subject: [PATCH 01/46] save --- cmd/integration/commands/stages.go | 12 +- eth/stagedsync/stage_call_traces.go | 4 +- eth/stagedsync/stage_log_index.go | 6 +- eth/stagedsync/stage_senders.go | 4 +- eth/stagedsync/stage_txlookup.go | 8 +- turbo/cli/default_flags.go | 14 +- turbo/cli/flags.go | 204 +++++++++++----------------- 7 files changed, 93 insertions(+), 159 deletions(-) diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 4ba696e0ef4..246aa764122 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -1016,9 +1016,6 @@ func stageExec(db kv.RwDB, ctx context.Context, logger log.Logger) error { chainConfig, pm := fromdb.ChainConfig(db), fromdb.PruneMode(db) if pruneTo > 0 { pm.History = prune.Distance(s.BlockNumber - pruneTo) - pm.Receipts = prune.Distance(s.BlockNumber - pruneTo) - pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo) - pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo) } syncCfg := ethconfig.Defaults.Sync @@ -1122,9 +1119,6 @@ func stageCustomTrace(db kv.RwDB, ctx context.Context, logger log.Logger) error chainConfig, pm := fromdb.ChainConfig(db), fromdb.PruneMode(db) if pruneTo > 0 { pm.History = prune.Distance(s.BlockNumber - pruneTo) - pm.Receipts = prune.Distance(s.BlockNumber - pruneTo) - pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo) - pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo) } syncCfg := ethconfig.Defaults.Sync @@ -1243,9 +1237,6 @@ func stageTxLookup(db kv.RwDB, ctx context.Context, logger log.Logger) error { s := stage(sync, tx, nil, stages.TxLookup) if pruneTo > 0 { pm.History = prune.Distance(s.BlockNumber - pruneTo) - pm.Receipts = prune.Distance(s.BlockNumber - pruneTo) - pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo) - pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo) } logger.Info("Stage", "name", s.ID, "progress", s.BlockNumber) @@ -1546,8 +1537,7 @@ func stage(st *stagedsync.Sync, tx kv.Tx, db kv.RoDB, stage stages.SyncStage) *s func overrideStorageMode(db kv.RwDB, logger log.Logger) error { chainConfig := fromdb.ChainConfig(db) - pm, err := prune.FromCli(chainConfig.ChainID.Uint64(), pruneFlag, pruneB, pruneH, pruneR, pruneT, pruneC, - pruneHBefore, pruneRBefore, pruneTBefore, pruneCBefore, pruneBBefore, experiments) + pm, err := prune.FromCli(chainConfig.ChainID.Uint64(), pruneB, pruneH, experiments) if err != nil { return err } diff --git a/eth/stagedsync/stage_call_traces.go b/eth/stagedsync/stage_call_traces.go index c0420a178d7..7d48cb5329f 100644 --- a/eth/stagedsync/stage_call_traces.go +++ b/eth/stagedsync/stage_call_traces.go @@ -393,8 +393,8 @@ func PruneCallTraces(s *PruneState, tx kv.RwTx, cfg CallTracesCfg, ctx context.C defer tx.Rollback() } - if cfg.prune.CallTraces.Enabled() { - if err = pruneCallTraces(tx, logPrefix, cfg.prune.CallTraces.PruneTo(s.ForwardProgress), ctx, cfg.tmpdir, logger); err != nil { + if cfg.prune.History.Enabled() { + if err = pruneCallTraces(tx, logPrefix, cfg.prune.History.PruneTo(s.ForwardProgress), ctx, cfg.tmpdir, logger); err != nil { return err } } diff --git a/eth/stagedsync/stage_log_index.go b/eth/stagedsync/stage_log_index.go index b7fd863ff79..9965eb99ced 100644 --- a/eth/stagedsync/stage_log_index.go +++ b/eth/stagedsync/stage_log_index.go @@ -102,7 +102,7 @@ func SpawnLogIndex(s *StageState, tx kv.RwTx, cfg LogIndexCfg, ctx context.Conte } startBlock := s.BlockNumber - pruneTo := cfg.prune.Receipts.PruneTo(endBlock) //endBlock - prune.r.older + pruneTo := cfg.prune.History.PruneTo(endBlock) //endBlock - prune.r.older // if startBlock < pruneTo { // startBlock = pruneTo // } @@ -435,7 +435,7 @@ func pruneOldLogChunks(tx kv.RwTx, bucket string, inMem *etl.Collector, pruneTo // Call pruneLogIndex with the current sync progresses and commit the data to db func PruneLogIndex(s *PruneState, tx kv.RwTx, cfg LogIndexCfg, ctx context.Context, logger log.Logger) (err error) { - if !cfg.prune.Receipts.Enabled() { + if !cfg.prune.History.Enabled() { return nil } logPrefix := s.LogPrefix() @@ -449,7 +449,7 @@ func PruneLogIndex(s *PruneState, tx kv.RwTx, cfg LogIndexCfg, ctx context.Conte defer tx.Rollback() } - pruneTo := cfg.prune.Receipts.PruneTo(s.ForwardProgress) + pruneTo := cfg.prune.History.PruneTo(s.ForwardProgress) if err = pruneLogIndex(logPrefix, tx, cfg.tmpdir, s.PruneProgress, pruneTo, ctx, logger, cfg.depositContract); err != nil { return err } diff --git a/eth/stagedsync/stage_senders.go b/eth/stagedsync/stage_senders.go index 3ed6010cf9a..bcd38b1daec 100644 --- a/eth/stagedsync/stage_senders.go +++ b/eth/stagedsync/stage_senders.go @@ -416,8 +416,8 @@ func PruneSendersStage(s *PruneState, tx kv.RwTx, cfg SendersCfg, ctx context.Co } if cfg.blockReader.FreezingCfg().Enabled { // noop. in this case senders will be deleted by BlockRetire.PruneAncientBlocks after data-freezing. - } else if cfg.prune.TxIndex.Enabled() { - to := cfg.prune.TxIndex.PruneTo(s.ForwardProgress) + } else if cfg.prune.History.Enabled() { + to := cfg.prune.History.PruneTo(s.ForwardProgress) if err = rawdb.PruneTable(tx, kv.Senders, to, ctx, 100); err != nil { return err } diff --git a/eth/stagedsync/stage_txlookup.go b/eth/stagedsync/stage_txlookup.go index 6d017673cf4..3537e9f27d0 100644 --- a/eth/stagedsync/stage_txlookup.go +++ b/eth/stagedsync/stage_txlookup.go @@ -87,8 +87,8 @@ func SpawnTxLookup(s *StageState, tx kv.RwTx, toBlock uint64, cfg TxLookupCfg, c } startBlock := s.BlockNumber - if cfg.prune.TxIndex.Enabled() { - pruneTo := cfg.prune.TxIndex.PruneTo(endBlock) + if cfg.prune.History.Enabled() { + pruneTo := cfg.prune.History.PruneTo(endBlock) if startBlock < pruneTo { startBlock = pruneTo if err = s.UpdatePrune(tx, pruneTo); err != nil { // prune func of this stage will use this value to prevent all ancient blocks traversal @@ -248,8 +248,8 @@ func PruneTxLookup(s *PruneState, tx kv.RwTx, cfg TxLookupCfg, ctx context.Conte var pruneBor bool // Forward stage doesn't write anything before PruneTo point - if cfg.prune.TxIndex.Enabled() { - blockTo = cfg.prune.TxIndex.PruneTo(s.ForwardProgress) + if cfg.prune.History.Enabled() { + blockTo = cfg.prune.History.PruneTo(s.ForwardProgress) pruneBor = true } else if cfg.blockReader.FreezingCfg().Enabled { blockTo = cfg.blockReader.CanPruneTo(s.ForwardProgress) diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go index 9e20cadd900..6d01d6b75ed 100644 --- a/turbo/cli/default_flags.go +++ b/turbo/cli/default_flags.go @@ -43,17 +43,9 @@ var DefaultFlags = []cli.Flag{ &utils.TxPoolLifetimeFlag, &utils.TxPoolTraceSendersFlag, &utils.TxPoolCommitEveryFlag, - &PruneFlag, - &PruneBlocksFlag, - &PruneHistoryFlag, - &PruneReceiptFlag, - &PruneTxIndexFlag, - &PruneCallTracesFlag, - &PruneBlocksBeforeFlag, - &PruneHistoryBeforeFlag, - &PruneReceiptBeforeFlag, - &PruneTxIndexBeforeFlag, - &PruneCallTracesBeforeFlag, + &PruneDistanceFlag, + &PruneBlocksDistanceFlag, + &PruneModeFlag, &BatchSizeFlag, &BodyCacheLimitFlag, &DatabaseVerbosityFlag, diff --git a/turbo/cli/flags.go b/turbo/cli/flags.go index 46e137682af..8d08d399d54 100644 --- a/turbo/cli/flags.go +++ b/turbo/cli/flags.go @@ -82,62 +82,22 @@ var ( Value: kv.ReadersLimit - 128, } - PruneFlag = cli.StringFlag{ - Name: "prune", - Usage: `Choose which ancient data delete from DB: - h - prune history (ChangeSets, HistoryIndices - used by historical state access, like eth_getStorageAt, eth_getBalanceAt, debug_traceTransaction, trace_block, trace_transaction, etc.) - r - prune receipts (Receipts, Logs, LogTopicIndex, LogAddressIndex - used by eth_getLogs and similar RPC methods) - t - prune transaction by it's hash index - c - prune call traces (used by trace_filter method) - Does delete data older than 90K blocks, --prune=h is shortcut for: --prune.h.older=90000. - Similarly, --prune=t is shortcut for: --prune.t.older=90000 and --prune=c is shortcut for: --prune.c.older=90000. - However, --prune=r means to prune receipts before the Beacon Chain genesis (Consensus Layer might need receipts after that). - If an item is NOT on the list - means NO pruning for this data. - Example: --prune=htc`, - Value: "disabled", - } - PruneBlocksFlag = cli.Uint64Flag{ - Name: "prune.b.older", - Usage: `Prune data older than this number of blocks from the tip of the chain (if --prune flag has 'b', then default is 90K)`, - } - PruneHistoryFlag = cli.Uint64Flag{ - Name: "prune.h.older", - Usage: `Prune data older than this number of blocks from the tip of the chain (if --prune flag has 'h', then default is 90K)`, - } - PruneReceiptFlag = cli.Uint64Flag{ - Name: "prune.r.older", - Usage: `Prune data older than this number of blocks from the tip of the chain`, - } - PruneTxIndexFlag = cli.Uint64Flag{ - Name: "prune.t.older", - Usage: `Prune data older than this number of blocks from the tip of the chain (if --prune flag has 't', then default is 90K)`, - } - PruneCallTracesFlag = cli.Uint64Flag{ - Name: "prune.c.older", - Usage: `Prune data older than this number of blocks from the tip of the chain (if --prune flag has 'c', then default is 90K)`, - } - - PruneHistoryBeforeFlag = cli.Uint64Flag{ - Name: "prune.h.before", - Usage: `Prune data before this block`, - } - PruneReceiptBeforeFlag = cli.Uint64Flag{ - Name: "prune.r.before", - Usage: `Prune data before this block`, - } - PruneTxIndexBeforeFlag = cli.Uint64Flag{ - Name: "prune.t.before", - Usage: `Prune data before this block`, - } - PruneCallTracesBeforeFlag = cli.Uint64Flag{ - Name: "prune.c.before", - Usage: `Prune data before this block`, - } - PruneBlocksBeforeFlag = cli.Uint64Flag{ - Name: "prune.b.before", - Usage: `Prune data before this block`, + PruneModeFlag = cli.StringFlag{ + Name: "prune.mode", + Usage: `Choose a pruning preset to run onto. Avaiable values: "archive","full","minimal". + Archive: Keep the entire indexed database, aka. no pruning. (Pruning is flexible), + Full: Keep only blocks and latest state (Pruning is not flexible) + Minimal: Keep only latest state (Pruning is not flexible)`, + Value: "archive", + } + PruneDistanceFlag = cli.Uint64Flag{ + Name: "prune.distance", + Usage: `Keep state history for the latest N blocks (default: everything)`, + } + PruneBlocksDistanceFlag = cli.Uint64Flag{ + Name: "prune.distance.blocks", + Usage: `Keep block history for the latest N blocks (default: everything)`, } - ExperimentsFlag = cli.StringFlag{ Name: "experiments", Usage: `Enable some experimental stages: @@ -312,29 +272,38 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config, logger log. if cfg.Genesis != nil { chainId = cfg.Genesis.Config.ChainID.Uint64() } - minimal := ctx.String(PruneFlag.Name) == "minimal" - pruneFlagString := ctx.String(PruneFlag.Name) - if minimal { - pruneFlagString = "htrcb" + // Sanitize prune flag + if ctx.String(PruneModeFlag.Name) != "archive" && (ctx.IsSet(PruneBlocksDistanceFlag.Name) || ctx.IsSet(PruneDistanceFlag.Name)) { + utils.Fatalf(fmt.Sprintf("error: --prune.distance and --prune.distance.blocks are only allowed with --prune.mode=archive")) + } + distance := ctx.Uint64(PruneDistanceFlag.Name) + blockDistance := ctx.Uint64(PruneBlocksDistanceFlag.Name) + + if !ctx.IsSet(PruneBlocksDistanceFlag.Name) { + blockDistance = math.MaxUint64 + } + if !ctx.IsSet(PruneDistanceFlag.Name) { + distance = math.MaxUint64 } mode, err := prune.FromCli( chainId, - pruneFlagString, - ctx.Uint64(PruneBlocksFlag.Name), - ctx.Uint64(PruneHistoryFlag.Name), - ctx.Uint64(PruneReceiptFlag.Name), - ctx.Uint64(PruneTxIndexFlag.Name), - ctx.Uint64(PruneCallTracesFlag.Name), - ctx.Uint64(PruneHistoryBeforeFlag.Name), - ctx.Uint64(PruneReceiptBeforeFlag.Name), - ctx.Uint64(PruneTxIndexBeforeFlag.Name), - ctx.Uint64(PruneCallTracesBeforeFlag.Name), - ctx.Uint64(PruneBlocksBeforeFlag.Name), + distance, + blockDistance, libcommon.CliString2Array(ctx.String(ExperimentsFlag.Name)), ) if err != nil { utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) } + // Full mode prunes all but the latest state + if ctx.String(PruneModeFlag.Name) == "full" { + mode.Blocks = prune.Distance(math.MaxUint64) + mode.History = prune.Distance(0) + } + // Minimal mode prunes all but the latest state including blocks + if ctx.String(PruneModeFlag.Name) == "minimal" { + mode.Blocks = prune.Distance(2048) // 2048 is just some blocks to allow reorgs + mode.History = prune.Distance(0) + } if err != nil { utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) @@ -357,15 +326,6 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config, logger log. etl.BufferOptimalSize = *size } - if minimal { - // Prune them all. - cfg.Prune.Blocks = prune.Before(math.MaxUint64) - cfg.Prune.History = prune.Before(math.MaxUint64) - cfg.Prune.Receipts = prune.Before(math.MaxUint64) - cfg.Prune.TxIndex = prune.Before(math.MaxUint64) - cfg.Prune.CallTraces = prune.Before(math.MaxUint64) - } - cfg.StateStream = !ctx.Bool(StateStreamDisableFlag.Name) if ctx.String(BodyCacheLimitFlag.Name) != "" { err := cfg.Sync.BodyCacheLimit.UnmarshalText([]byte(ctx.String(BodyCacheLimitFlag.Name))) @@ -433,56 +393,48 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config, logger log. } func ApplyFlagsForEthConfigCobra(f *pflag.FlagSet, cfg *ethconfig.Config) { - if v := f.String(PruneFlag.Name, PruneFlag.Value, PruneFlag.Usage); v != nil { - var experiments []string - if exp := f.StringSlice(ExperimentsFlag.Name, nil, ExperimentsFlag.Usage); exp != nil { - experiments = *exp - } - var exactB, exactH, exactR, exactT, exactC uint64 - if v := f.Uint64(PruneBlocksFlag.Name, PruneBlocksFlag.Value, PruneBlocksFlag.Usage); v != nil { - exactB = *v - } - if v := f.Uint64(PruneHistoryFlag.Name, PruneHistoryFlag.Value, PruneHistoryFlag.Usage); v != nil { - exactH = *v - } - if v := f.Uint64(PruneReceiptFlag.Name, PruneReceiptFlag.Value, PruneReceiptFlag.Usage); v != nil { - exactR = *v - } - if v := f.Uint64(PruneTxIndexFlag.Name, PruneTxIndexFlag.Value, PruneTxIndexFlag.Usage); v != nil { - exactT = *v - } - if v := f.Uint64(PruneCallTracesFlag.Name, PruneCallTracesFlag.Value, PruneCallTracesFlag.Usage); v != nil { - exactC = *v - } + pruneMode := f.String(PruneModeFlag.Name, PruneModeFlag.DefaultText, PruneModeFlag.Usage) + pruneBlockDistance := f.Uint64(PruneBlocksDistanceFlag.Name, PruneBlocksDistanceFlag.Value, PruneBlocksDistanceFlag.Usage) + pruneDistance := f.Uint64(PruneDistanceFlag.Name, PruneDistanceFlag.Value, PruneDistanceFlag.Usage) - var beforeB, beforeH, beforeR, beforeT, beforeC uint64 - if v := f.Uint64(PruneBlocksBeforeFlag.Name, PruneBlocksBeforeFlag.Value, PruneBlocksBeforeFlag.Usage); v != nil { - beforeB = *v - } - if v := f.Uint64(PruneHistoryBeforeFlag.Name, PruneHistoryBeforeFlag.Value, PruneHistoryBeforeFlag.Usage); v != nil { - beforeH = *v - } - if v := f.Uint64(PruneReceiptBeforeFlag.Name, PruneReceiptBeforeFlag.Value, PruneReceiptBeforeFlag.Usage); v != nil { - beforeR = *v - } - if v := f.Uint64(PruneTxIndexBeforeFlag.Name, PruneTxIndexBeforeFlag.Value, PruneTxIndexBeforeFlag.Usage); v != nil { - beforeT = *v - } - if v := f.Uint64(PruneCallTracesBeforeFlag.Name, PruneCallTracesBeforeFlag.Value, PruneCallTracesBeforeFlag.Usage); v != nil { - beforeC = *v - } - - chainId := cfg.NetworkID - if cfg.Genesis != nil { - chainId = cfg.Genesis.Config.ChainID.Uint64() - } + chainId := cfg.NetworkID + if *pruneMode != "archive" && (pruneBlockDistance != nil || pruneDistance != nil) { + utils.Fatalf(fmt.Sprintf("error: --prune.distance and --prune.distance.blocks are only allowed with --prune.mode=archive")) + } + var distance, blockDistance uint64 = math.MaxUint64, math.MaxUint64 + if pruneBlockDistance != nil { + blockDistance = *pruneBlockDistance + } + if pruneDistance != nil { + distance = *pruneDistance + } - mode, err := prune.FromCli(chainId, *v, exactB, exactH, exactR, exactT, exactC, beforeH, beforeR, beforeT, beforeC, beforeB, experiments) - if err != nil { - utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) - } - cfg.Prune = mode + experiments := f.String(ExperimentsFlag.Name, ExperimentsFlag.Value, ExperimentsFlag.Usage) + experimentsVal := "" + if experiments != nil { + experimentsVal = *experiments + } + mode, err := prune.FromCli( + chainId, + distance, + blockDistance, + libcommon.CliString2Array(experimentsVal), + ) + if err != nil { + utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) } + // Full mode prunes all but the latest state + if *pruneMode == "full" { + mode.Blocks = prune.Distance(math.MaxUint64) + mode.History = prune.Distance(0) + } + // Minimal mode prunes all but the latest state including blocks + if *pruneMode == "minimal" { + mode.Blocks = prune.Distance(2048) // 2048 is just some blocks to allow reorgs + mode.History = prune.Distance(0) + } + cfg.Prune = mode + if v := f.String(BatchSizeFlag.Name, BatchSizeFlag.Value, BatchSizeFlag.Usage); v != nil { err := cfg.BatchSize.UnmarshalText([]byte(*v)) if err != nil { From 8ba2cbaa2d7788b3e192fb9b5adcb77f183ca2e2 Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 22:05:34 +0200 Subject: [PATCH 02/46] save --- ethdb/prune/storage_mode.go | 161 ++----------------------------- ethdb/prune/storage_mode_test.go | 32 +----- 2 files changed, 9 insertions(+), 184 deletions(-) diff --git a/ethdb/prune/storage_mode.go b/ethdb/prune/storage_mode.go index 8ef85275ee6..250d27987a3 100644 --- a/ethdb/prune/storage_mode.go +++ b/ethdb/prune/storage_mode.go @@ -17,7 +17,6 @@ package prune import ( - "bytes" "encoding/binary" "errors" "fmt" @@ -26,16 +25,12 @@ import ( "strings" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/log/v3" "github.com/ledgerwatch/erigon/params" ) var DefaultMode = Mode{ Initialised: true, - History: Distance(math.MaxUint64), // all off - Receipts: Distance(math.MaxUint64), - TxIndex: Distance(math.MaxUint64), - CallTraces: Distance(math.MaxUint64), + History: Distance(math.MaxUint64), Blocks: Distance(math.MaxUint64), Experiments: Experiments{}, // all off } @@ -43,60 +38,11 @@ var DefaultMode = Mode{ type Experiments struct { } -func FromCli(chainId uint64, flags string, exactBlocks, exactHistory, exactReceipts, exactTxIndex, exactCallTraces, - beforeB, beforeH, beforeR, beforeT, beforeC uint64, experiments []string) (Mode, error) { +func FromCli(chainId uint64, distanceHistory, distanceBlocks uint64, experiments []string) (Mode, error) { mode := DefaultMode - if flags != "default" && flags != "disabled" { - for _, flag := range flags { - switch flag { - case 'h': - mode.History = Distance(params.FullImmutabilityThreshold) - case 'r': - mode.Receipts = Distance(params.FullImmutabilityThreshold) - case 't': - mode.TxIndex = Distance(params.FullImmutabilityThreshold) - case 'c': - mode.CallTraces = Distance(params.FullImmutabilityThreshold) - case 'b': - mode.Blocks = Distance(params.FullImmutabilityThreshold) - default: - return DefaultMode, fmt.Errorf("unexpected flag found: %c", flag) - } - } - } - - if exactBlocks > 0 { - mode.Blocks = Distance(exactBlocks) - } - if exactHistory > 0 { - mode.History = Distance(exactHistory) - } - if exactReceipts > 0 { - mode.Receipts = Distance(exactReceipts) - } - if exactTxIndex > 0 { - mode.TxIndex = Distance(exactTxIndex) - } - if exactCallTraces > 0 { - mode.CallTraces = Distance(exactCallTraces) - } - - if beforeH > 0 { - mode.History = Before(beforeH) - } - if beforeR > 0 { - mode.Receipts = Before(beforeR) - } - if beforeT > 0 { - mode.TxIndex = Before(beforeT) - } - if beforeC > 0 { - mode.CallTraces = Before(beforeC) - } - if beforeB > 0 { - mode.Blocks = Before(beforeB) - } + mode.History = Distance(distanceBlocks) + mode.Blocks = Distance(distanceHistory) for _, ex := range experiments { switch ex { @@ -121,30 +67,6 @@ func Get(db kv.Getter) (Mode, error) { prune.History = blockAmount } - blockAmount, err = get(db, kv.PruneReceipts) - if err != nil { - return prune, err - } - if blockAmount != nil { - prune.Receipts = blockAmount - } - - blockAmount, err = get(db, kv.PruneTxIndex) - if err != nil { - return prune, err - } - if blockAmount != nil { - prune.TxIndex = blockAmount - } - - blockAmount, err = get(db, kv.PruneCallTraces) - if err != nil { - return prune, err - } - if blockAmount != nil { - prune.CallTraces = blockAmount - } - blockAmount, err = get(db, kv.PruneBlocks) if err != nil { return prune, err @@ -159,9 +81,6 @@ func Get(db kv.Getter) (Mode, error) { type Mode struct { Initialised bool // Set when the values are initialised (not default) History BlockAmount - Receipts BlockAmount - TxIndex BlockAmount - CallTraces BlockAmount Blocks BlockAmount Experiments Experiments } @@ -189,31 +108,12 @@ func (p Distance) useDefaultValue() bool { return uint64(p) == params.FullImmuta func (p Distance) dbType() []byte { return kv.PruneTypeOlder } func (p Distance) PruneTo(stageHead uint64) uint64 { - if p == 0 { - panic("pruning distance were not set") - } if uint64(p) > stageHead { return 0 } return stageHead - uint64(p) } -// Before number after which keep in DB -type Before uint64 - -func (b Before) Enabled() bool { return b > 0 } -func (b Before) toValue() uint64 { return uint64(b) } -func (b Before) useDefaultValue() bool { return uint64(b) == 0 } -func (b Before) dbType() []byte { return kv.PruneTypeBefore } - -func (b Before) PruneTo(uint64) uint64 { - if b == 0 { - return uint64(b) - } - - return uint64(b) - 1 -} - func (m Mode) String() string { if !m.Initialised { return "default" @@ -235,27 +135,6 @@ func (m Mode) String() string { long += fmt.Sprintf(" --prune.b.%s=%d", m.Blocks.dbType(), m.Blocks.toValue()) } } - if m.Receipts.Enabled() { - if m.Receipts.useDefaultValue() { - short += fmt.Sprintf(" --prune.r.older=%d", defaultVal) - } else { - long += fmt.Sprintf(" --prune.r.%s=%d", m.Receipts.dbType(), m.Receipts.toValue()) - } - } - if m.TxIndex.Enabled() { - if m.TxIndex.useDefaultValue() { - short += fmt.Sprintf(" --prune.t.older=%d", defaultVal) - } else { - long += fmt.Sprintf(" --prune.t.%s=%d", m.TxIndex.dbType(), m.TxIndex.toValue()) - } - } - if m.CallTraces.Enabled() { - if m.CallTraces.useDefaultValue() { - short += fmt.Sprintf(" --prune.c.older=%d", defaultVal) - } else { - long += fmt.Sprintf(" --prune.c.%s=%d", m.CallTraces.dbType(), m.CallTraces.toValue()) - } - } return strings.TrimLeft(short+long, " ") } @@ -270,21 +149,6 @@ func Override(db kv.RwTx, sm Mode) error { return err } - err = set(db, kv.PruneReceipts, sm.Receipts) - if err != nil { - return err - } - - err = set(db, kv.PruneTxIndex, sm.TxIndex) - if err != nil { - return err - } - - err = set(db, kv.PruneCallTraces, sm.CallTraces) - if err != nil { - return err - } - err = set(db, kv.PruneBlocks, sm.Blocks) if err != nil { return err @@ -306,17 +170,9 @@ func EnsureNotChanged(tx kv.GetPut, pruneMode Mode) (Mode, error) { } if pruneMode.Initialised { - // Don't change from previous default as default for Receipts pruning has now changed - if pruneMode.Receipts.useDefaultValue() { - pruneMode.Receipts = pm.Receipts - } // If storage mode is not explicitly specified, we take whatever is in the database if !reflect.DeepEqual(pm, pruneMode) { - if bytes.Equal(pm.Receipts.dbType(), kv.PruneTypeOlder) && bytes.Equal(pruneMode.Receipts.dbType(), kv.PruneTypeBefore) { - log.Error("--prune=r flag has been changed to mean pruning of receipts before the Beacon Chain genesis. Please re-sync Erigon from scratch. " + - "Alternatively, enforce the old behaviour explicitly by --prune.r.older=90000 flag at the risk of breaking the Consensus Layer.") - } return pm, errors.New("not allowed change of --prune flag, last time you used: " + pm.String()) } } @@ -332,11 +188,8 @@ func setIfNotExist(db kv.GetPut, pm Mode) error { } pruneDBData := map[string]BlockAmount{ - string(kv.PruneHistory): pm.History, - string(kv.PruneReceipts): pm.Receipts, - string(kv.PruneTxIndex): pm.TxIndex, - string(kv.PruneCallTraces): pm.CallTraces, - string(kv.PruneBlocks): pm.Blocks, + string(kv.PruneHistory): pm.History, + string(kv.PruneBlocks): pm.Blocks, } for key, value := range pruneDBData { @@ -355,8 +208,6 @@ func createBlockAmount(pruneType []byte, v []byte) (BlockAmount, error) { switch string(pruneType) { case string(kv.PruneTypeOlder): blockAmount = Distance(binary.BigEndian.Uint64(v)) - case string(kv.PruneTypeBefore): - blockAmount = Before(binary.BigEndian.Uint64(v)) default: return nil, fmt.Errorf("unexpected block amount type: %s", string(pruneType)) } diff --git a/ethdb/prune/storage_mode_test.go b/ethdb/prune/storage_mode_test.go index f6d10332967..ec1a538ebfe 100644 --- a/ethdb/prune/storage_mode_test.go +++ b/ethdb/prune/storage_mode_test.go @@ -17,7 +17,6 @@ package prune import ( - "math/rand" "strconv" "testing" @@ -30,17 +29,14 @@ func TestSetStorageModeIfNotExist(t *testing.T) { _, tx := memdb.NewTestTx(t) prune, err := Get(tx) assert.NoError(t, err) - assert.Equal(t, Mode{true, Distance(math.MaxUint64), Distance(math.MaxUint64), - Distance(math.MaxUint64), Distance(math.MaxUint64), Distance(math.MaxUint64), Experiments{}}, prune) + assert.Equal(t, Mode{true, Distance(math.MaxUint64), Distance(math.MaxUint64), Experiments{}}, prune) - err = setIfNotExist(tx, Mode{true, Distance(1), Distance(2), - Before(3), Before(4), Before(100), Experiments{}}) + err = setIfNotExist(tx, Mode{true, Distance(1), Distance(2), Experiments{}}) assert.NoError(t, err) prune, err = Get(tx) assert.NoError(t, err) - assert.Equal(t, Mode{true, Distance(1), Distance(2), - Before(3), Before(4), Before(100), Experiments{}}, prune) + assert.Equal(t, Mode{true, Distance(1), Distance(2), Experiments{}}, prune) } var distanceTests = []struct { @@ -67,25 +63,3 @@ func TestDistancePruneTo(t *testing.T) { }) } } - -var beforeTests = []struct { - pruneTo uint64 - expected uint64 -}{ - {0, 0}, - {1_000_000, 999_999}, -} - -func TestBeforePruneTo(t *testing.T) { - for _, tt := range beforeTests { - t.Run(strconv.FormatUint(tt.pruneTo, 10), func(t *testing.T) { - stageHead := uint64(rand.Int63n(10_000_000)) - b := Before(tt.pruneTo) - pruneTo := b.PruneTo(stageHead) - - if pruneTo != tt.expected { - t.Errorf("got %d, want %d", pruneTo, tt.expected) - } - }) - } -} From 0318a39024f71d0764d4e0764dadbe150db6c95d Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 22:34:21 +0200 Subject: [PATCH 03/46] save --- turbo/snapshotsync/snapshotsync.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 83da3b78a11..7f520d47c2a 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -261,15 +261,7 @@ func getMaxStepRangeInSnapshots(preverified snapcfg.Preverified) (uint64, error) func computeBlocksToPrune(blockReader services.FullBlockReader, p prune.Mode) (blocksToPrune uint64, historyToPrune uint64) { frozenBlocks := blockReader.Snapshots().SegmentsMax() - blocksPruneTo := p.Blocks.PruneTo(frozenBlocks) - historyPruneTo := p.History.PruneTo(frozenBlocks) - if blocksPruneTo <= frozenBlocks { - blocksToPrune = frozenBlocks - blocksPruneTo - } - if historyPruneTo <= frozenBlocks { - historyToPrune = frozenBlocks - historyPruneTo - } - return blocksToPrune, historyToPrune + return p.Blocks.PruneTo(frozenBlocks), p.History.PruneTo(frozenBlocks) } // WaitForDownloader - wait for Downloader service to download all expected snapshots From cdc753dfd8b85d469605957d3aa611018a8bf86f Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 22:59:18 +0200 Subject: [PATCH 04/46] save --- turbo/snapshotsync/snapshotsync.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 7f520d47c2a..2a7cc38c23b 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -213,7 +213,9 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, minToDownload := uint64(math.MaxUint64) minStepToDownload := minStep stateTxNum := minStep * config3.HistoryV3AggregationStep + lastStep := uint64(0) if err := blockReader.IterateFrozenBodies(func(blockNum, baseTxNum, txAmount uint64) error { + lastStep = baseTxNum / config3.HistoryV3AggregationStep if blockNum == frozenBlocks-expectedPruneHistoryAmount { minStepToDownload = (baseTxNum / config3.HistoryV3AggregationStep) - 1 } @@ -231,6 +233,10 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, }); err != nil { return 0, 0, err } + // If we are near the last step, we may also not bother downloading the last steps. + if minStepToDownload+3 > lastStep { + minStepToDownload = math.MaxUint64 + } if expectedPruneBlockAmount == 0 { return minToDownload, 0, nil } From a372b4f5debf21187792fe2419dcc2821513fc68 Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 23:07:38 +0200 Subject: [PATCH 05/46] save --- turbo/cli/flags.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/turbo/cli/flags.go b/turbo/cli/flags.go index 8d08d399d54..7b17c947621 100644 --- a/turbo/cli/flags.go +++ b/turbo/cli/flags.go @@ -423,15 +423,16 @@ func ApplyFlagsForEthConfigCobra(f *pflag.FlagSet, cfg *ethconfig.Config) { if err != nil { utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) } - // Full mode prunes all but the latest state - if *pruneMode == "full" { + switch *pruneMode { + case "archive": + case "full": mode.Blocks = prune.Distance(math.MaxUint64) mode.History = prune.Distance(0) - } - // Minimal mode prunes all but the latest state including blocks - if *pruneMode == "minimal" { + case "minimal": mode.Blocks = prune.Distance(2048) // 2048 is just some blocks to allow reorgs mode.History = prune.Distance(0) + default: + utils.Fatalf(fmt.Sprintf("error: --prune.mode must be one of archive, full, minimal")) } cfg.Prune = mode From d1f1e3dd07258859a61d62d212d495dfd572b46a Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 23:08:08 +0200 Subject: [PATCH 06/46] save --- turbo/cli/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbo/cli/flags.go b/turbo/cli/flags.go index 7b17c947621..2d10554acbe 100644 --- a/turbo/cli/flags.go +++ b/turbo/cli/flags.go @@ -432,7 +432,7 @@ func ApplyFlagsForEthConfigCobra(f *pflag.FlagSet, cfg *ethconfig.Config) { mode.Blocks = prune.Distance(2048) // 2048 is just some blocks to allow reorgs mode.History = prune.Distance(0) default: - utils.Fatalf(fmt.Sprintf("error: --prune.mode must be one of archive, full, minimal")) + utils.Fatalf("error: --prune.mode must be one of archive, full, minimal") } cfg.Prune = mode From d0d21f067396fdbaa014b1da352be6d1d95710e2 Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 23:12:35 +0200 Subject: [PATCH 07/46] save --- turbo/snapshotsync/snapshotsync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 2a7cc38c23b..8bc3551da22 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -234,8 +234,8 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, return 0, 0, err } // If we are near the last step, we may also not bother downloading the last steps. - if minStepToDownload+3 > lastStep { - minStepToDownload = math.MaxUint64 + if minStepToDownload+16 > lastStep { + minStepToDownload = minStep } if expectedPruneBlockAmount == 0 { return minToDownload, 0, nil From 219020cf7251e304149476a739e49b153eb413b3 Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 23:13:02 +0200 Subject: [PATCH 08/46] save --- turbo/snapshotsync/snapshotsync.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 8bc3551da22..dd53a015a1f 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -237,9 +237,6 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, if minStepToDownload+16 > lastStep { minStepToDownload = minStep } - if expectedPruneBlockAmount == 0 { - return minToDownload, 0, nil - } // return the minimum number of blocks to download and the minimum step. return minToDownload, minStep - minStepToDownload, nil } From cdd33922557d1ed1ff0a60edf7d555086df433d6 Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 23:17:45 +0200 Subject: [PATCH 09/46] save --- turbo/snapshotsync/snapshotsync.go | 1 + 1 file changed, 1 insertion(+) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index dd53a015a1f..08515701769 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -315,6 +315,7 @@ func WaitForDownloader(ctx context.Context, logPrefix string, headerchain, blobs if err != nil { return err } + fmt.Println("minBlockAmountToDownload", minBlockAmountToDownload, "minStepToDownload", minStepToDownload) blackListForPruning, err = buildBlackListForPruning(wantToPrune, minStepToDownload, minBlockAmountToDownload, blockPrune, preverifiedBlockSnapshots) if err != nil { return err From 3ce0d86326b6e4ee26d7e0d19d5b2af65ccb4f38 Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 23:23:29 +0200 Subject: [PATCH 10/46] save --- turbo/snapshotsync/snapshotsync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 08515701769..830ac1a263d 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -238,7 +238,7 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, minStepToDownload = minStep } // return the minimum number of blocks to download and the minimum step. - return minToDownload, minStep - minStepToDownload, nil + return frozenBlocks - minToDownload, minStep - minStepToDownload, nil } func getMaxStepRangeInSnapshots(preverified snapcfg.Preverified) (uint64, error) { From 16df0182366a726e68469701e76d4e2d3681522d Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 23:33:40 +0200 Subject: [PATCH 11/46] save --- turbo/snapshotsync/snapshotsync.go | 59 ++++++------------------------ 1 file changed, 12 insertions(+), 47 deletions(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 830ac1a263d..54ba3398c12 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -22,7 +22,6 @@ import ( "fmt" "math" "runtime" - "sort" "strconv" "strings" "time" @@ -113,7 +112,6 @@ func adjustBlockPrune(blocks, minBlocksToDownload uint64) uint64 { return blocks } ret := blocks + snaptype.Erigon2MergeLimit - // round to nearest multiple of 64. if less than 64, round to 64 return ret - ret%snaptype.Erigon2MergeLimit } @@ -126,84 +124,51 @@ func canSnapshotBePruned(name string) bool { } func buildBlackListForPruning(pruneMode bool, stepPrune, minBlockToDownload, blockPrune uint64, preverified snapcfg.Preverified) (map[string]struct{}, error) { - type snapshotFileData struct { - from, to uint64 - stepBased bool - name string - } + blackList := make(map[string]struct{}) if !pruneMode { return blackList, nil } stepPrune = adjustStepPrune(stepPrune) blockPrune = adjustBlockPrune(blockPrune, minBlockToDownload) - snapshotKindToNames := make(map[string][]snapshotFileData) + fmt.Println("stepPrune", stepPrune, "blockPrune", blockPrune, "minBlockToDownload", minBlockToDownload) for _, p := range preverified { name := p.Name // Don't prune unprunable files if !canSnapshotBePruned(name) { continue } - var from, to uint64 + var _, to uint64 var err error - var kind string if shouldUseStepsForPruning(name) { // parse "from" (0) and "to" (64) from the name // parse the snapshot "kind". e.g kind of 'idx/v1-accounts.0-64.ef' is "idx/v1-accounts" rangeString := strings.Split(name, ".")[1] rangeNums := strings.Split(rangeString, "-") // convert the range to uint64 - from, err = strconv.ParseUint(rangeNums[0], 10, 64) - if err != nil { - return nil, err - } to, err = strconv.ParseUint(rangeNums[1], 10, 64) if err != nil { return nil, err } - kind = strings.Split(name, ".")[0] + if stepPrune < to { + continue + } + blackList[name] = struct{}{} } else { // e.g 'v1-000000-000100-beaconblocks.seg' // parse "from" (000000) and "to" (000100) from the name. 100 is 100'000 blocks - minusSplit := strings.Split(name, "-") s, _, ok := snaptype.ParseFileName("", name) if !ok { continue } - from = s.From to = s.To - kind = minusSplit[3] - } - blackList[p.Name] = struct{}{} // Add all of them to the blacklist and remove the ones that are not blacklisted later. - snapshotKindToNames[kind] = append(snapshotKindToNames[kind], snapshotFileData{ - from: from, - to: to, - stepBased: shouldUseStepsForPruning(name), - name: name, - }) - } - // sort the snapshots by "from" and "to" in ascending order - for _, snapshots := range snapshotKindToNames { - prunedDistance := uint64(0) // keep track of pruned distance for snapshots - // sort the snapshots by "from" and "to" in descending order - sort.Slice(snapshots, func(i, j int) bool { - if snapshots[i].from == snapshots[j].from { - return snapshots[i].to > snapshots[j].to - } - return snapshots[i].from > snapshots[j].from - }) - for _, snapshot := range snapshots { - if snapshot.stepBased { - if prunedDistance >= stepPrune { - break - } - } else if prunedDistance >= blockPrune { - break + if blockPrune < to { + continue } - delete(blackList, snapshot.name) - prunedDistance += snapshot.to - snapshot.from + blackList[name] = struct{}{} } } + return blackList, nil } @@ -235,7 +200,7 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, } // If we are near the last step, we may also not bother downloading the last steps. if minStepToDownload+16 > lastStep { - minStepToDownload = minStep + minStep = math.MaxUint64 } // return the minimum number of blocks to download and the minimum step. return frozenBlocks - minToDownload, minStep - minStepToDownload, nil From 9f6b6b2e25bc6c88470027c4753ef320e6f6d88d Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 23:39:12 +0200 Subject: [PATCH 12/46] save --- turbo/snapshotsync/snapshotsync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 54ba3398c12..a9d0367ccb6 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -105,7 +105,7 @@ func adjustBlockPrune(blocks, minBlocksToDownload uint64) uint64 { if minBlocksToDownload < snaptype.Erigon2MergeLimit { minBlocksToDownload = snaptype.Erigon2MergeLimit } - if blocks < minBlocksToDownload { + if blocks > minBlocksToDownload { blocks = minBlocksToDownload } if blocks%snaptype.Erigon2MergeLimit == 0 { From 930fc93d0fa01bdf69e60363119693ed14be10d9 Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 23:42:01 +0200 Subject: [PATCH 13/46] save --- turbo/snapshotsync/snapshotsync.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index a9d0367ccb6..0fcc4966493 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -111,8 +111,7 @@ func adjustBlockPrune(blocks, minBlocksToDownload uint64) uint64 { if blocks%snaptype.Erigon2MergeLimit == 0 { return blocks } - ret := blocks + snaptype.Erigon2MergeLimit - return ret - ret%snaptype.Erigon2MergeLimit + return blocks - blocks%snaptype.Erigon2MergeLimit } func shouldUseStepsForPruning(name string) bool { From 6f311b42f20cdc3e8b5ad5df52de5efb8f6e1285 Mon Sep 17 00:00:00 2001 From: Giulio Date: Tue, 16 Jul 2024 23:50:39 +0200 Subject: [PATCH 14/46] save --- turbo/snapshotsync/snapshotsync.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 0fcc4966493..b11df641743 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -130,7 +130,6 @@ func buildBlackListForPruning(pruneMode bool, stepPrune, minBlockToDownload, blo } stepPrune = adjustStepPrune(stepPrune) blockPrune = adjustBlockPrune(blockPrune, minBlockToDownload) - fmt.Println("stepPrune", stepPrune, "blockPrune", blockPrune, "minBlockToDownload", minBlockToDownload) for _, p := range preverified { name := p.Name // Don't prune unprunable files @@ -177,9 +176,7 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, minToDownload := uint64(math.MaxUint64) minStepToDownload := minStep stateTxNum := minStep * config3.HistoryV3AggregationStep - lastStep := uint64(0) if err := blockReader.IterateFrozenBodies(func(blockNum, baseTxNum, txAmount uint64) error { - lastStep = baseTxNum / config3.HistoryV3AggregationStep if blockNum == frozenBlocks-expectedPruneHistoryAmount { minStepToDownload = (baseTxNum / config3.HistoryV3AggregationStep) - 1 } @@ -197,10 +194,6 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, }); err != nil { return 0, 0, err } - // If we are near the last step, we may also not bother downloading the last steps. - if minStepToDownload+16 > lastStep { - minStep = math.MaxUint64 - } // return the minimum number of blocks to download and the minimum step. return frozenBlocks - minToDownload, minStep - minStepToDownload, nil } From 5cd4e196f6e106fa66c821c7108d9745d1cad3ef Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 00:18:10 +0200 Subject: [PATCH 15/46] save --- turbo/snapshotsync/snapshotsync.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index b11df641743..7338aa2d317 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -108,9 +108,6 @@ func adjustBlockPrune(blocks, minBlocksToDownload uint64) uint64 { if blocks > minBlocksToDownload { blocks = minBlocksToDownload } - if blocks%snaptype.Erigon2MergeLimit == 0 { - return blocks - } return blocks - blocks%snaptype.Erigon2MergeLimit } @@ -268,12 +265,12 @@ func WaitForDownloader(ctx context.Context, logPrefix string, headerchain, blobs if err != nil { return err } - minBlockAmountToDownload, minStepToDownload, err := getMinimumBlocksToDownload(tx, blockReader, minStep, blockPrune, historyPrune) + minBlockToDownload, minStepToDownload, err := getMinimumBlocksToDownload(tx, blockReader, minStep, blockPrune, historyPrune) if err != nil { return err } - fmt.Println("minBlockAmountToDownload", minBlockAmountToDownload, "minStepToDownload", minStepToDownload) - blackListForPruning, err = buildBlackListForPruning(wantToPrune, minStepToDownload, minBlockAmountToDownload, blockPrune, preverifiedBlockSnapshots) + fmt.Println("minBlockToDownload", minBlockToDownload, "minStepToDownload", minStepToDownload, "blockPrune", blockPrune) + blackListForPruning, err = buildBlackListForPruning(wantToPrune, minStepToDownload, minBlockToDownload, blockPrune, preverifiedBlockSnapshots) if err != nil { return err } From 0094f5697d9cd7898c151a688ee700cb28cdd16c Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 00:34:18 +0200 Subject: [PATCH 16/46] save --- ethdb/prune/storage_mode.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethdb/prune/storage_mode.go b/ethdb/prune/storage_mode.go index 250d27987a3..e81921219f3 100644 --- a/ethdb/prune/storage_mode.go +++ b/ethdb/prune/storage_mode.go @@ -41,8 +41,8 @@ type Experiments struct { func FromCli(chainId uint64, distanceHistory, distanceBlocks uint64, experiments []string) (Mode, error) { mode := DefaultMode - mode.History = Distance(distanceBlocks) - mode.Blocks = Distance(distanceHistory) + mode.History = Distance(distanceHistory) + mode.Blocks = Distance(distanceBlocks) for _, ex := range experiments { switch ex { From ed8e38a3b591617a8eb9d011cc4ab382ad344471 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 00:39:31 +0200 Subject: [PATCH 17/46] save --- turbo/snapshotsync/snapshotsync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 7338aa2d317..9754c0e29da 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -269,7 +269,7 @@ func WaitForDownloader(ctx context.Context, logPrefix string, headerchain, blobs if err != nil { return err } - fmt.Println("minBlockToDownload", minBlockToDownload, "minStepToDownload", minStepToDownload, "blockPrune", blockPrune) + fmt.Println("minBlockToDownload", minBlockToDownload, "minStepToDownload", minStepToDownload, "blockPrune", blockPrune, "minStep", minStep) blackListForPruning, err = buildBlackListForPruning(wantToPrune, minStepToDownload, minBlockToDownload, blockPrune, preverifiedBlockSnapshots) if err != nil { return err From 3ac67d6e54fe1f2196737e1bbfaeed7e5b16999a Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 00:49:54 +0200 Subject: [PATCH 18/46] save --- turbo/snapshotsync/snapshotsync.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 9754c0e29da..d2703f98131 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -168,14 +168,16 @@ func buildBlackListForPruning(pruneMode bool, stepPrune, minBlockToDownload, blo } // getMinimumBlocksToDownload - get the minimum number of blocks to download -func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, minStep uint64, expectedPruneBlockAmount, expectedPruneHistoryAmount uint64) (uint64, uint64, error) { +func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, minStep uint64, blockPruneTo, historyPruneTo uint64) (uint64, uint64, error) { frozenBlocks := blockReader.Snapshots().SegmentsMax() minToDownload := uint64(math.MaxUint64) - minStepToDownload := minStep + minStepToDownload := uint64(0) + minStepToDownloadSet := false stateTxNum := minStep * config3.HistoryV3AggregationStep if err := blockReader.IterateFrozenBodies(func(blockNum, baseTxNum, txAmount uint64) error { - if blockNum == frozenBlocks-expectedPruneHistoryAmount { - minStepToDownload = (baseTxNum / config3.HistoryV3AggregationStep) - 1 + if blockNum == historyPruneTo { + minStepToDownload = (baseTxNum - (config3.HistoryV3AggregationStep - 1)) / config3.HistoryV3AggregationStep + minStepToDownloadSet = true } if stateTxNum <= baseTxNum { // only cosnider the block if it return nil @@ -191,8 +193,11 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, }); err != nil { return 0, 0, err } + if !minStepToDownloadSet { + minStepToDownload = math.MaxUint32 + } // return the minimum number of blocks to download and the minimum step. - return frozenBlocks - minToDownload, minStep - minStepToDownload, nil + return frozenBlocks - minToDownload, minStepToDownload, nil } func getMaxStepRangeInSnapshots(preverified snapcfg.Preverified) (uint64, error) { From c64bc9e030ee15267fd642b5fb1515b5c119e91e Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 00:54:01 +0200 Subject: [PATCH 19/46] save --- turbo/snapshotsync/snapshotsync.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index d2703f98131..7de59fbb4a7 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -171,13 +171,13 @@ func buildBlackListForPruning(pruneMode bool, stepPrune, minBlockToDownload, blo func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, minStep uint64, blockPruneTo, historyPruneTo uint64) (uint64, uint64, error) { frozenBlocks := blockReader.Snapshots().SegmentsMax() minToDownload := uint64(math.MaxUint64) - minStepToDownload := uint64(0) - minStepToDownloadSet := false + minStepToDownload := uint64(math.MaxUint32) stateTxNum := minStep * config3.HistoryV3AggregationStep + fmt.Println("historyPruneTo", historyPruneTo) if err := blockReader.IterateFrozenBodies(func(blockNum, baseTxNum, txAmount uint64) error { if blockNum == historyPruneTo { + fmt.Println("blockNum", blockNum, "historyPruneTo", historyPruneTo) minStepToDownload = (baseTxNum - (config3.HistoryV3AggregationStep - 1)) / config3.HistoryV3AggregationStep - minStepToDownloadSet = true } if stateTxNum <= baseTxNum { // only cosnider the block if it return nil @@ -193,9 +193,7 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, }); err != nil { return 0, 0, err } - if !minStepToDownloadSet { - minStepToDownload = math.MaxUint32 - } + // return the minimum number of blocks to download and the minimum step. return frozenBlocks - minToDownload, minStepToDownload, nil } From 4e2268537f07fcc7d83d30b02deb266395c36644 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 00:57:45 +0200 Subject: [PATCH 20/46] save --- turbo/snapshotsync/snapshotsync.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 7de59fbb4a7..2401e2c5671 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -178,6 +178,9 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, if blockNum == historyPruneTo { fmt.Println("blockNum", blockNum, "historyPruneTo", historyPruneTo) minStepToDownload = (baseTxNum - (config3.HistoryV3AggregationStep - 1)) / config3.HistoryV3AggregationStep + if baseTxNum < (config3.HistoryV3AggregationStep - 1) { + minStepToDownload = 0 + } } if stateTxNum <= baseTxNum { // only cosnider the block if it return nil From 23d3c451fa71055e07d17cb942d2e6dbe9e31671 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 01:22:20 +0200 Subject: [PATCH 21/46] save --- turbo/snapshotsync/snapshotsync.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 2401e2c5671..dab8a0173f8 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -173,10 +173,8 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, minToDownload := uint64(math.MaxUint64) minStepToDownload := uint64(math.MaxUint32) stateTxNum := minStep * config3.HistoryV3AggregationStep - fmt.Println("historyPruneTo", historyPruneTo) if err := blockReader.IterateFrozenBodies(func(blockNum, baseTxNum, txAmount uint64) error { if blockNum == historyPruneTo { - fmt.Println("blockNum", blockNum, "historyPruneTo", historyPruneTo) minStepToDownload = (baseTxNum - (config3.HistoryV3AggregationStep - 1)) / config3.HistoryV3AggregationStep if baseTxNum < (config3.HistoryV3AggregationStep - 1) { minStepToDownload = 0 From 302b04b4910dd3928e031f38d50183a9b505fe3d Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 01:36:28 +0200 Subject: [PATCH 22/46] save --- erigon-lib/state/aggregator.go | 6 ++++++ erigon-lib/state/inverted_index.go | 5 +++-- eth/stagedsync/exec3.go | 11 +++++++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index eac3dbbfc90..c83fc8640fa 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -381,6 +381,12 @@ func (a *Aggregator) DiscardHistory(name kv.Domain) *Aggregator { a.d[name].historyDisabled = true return a } + +func (a *Aggregator) DiscardInveredIndex(name kv.InvertedIdxPos) *Aggregator { + a.iis[name].discardIdx = true + return a +} + func (a *Aggregator) EnableHistory(name kv.Domain) *Aggregator { a.d[name].historyDisabled = false return a diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index 4daa68384fb..5d028a1e921 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -85,7 +85,8 @@ type InvertedIndex struct { // fields for history write logger log.Logger - noFsync bool // fsync is enabled by default, but tests can manually disable + noFsync bool // fsync is enabled by default, but tests can manually disable + discardIdx bool compression FileCompression compressWorkers int @@ -381,7 +382,7 @@ func (w *invertedIndexBufferedWriter) Add(key []byte) error { } func (iit *InvertedIndexRoTx) NewWriter() *invertedIndexBufferedWriter { - return iit.newWriter(iit.ii.dirs.Tmp, false) + return iit.newWriter(iit.ii.dirs.Tmp, iit.ii.discardIdx) } type invertedIndexBufferedWriter struct { diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 13c5d120be8..e5699f2aa7a 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -192,14 +192,17 @@ func ExecV3(ctx context.Context, if initialCycle { agg.SetCollateAndBuildWorkers(min(2, estimate.StateV3Collate.Workers())) agg.SetCompressWorkers(estimate.CompressSnapshot.Workers()) - //if blockNum < cfg.blockReader.FrozenBlocks() { - //defer agg.DiscardHistory(kv.CommitmentDomain).EnableHistory(kv.CommitmentDomain) - //defer agg.LimitRecentHistoryWithoutFiles(0).LimitRecentHistoryWithoutFiles(agg.StepSize() / 10) - //} } else { agg.SetCompressWorkers(1) agg.SetCollateAndBuildWorkers(1) } + // Disable all inverted indexes if we do max pruning + if cfg.prune.History.Enabled() && cfg.prune.History.PruneTo(execStage.BlockNumber) == execStage.BlockNumber { + agg.DiscardInveredIndex(kv.LogAddrIdxPos) + agg.DiscardInveredIndex(kv.LogTopicIdxPos) + agg.DiscardInveredIndex(kv.TracesFromIdxPos) + agg.DiscardInveredIndex(kv.TracesToIdxPos) + } var err error inMemExec := txc.Doms != nil From b5ca96cccb36c073ec61a0959fcfa8b4e90ca30f Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 02:01:11 +0200 Subject: [PATCH 23/46] save --- eth/stagedsync/exec3.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index e5699f2aa7a..07bb43a8b7f 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -96,21 +96,21 @@ func (p *Progress) Log(rs *state.StateV3, in *state.QueueWithRetry, rws *state.R interval := currentTime.Sub(p.prevTime) speedTx := float64(doneCount-p.prevCount) / (float64(interval) / float64(time.Second)) //var repeatRatio float64 - //if doneCount > p.prevCount { - // repeatRatio = 100.0 * float64(repeatCount-p.prevRepeatCount) / float64(doneCount-p.prevCount) - //} - p.logger.Info(fmt.Sprintf("[%s] Transaction replay", p.logPrefix), - //"workers", workerCount, + logArgs := []interface{}{ "blk", outputBlockNum, "tx/s", fmt.Sprintf("%.1f", speedTx), //"pipe", fmt.Sprintf("(%d+%d)->%d/%d->%d/%d", in.NewTasksLen(), in.RetriesLen(), rws.ResultChLen(), rws.ResultChCap(), rws.Len(), rws.Limit()), //"repeatRatio", fmt.Sprintf("%.2f%%", repeatRatio), //"workers", p.workersCount, "buffer", fmt.Sprintf("%s/%s", common.ByteCount(sizeEstimate), common.ByteCount(p.commitThreshold)), - "stepsInDB", fmt.Sprintf("%.2f", idxStepsAmountInDB), "step", fmt.Sprintf("%.1f", float64(outTxNum)/float64(config3.HistoryV3AggregationStep)), "alloc", common.ByteCount(m.Alloc), "sys", common.ByteCount(m.Sys), - ) + } + if idxStepsAmountInDB > 0.01 { + logArgs = append(logArgs, "stepsInDB", fmt.Sprintf("%.2f", idxStepsAmountInDB)) + } + + p.logger.Info(fmt.Sprintf("[%s] Transaction replay", p.logPrefix), logArgs...) p.prevTime = currentTime p.prevCount = doneCount From 5f1fcf4c5413be370cd738c1e3d274856297b755 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 02:13:24 +0200 Subject: [PATCH 24/46] save --- erigon-lib/state/aggregator.go | 2 +- eth/stagedsync/exec3.go | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index c83fc8640fa..e229961302f 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -382,7 +382,7 @@ func (a *Aggregator) DiscardHistory(name kv.Domain) *Aggregator { return a } -func (a *Aggregator) DiscardInveredIndex(name kv.InvertedIdxPos) *Aggregator { +func (a *Aggregator) DiscardInvertedIndex(name kv.InvertedIdxPos) *Aggregator { a.iis[name].discardIdx = true return a } diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 07bb43a8b7f..e25baa802d6 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -198,10 +198,13 @@ func ExecV3(ctx context.Context, } // Disable all inverted indexes if we do max pruning if cfg.prune.History.Enabled() && cfg.prune.History.PruneTo(execStage.BlockNumber) == execStage.BlockNumber { - agg.DiscardInveredIndex(kv.LogAddrIdxPos) - agg.DiscardInveredIndex(kv.LogTopicIdxPos) - agg.DiscardInveredIndex(kv.TracesFromIdxPos) - agg.DiscardInveredIndex(kv.TracesToIdxPos) + agg.DiscardInvertedIndex(kv.LogAddrIdxPos) + agg.DiscardInvertedIndex(kv.LogTopicIdxPos) + agg.DiscardInvertedIndex(kv.TracesFromIdxPos) + agg.DiscardInvertedIndex(kv.TracesToIdxPos) + agg.DiscardHistory(kv.AccountsDomain) + agg.DiscardHistory(kv.StorageDomain) + agg.DiscardHistory(kv.CodeDomain) } var err error From fe2d6714ccf58c2da9514258827883758081dc44 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 22:00:34 +0200 Subject: [PATCH 25/46] save --- erigon-lib/state/domain.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index b921dea8077..5a697288b77 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1646,6 +1646,7 @@ func (dt *DomainRoTx) DomainRangeLatest(roTx kv.Tx, fromKey, toKey []byte, limit func (dt *DomainRoTx) CanPruneUntil(tx kv.Tx, untilTx uint64) bool { canDomain, _ := dt.canPruneDomainTables(tx, untilTx) canHistory, _ := dt.ht.canPruneUntil(tx, untilTx) + fmt.Println("CanPruneUntil", canDomain, canHistory) return canHistory || canDomain } @@ -1739,6 +1740,7 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT return nil, fmt.Errorf("prune history at step %d [%d, %d): %w", step, txFrom, txTo, err) } canPrune, maxPrunableStep := dt.canPruneDomainTables(rwTx, txTo) + fmt.Println("dt.canPruneDomainTables", canPrune, maxPrunableStep) if !canPrune { return stat, nil } From f442e2eb16f755e5a14b2787d859cd1716082f43 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 22:10:43 +0200 Subject: [PATCH 26/46] save --- erigon-lib/state/domain.go | 1 + 1 file changed, 1 insertion(+) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 5a697288b77..c6fae1197e9 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1671,6 +1671,7 @@ func (dt *DomainRoTx) canPruneDomainTables(tx kv.Tx, untilTx uint64) (can bool, dt.d.logger.Error("get domain pruning progress", "name", dt.d.filenameBase, "error", err) return false, maxStepToPrune } + fmt.Println(sm, maxStepToPrune, untilStep) delta := float64(max(maxStepToPrune, sm) - min(maxStepToPrune, sm)) // maxStep could be 0 switch dt.d.filenameBase { From 06afbd5781a105398d4f0412386a985edd13a4e7 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 22:24:53 +0200 Subject: [PATCH 27/46] save --- eth/stagedsync/exec3.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index e25baa802d6..127db8c5120 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -196,15 +196,14 @@ func ExecV3(ctx context.Context, agg.SetCompressWorkers(1) agg.SetCollateAndBuildWorkers(1) } + + pruneNonEssentials := cfg.prune.History.Enabled() && cfg.prune.History.PruneTo(execStage.BlockNumber) == execStage.BlockNumber // Disable all inverted indexes if we do max pruning - if cfg.prune.History.Enabled() && cfg.prune.History.PruneTo(execStage.BlockNumber) == execStage.BlockNumber { + if pruneNonEssentials { agg.DiscardInvertedIndex(kv.LogAddrIdxPos) agg.DiscardInvertedIndex(kv.LogTopicIdxPos) agg.DiscardInvertedIndex(kv.TracesFromIdxPos) agg.DiscardInvertedIndex(kv.TracesToIdxPos) - agg.DiscardHistory(kv.AccountsDomain) - agg.DiscardHistory(kv.StorageDomain) - agg.DiscardHistory(kv.CodeDomain) } var err error From 294865beed69a6aabd51287804dddc1cd4b73336 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 17 Jul 2024 22:47:48 +0200 Subject: [PATCH 28/46] save --- eth/stagedsync/exec3.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 127db8c5120..d5efab2ecda 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -204,6 +204,9 @@ func ExecV3(ctx context.Context, agg.DiscardInvertedIndex(kv.LogTopicIdxPos) agg.DiscardInvertedIndex(kv.TracesFromIdxPos) agg.DiscardInvertedIndex(kv.TracesToIdxPos) + // We do not discard account because it's used in snapshot generation. + agg.DiscardHistory(kv.StorageDomain) + agg.DiscardHistory(kv.CodeDomain) } var err error From cf37670d0642be15ed4cb894b0099bf516063393 Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 15:12:04 +0200 Subject: [PATCH 29/46] save --- eth/stagedsync/exec3.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index d5efab2ecda..9d7c28c8d41 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -200,13 +200,8 @@ func ExecV3(ctx context.Context, pruneNonEssentials := cfg.prune.History.Enabled() && cfg.prune.History.PruneTo(execStage.BlockNumber) == execStage.BlockNumber // Disable all inverted indexes if we do max pruning if pruneNonEssentials { - agg.DiscardInvertedIndex(kv.LogAddrIdxPos) - agg.DiscardInvertedIndex(kv.LogTopicIdxPos) agg.DiscardInvertedIndex(kv.TracesFromIdxPos) agg.DiscardInvertedIndex(kv.TracesToIdxPos) - // We do not discard account because it's used in snapshot generation. - agg.DiscardHistory(kv.StorageDomain) - agg.DiscardHistory(kv.CodeDomain) } var err error From 76a664ee65b232e339f5327a45ea960acb92db2c Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 15:31:31 +0200 Subject: [PATCH 30/46] save --- core/state/rw_v3.go | 12 ++++++++++ core/state/txtask.go | 38 ++++++++++++++++-------------- erigon-lib/state/inverted_index.go | 2 +- eth/stagedsync/exec3.go | 37 +++++++++++++---------------- 4 files changed, 50 insertions(+), 39 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 24b106affff..dc7b6935081 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -213,18 +213,30 @@ func (rs *StateV3) ApplyLogsAndTraces4(txTask *TxTask, domains *libstate.SharedD if dbg.DiscardHistory() { return nil } + shouldPruneNonEssentials := txTask.PruneNonEssentials && txTask.Config != nil for addr := range txTask.TraceFroms { + if shouldPruneNonEssentials && addr != txTask.Config.DepositContract { + continue + } if err := domains.IndexAdd(kv.TblTracesFromIdx, addr[:]); err != nil { return err } } + for addr := range txTask.TraceTos { + if shouldPruneNonEssentials && addr != txTask.Config.DepositContract { + continue + } if err := domains.IndexAdd(kv.TblTracesToIdx, addr[:]); err != nil { return err } } + for _, lg := range txTask.Logs { + if shouldPruneNonEssentials && lg.Address != txTask.Config.DepositContract { + continue + } if err := domains.IndexAdd(kv.TblLogAddressIdx, lg.Address[:]); err != nil { return err } diff --git a/core/state/txtask.go b/core/state/txtask.go index cf2ee0f7c77..439aaf8317f 100644 --- a/core/state/txtask.go +++ b/core/state/txtask.go @@ -36,24 +36,25 @@ import ( // which is processed by a single thread that writes into the ReconState1 and // flushes to the database type TxTask struct { - TxNum uint64 - BlockNum uint64 - Rules *chain.Rules - Header *types.Header - Txs types.Transactions - Uncles []*types.Header - Coinbase libcommon.Address - Withdrawals types.Withdrawals - BlockHash libcommon.Hash - Sender *libcommon.Address - SkipAnalysis bool - TxIndex int // -1 for block initialisation - Final bool - Failed bool - Tx types.Transaction - GetHashFn func(n uint64) libcommon.Hash - TxAsMessage types.Message - EvmBlockContext evmtypes.BlockContext + TxNum uint64 + BlockNum uint64 + Rules *chain.Rules + Header *types.Header + Txs types.Transactions + Uncles []*types.Header + Coinbase libcommon.Address + Withdrawals types.Withdrawals + BlockHash libcommon.Hash + Sender *libcommon.Address + SkipAnalysis bool + PruneNonEssentials bool + TxIndex int // -1 for block initialisation + Final bool + Failed bool + Tx types.Transaction + GetHashFn func(n uint64) libcommon.Hash + TxAsMessage types.Message + EvmBlockContext evmtypes.BlockContext HistoryExecution bool // use history reader for that txn instead of state reader @@ -79,6 +80,7 @@ type TxTask struct { BlockReceipts types.Receipts Requests types.Requests + Config *chain.Config } func (t *TxTask) CreateReceipt(cumulativeGasUsed uint64) *types.Receipt { diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index 5d028a1e921..262a1b814ac 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -382,7 +382,7 @@ func (w *invertedIndexBufferedWriter) Add(key []byte) error { } func (iit *InvertedIndexRoTx) NewWriter() *invertedIndexBufferedWriter { - return iit.newWriter(iit.ii.dirs.Tmp, iit.ii.discardIdx) + return iit.newWriter(iit.ii.dirs.Tmp, false) } type invertedIndexBufferedWriter struct { diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 9d7c28c8d41..6be67f68548 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -198,11 +198,6 @@ func ExecV3(ctx context.Context, } pruneNonEssentials := cfg.prune.History.Enabled() && cfg.prune.History.PruneTo(execStage.BlockNumber) == execStage.BlockNumber - // Disable all inverted indexes if we do max pruning - if pruneNonEssentials { - agg.DiscardInvertedIndex(kv.TracesFromIdxPos) - agg.DiscardInvertedIndex(kv.TracesToIdxPos) - } var err error inMemExec := txc.Doms != nil @@ -721,26 +716,28 @@ Loop: for txIndex := -1; txIndex <= len(txs); txIndex++ { // Do not oversend, wait for the result heap to go under certain size txTask := &state.TxTask{ - BlockNum: blockNum, - Header: header, - Coinbase: b.Coinbase(), - Uncles: b.Uncles(), - Rules: rules, - Txs: txs, - TxNum: inputTxNum, - TxIndex: txIndex, - BlockHash: b.Hash(), - SkipAnalysis: skipAnalysis, - Final: txIndex == len(txs), - GetHashFn: getHashFn, - EvmBlockContext: blockContext, - Withdrawals: b.Withdrawals(), - Requests: b.Requests(), + BlockNum: blockNum, + Header: header, + Coinbase: b.Coinbase(), + Uncles: b.Uncles(), + Rules: rules, + Txs: txs, + TxNum: inputTxNum, + TxIndex: txIndex, + BlockHash: b.Hash(), + SkipAnalysis: skipAnalysis, + Final: txIndex == len(txs), + GetHashFn: getHashFn, + EvmBlockContext: blockContext, + Withdrawals: b.Withdrawals(), + Requests: b.Requests(), + PruneNonEssentials: pruneNonEssentials, // use history reader instead of state reader to catch up to the tx where we left off HistoryExecution: offsetFromBlockBeginning > 0 && txIndex < int(offsetFromBlockBeginning), BlockReceipts: receipts, + Config: cfg.genesis.Config, } if txTask.TxNum <= txNumInDB && txTask.TxNum > 0 { inputTxNum++ From 7f65033084fc988e886825fc91aabaa4c14b8942 Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 15:33:21 +0200 Subject: [PATCH 31/46] save --- cl/sentinel/discovery.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cl/sentinel/discovery.go b/cl/sentinel/discovery.go index 4499d3d23f4..09a74ff2ba7 100644 --- a/cl/sentinel/discovery.go +++ b/cl/sentinel/discovery.go @@ -122,9 +122,12 @@ func (s *Sentinel) listenForPeers() { continue } - if err := s.ConnectWithPeer(s.ctx, *peerInfo); err != nil { - log.Trace("[Sentinel] Could not connect with peer", "err", err) - } + go func() { + if err := s.ConnectWithPeer(s.ctx, *peerInfo); err != nil { + log.Trace("[Sentinel] Could not connect with peer", "err", err) + } + }() + } } From c9918536a49b16665319bcb347ad2635759fd029 Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 15:36:11 +0200 Subject: [PATCH 32/46] save --- erigon-lib/state/domain.go | 1 - 1 file changed, 1 deletion(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index c6fae1197e9..3ad3e027703 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1646,7 +1646,6 @@ func (dt *DomainRoTx) DomainRangeLatest(roTx kv.Tx, fromKey, toKey []byte, limit func (dt *DomainRoTx) CanPruneUntil(tx kv.Tx, untilTx uint64) bool { canDomain, _ := dt.canPruneDomainTables(tx, untilTx) canHistory, _ := dt.ht.canPruneUntil(tx, untilTx) - fmt.Println("CanPruneUntil", canDomain, canHistory) return canHistory || canDomain } From 69c0b0e415bb4f495a3e7960a50d24f6dd5807f5 Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 15:39:36 +0200 Subject: [PATCH 33/46] save --- erigon-lib/state/aggregator.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 2f7ddc87bef..0fc52e5e285 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -382,11 +382,6 @@ func (a *Aggregator) DiscardHistory(name kv.Domain) *Aggregator { return a } -func (a *Aggregator) DiscardInvertedIndex(name kv.InvertedIdxPos) *Aggregator { - a.iis[name].discardIdx = true - return a -} - func (a *Aggregator) EnableHistory(name kv.Domain) *Aggregator { a.d[name].historyDisabled = false return a From 64af438da9b06806c5813685c438f4925bcb96df Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 15:42:05 +0200 Subject: [PATCH 34/46] save --- erigon-lib/state/domain.go | 2 -- erigon-lib/state/inverted_index.go | 3 +-- eth/stagedsync/exec3.go | 14 +++++++------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index b13292bc68f..e7fed829849 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1670,7 +1670,6 @@ func (dt *DomainRoTx) canPruneDomainTables(tx kv.Tx, untilTx uint64) (can bool, dt.d.logger.Error("get domain pruning progress", "name", dt.d.filenameBase, "error", err) return false, maxStepToPrune } - fmt.Println(sm, maxStepToPrune, untilStep) delta := float64(max(maxStepToPrune, sm) - min(maxStepToPrune, sm)) // maxStep could be 0 switch dt.d.filenameBase { @@ -1740,7 +1739,6 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT return nil, fmt.Errorf("prune history at step %d [%d, %d): %w", step, txFrom, txTo, err) } canPrune, maxPrunableStep := dt.canPruneDomainTables(rwTx, txTo) - fmt.Println("dt.canPruneDomainTables", canPrune, maxPrunableStep) if !canPrune { return stat, nil } diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index 66882e8ee54..d84289061c9 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -85,8 +85,7 @@ type InvertedIndex struct { // fields for history write logger log.Logger - noFsync bool // fsync is enabled by default, but tests can manually disable - discardIdx bool + noFsync bool // fsync is enabled by default, but tests can manually disable compression FileCompression compressWorkers int diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 75b9e156512..7843cdec87c 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -96,21 +96,21 @@ func (p *Progress) Log(rs *state.StateV3, in *state.QueueWithRetry, rws *state.R interval := currentTime.Sub(p.prevTime) speedTx := float64(doneCount-p.prevCount) / (float64(interval) / float64(time.Second)) //var repeatRatio float64 - logArgs := []interface{}{ + //if doneCount > p.prevCount { + // repeatRatio = 100.0 * float64(repeatCount-p.prevRepeatCount) / float64(doneCount-p.prevCount) + //} + p.logger.Info(fmt.Sprintf("[%s] Transaction replay", p.logPrefix), + //"workers", workerCount, "blk", outputBlockNum, "tx/s", fmt.Sprintf("%.1f", speedTx), //"pipe", fmt.Sprintf("(%d+%d)->%d/%d->%d/%d", in.NewTasksLen(), in.RetriesLen(), rws.ResultChLen(), rws.ResultChCap(), rws.Len(), rws.Limit()), //"repeatRatio", fmt.Sprintf("%.2f%%", repeatRatio), //"workers", p.workersCount, "buffer", fmt.Sprintf("%s/%s", common.ByteCount(sizeEstimate), common.ByteCount(p.commitThreshold)), + "stepsInDB", fmt.Sprintf("%.2f", idxStepsAmountInDB), "step", fmt.Sprintf("%.1f", float64(outTxNum)/float64(config3.HistoryV3AggregationStep)), "alloc", common.ByteCount(m.Alloc), "sys", common.ByteCount(m.Sys), - } - if idxStepsAmountInDB > 0.01 { - logArgs = append(logArgs, "stepsInDB", fmt.Sprintf("%.2f", idxStepsAmountInDB)) - } - - p.logger.Info(fmt.Sprintf("[%s] Transaction replay", p.logPrefix), logArgs...) + ) p.prevTime = currentTime p.prevCount = doneCount From 8ada3d14511047bf15cdb99633d4e5698b761de4 Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 15:43:11 +0200 Subject: [PATCH 35/46] save --- core/rawdb/rawdbhelpers/rawdbhelpers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/rawdb/rawdbhelpers/rawdbhelpers.go b/core/rawdb/rawdbhelpers/rawdbhelpers.go index c5af8c22080..e731db8a616 100644 --- a/core/rawdb/rawdbhelpers/rawdbhelpers.go +++ b/core/rawdb/rawdbhelpers/rawdbhelpers.go @@ -24,8 +24,8 @@ import ( ) func IdxStepsCountV3(tx kv.Tx) float64 { - fst, _ := kv.FirstKey(tx, kv.TblTracesToKeys) - lst, _ := kv.LastKey(tx, kv.TblTracesToKeys) + fst, _ := kv.FirstKey(tx, kv.TblAccountHistoryKeys) + lst, _ := kv.LastKey(tx, kv.TblAccountHistoryKeys) if len(fst) > 0 && len(lst) > 0 { fstTxNum := binary.BigEndian.Uint64(fst) lstTxNum := binary.BigEndian.Uint64(lst) From d0735e96199db843e33af42914c0aaff4e8c64be Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 15:47:01 +0200 Subject: [PATCH 36/46] save --- ethdb/prune/storage_mode.go | 2 +- turbo/snapshotsync/snapshotsync.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ethdb/prune/storage_mode.go b/ethdb/prune/storage_mode.go index 9b5a9603864..c889bbc4c96 100644 --- a/ethdb/prune/storage_mode.go +++ b/ethdb/prune/storage_mode.go @@ -23,8 +23,8 @@ import ( "math" "reflect" "strings" + "github.com/erigontech/erigon-lib/kv" - "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/params" ) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index b2138aed581..653917e1e15 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -273,7 +273,7 @@ func WaitForDownloader(ctx context.Context, logPrefix string, headerchain, blobs if err != nil { return err } - fmt.Println("minBlockToDownload", minBlockToDownload, "minStepToDownload", minStepToDownload, "blockPrune", blockPrune, "minStep", minStep) + blackListForPruning, err = buildBlackListForPruning(wantToPrune, minStepToDownload, minBlockToDownload, blockPrune, preverifiedBlockSnapshots) if err != nil { return err From 8f5acd4588599db404fc2e2af30d5bed835a3a77 Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 18:33:19 +0200 Subject: [PATCH 37/46] save --- cl/phase1/stages/clstages.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cl/phase1/stages/clstages.go b/cl/phase1/stages/clstages.go index 596acc37496..4b83c6b3f71 100644 --- a/cl/phase1/stages/clstages.go +++ b/cl/phase1/stages/clstages.go @@ -302,11 +302,25 @@ func ConsensusClStages(ctx context.Context, }) for i, block := range blocks { + blockRoot, err := block.Block.HashSSZ() + if err != nil { + logger.Warn("failed to hash block", "err", err) + blocks = blocks[i:] + break + } + if err := processBlock(cfg.indiciesDB, block, false, true, false); err != nil { log.Warn("bad blocks segment received", "err", err) blocks = blocks[i:] break } + + st, err := cfg.forkChoice.GetStateAtBlockRoot(blockRoot, false) + if err == nil { + if err := cfg.forkChoice.DumpBeaconStateOnDisk(st); err != nil { + logger.Warn("failed to dump state", "err", err) + } + } if shouldInsert && block.Version() >= clparams.BellatrixVersion { if err := cfg.blockCollector.AddBlock(block.Block); err != nil { logger.Warn("failed to add block to collector", "err", err) From 5320d7f4fadd12d4328ff2d4901d52f4683cd55e Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 18:34:13 +0200 Subject: [PATCH 38/46] save --- cl/phase1/stages/clstages.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cl/phase1/stages/clstages.go b/cl/phase1/stages/clstages.go index 4b83c6b3f71..63fe0379c22 100644 --- a/cl/phase1/stages/clstages.go +++ b/cl/phase1/stages/clstages.go @@ -316,7 +316,7 @@ func ConsensusClStages(ctx context.Context, } st, err := cfg.forkChoice.GetStateAtBlockRoot(blockRoot, false) - if err == nil { + if err == nil && block.Block.Slot%(cfg.beaconCfg.SlotsPerEpoch*2) == 0 { if err := cfg.forkChoice.DumpBeaconStateOnDisk(st); err != nil { logger.Warn("failed to dump state", "err", err) } From 8523a5c40616f6a5179bf1755aa10210831b2555 Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 19:08:53 +0200 Subject: [PATCH 39/46] save --- turbo/stages/blockchain_test.go | 53 +++++---------------------------- 1 file changed, 7 insertions(+), 46 deletions(-) diff --git a/turbo/stages/blockchain_test.go b/turbo/stages/blockchain_test.go index d153d4cdcde..9141009c422 100644 --- a/turbo/stages/blockchain_test.go +++ b/turbo/stages/blockchain_test.go @@ -24,13 +24,14 @@ import ( "encoding/binary" "errors" "fmt" - protosentry "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" - "github.com/erigontech/erigon/eth/protocols/eth" - "github.com/erigontech/erigon/rlp" "math" "math/big" "testing" + protosentry "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" + "github.com/erigontech/erigon/eth/protocols/eth" + "github.com/erigontech/erigon/rlp" + "github.com/erigontech/erigon-lib/common/hexutil" "github.com/erigontech/erigon-lib/log/v3" @@ -769,18 +770,8 @@ func TestModes(t *testing.T) { ) } -func TestBeforeModes(t *testing.T) { - t.Parallel() - mode := prune.DefaultMode - mode.History = prune.Before(0) - mode.Receipts = prune.Before(1) - mode.TxIndex = prune.Before(2) - mode.CallTraces = prune.Before(3) - doModesTest(t, mode) -} - func doModesTest(t *testing.T, pm prune.Mode) error { - fmt.Printf("h=%v, r=%v, t=%v\n", pm.History.Enabled(), pm.Receipts.Enabled(), pm.TxIndex.Enabled()) + fmt.Printf("h=%v\n", pm.History.Enabled()) require := require.New(t) // Configure and generate a sample block chain var ( @@ -849,32 +840,6 @@ func doModesTest(t *testing.T, pm prune.Mode) error { require.NoError(err) defer tx.Rollback() - if pm.Receipts.Enabled() { - receiptsAvailable, err := rawdb.ReceiptsAvailableFrom(tx) - require.NoError(err) - found := uint64(0) - err = tx.ForEach(kv.Receipts, nil, func(k, v []byte) error { - found++ - return nil - }) - require.NoError(err) - if m.HistoryV3 { - // receipts are not stored in erigon3 - } else { - require.GreaterOrEqual(receiptsAvailable, pm.Receipts.PruneTo(head)) - require.Greater(found, uint64(0)) - } - } else { - if m.HistoryV3 { - // receipts are not stored in erigon3 - } else { - receiptsAvailable, err := rawdb.ReceiptsAvailableFrom(tx) - require.NoError(err) - require.Equal(uint64(0), receiptsAvailable) - } - - } - if m.HistoryV3 { //TODO: e3 not implemented Prune feature yet /* @@ -916,7 +881,7 @@ func doModesTest(t *testing.T, pm prune.Mode) error { } } - if pm.TxIndex.Enabled() { + if pm.History.Enabled() { b, err := m.BlockReader.BlockByNumber(m.Ctx, tx, 1) require.NoError(err) for _, txn := range b.Transactions() { @@ -977,7 +942,7 @@ func runWithModesPermuations(t *testing.T, testFunc func(*testing.T, prune.Mode) } func runPermutation(t *testing.T, testFunc func(*testing.T, prune.Mode) error, current int, pm prune.Mode) error { - if current == 3 { + if current == 1 { return testFunc(t, pm) } if err := runPermutation(t, testFunc, current+1, pm); err != nil { @@ -992,10 +957,6 @@ func runPermutation(t *testing.T, testFunc func(*testing.T, prune.Mode) error, c switch current { case 0: pm.History = invert(pm.History) - case 1: - pm.Receipts = invert(pm.Receipts) - case 2: - pm.TxIndex = invert(pm.TxIndex) default: panic("unexpected current item") } From 1e6030b817acbada6da11d39645734230572f813 Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 22:33:20 +0200 Subject: [PATCH 40/46] save --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e01005c194c..8d05843ee15 100644 --- a/README.md +++ b/README.md @@ -791,7 +791,7 @@ Supported networks: all (except Mumbai). - datadir/chaindata is small now - to prevent it's grow: we recommend set `--batchSize <= 2G`. And it's fine to `rm -rf chaindata` - can symlink/mount latest state to fast drive and history to cheap drive -- ArchiveNode is default. FullNode same as in E2: --prune=hrtc +- Archive Node is default. Full Node: `--prune.mode=full`, Minimal Node: `--prune.mode=minimal` ### Known Problems of E3: From 2698ef7809c450345e4c0c56837af30b56698536 Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 22:49:55 +0200 Subject: [PATCH 41/46] save --- erigon-lib/state/domain.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index e7fed829849..a868b4e5b44 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1783,9 +1783,6 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT var k, v []byte if prunedKey != nil { k, v, err = valsCursor.Seek(prunedKey) - if err != nil { - return stat, err - } } else { k, v, err = valsCursor.First() } @@ -1793,6 +1790,7 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT return nil, err } var stepBytes []byte + a := 0 for ; k != nil; k, v, err = valsCursor.Next() { if err != nil { return stat, fmt.Errorf("iterate over %s domain keys: %w", dt.d.filenameBase, err) @@ -1817,6 +1815,7 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT } return stat, nil } + a++ limit-- if err := ancientDomainValsCollector.Collect(k, v); err != nil { return nil, err @@ -1833,6 +1832,7 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT default: } } + fmt.Println("pruned keys", a) mxPruneSizeDomain.AddUint64(stat.Values) if err := ancientDomainValsCollector.Load(rwTx, dt.d.valsTable, loadFunc, etl.TransformArgs{Quit: ctx.Done()}); err != nil { return stat, fmt.Errorf("load domain values: %w", err) From bb818694a76910ec64e54bf1cf71435536fe8101 Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 23:48:44 +0200 Subject: [PATCH 42/46] save --- erigon-lib/state/domain.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index a868b4e5b44..cfb57500ad8 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1810,6 +1810,7 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT if err := ancientDomainValsCollector.Load(rwTx, dt.d.valsTable, loadFunc, etl.TransformArgs{Quit: ctx.Done()}); err != nil { return stat, fmt.Errorf("load domain values: %w", err) } + fmt.Println("pruned keys", a) if err := SaveExecV3PruneProgress(rwTx, dt.d.valsTable, k); err != nil { return stat, fmt.Errorf("save domain pruning progress: %s, %w", dt.d.filenameBase, err) } @@ -1817,6 +1818,7 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT } a++ limit-- + stat.Values++ if err := ancientDomainValsCollector.Collect(k, v); err != nil { return nil, err } From 692eb5952cec990b12be3aef19cd0f198a556152 Mon Sep 17 00:00:00 2001 From: Giulio Date: Thu, 18 Jul 2024 23:49:32 +0200 Subject: [PATCH 43/46] save --- erigon-lib/state/domain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index cfb57500ad8..b01725017a5 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1810,7 +1810,7 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT if err := ancientDomainValsCollector.Load(rwTx, dt.d.valsTable, loadFunc, etl.TransformArgs{Quit: ctx.Done()}); err != nil { return stat, fmt.Errorf("load domain values: %w", err) } - fmt.Println("pruned keys", a) + fmt.Println("pruned keys B", dt.d.filenameBase, a) if err := SaveExecV3PruneProgress(rwTx, dt.d.valsTable, k); err != nil { return stat, fmt.Errorf("save domain pruning progress: %s, %w", dt.d.filenameBase, err) } @@ -1834,7 +1834,7 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT default: } } - fmt.Println("pruned keys", a) + fmt.Println("pruned keys A", dt.d.filenameBase, a) mxPruneSizeDomain.AddUint64(stat.Values) if err := ancientDomainValsCollector.Load(rwTx, dt.d.valsTable, loadFunc, etl.TransformArgs{Quit: ctx.Done()}); err != nil { return stat, fmt.Errorf("load domain values: %w", err) From b648f74462d8d1fddd96b84869cfbfd5b8365895 Mon Sep 17 00:00:00 2001 From: Giulio Date: Fri, 19 Jul 2024 00:15:44 +0200 Subject: [PATCH 44/46] save --- erigon-lib/state/domain.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index b01725017a5..36a55e27a38 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1781,7 +1781,7 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT } var k, v []byte - if prunedKey != nil { + if prunedKey != nil && limit < 100_000 { k, v, err = valsCursor.Seek(prunedKey) } else { k, v, err = valsCursor.First() @@ -1790,7 +1790,6 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT return nil, err } var stepBytes []byte - a := 0 for ; k != nil; k, v, err = valsCursor.Next() { if err != nil { return stat, fmt.Errorf("iterate over %s domain keys: %w", dt.d.filenameBase, err) @@ -1810,13 +1809,11 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT if err := ancientDomainValsCollector.Load(rwTx, dt.d.valsTable, loadFunc, etl.TransformArgs{Quit: ctx.Done()}); err != nil { return stat, fmt.Errorf("load domain values: %w", err) } - fmt.Println("pruned keys B", dt.d.filenameBase, a) if err := SaveExecV3PruneProgress(rwTx, dt.d.valsTable, k); err != nil { return stat, fmt.Errorf("save domain pruning progress: %s, %w", dt.d.filenameBase, err) } return stat, nil } - a++ limit-- stat.Values++ if err := ancientDomainValsCollector.Collect(k, v); err != nil { @@ -1834,7 +1831,6 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT default: } } - fmt.Println("pruned keys A", dt.d.filenameBase, a) mxPruneSizeDomain.AddUint64(stat.Values) if err := ancientDomainValsCollector.Load(rwTx, dt.d.valsTable, loadFunc, etl.TransformArgs{Quit: ctx.Done()}); err != nil { return stat, fmt.Errorf("load domain values: %w", err) From 4af3bac5c0b26d96c300640d289d1ac2d6874092 Mon Sep 17 00:00:00 2001 From: Giulio Date: Fri, 19 Jul 2024 00:43:24 +0200 Subject: [PATCH 45/46] save --- eth/stagedsync/exec3.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 7843cdec87c..68e1607646f 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -884,7 +884,8 @@ Loop: // } //} // If we skip post evaluation, then we should compute root hash ASAP for fail-fast - if !skipPostEvaluation && (rs.SizeEstimate() < commitThreshold || inMemExec) { + aggregatorRo := applyTx.(state2.HasAggTx).AggTx().(*state2.AggregatorRoTx) + if !skipPostEvaluation && (rs.SizeEstimate() < commitThreshold || inMemExec) && !aggregatorRo.CanPrune(applyTx, outputTxNum.Load()) { break } var ( @@ -903,7 +904,7 @@ Loop: t1 = time.Since(tt) + ts tt = time.Now() - if _, err := applyTx.(state2.HasAggTx).AggTx().(*state2.AggregatorRoTx).PruneSmallBatches(ctx, 10*time.Hour, applyTx); err != nil { + if _, err := aggregatorRo.PruneSmallBatches(ctx, 10*time.Hour, applyTx); err != nil { return err } t3 = time.Since(tt) From 678416b6bb578c92661bf34c553c673f6f2e7117 Mon Sep 17 00:00:00 2001 From: Giulio Date: Fri, 19 Jul 2024 02:04:54 +0200 Subject: [PATCH 46/46] save --- erigon-lib/state/domain.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 36a55e27a38..47bae813cce 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1819,7 +1819,8 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT if err := ancientDomainValsCollector.Collect(k, v); err != nil { return nil, err } - + stat.MinStep = min(stat.MinStep, is) + stat.MaxStep = max(stat.MaxStep, is) select { case <-ctx.Done(): // consider ctx exiting as incorrect outcome, error is returned