diff --git a/README.md b/README.md index e01005c194c..8d05843ee15 100644 --- a/README.md +++ b/README.md @@ -791,7 +791,7 @@ Supported networks: all (except Mumbai). - datadir/chaindata is small now - to prevent it's grow: we recommend set `--batchSize <= 2G`. And it's fine to `rm -rf chaindata` - can symlink/mount latest state to fast drive and history to cheap drive -- ArchiveNode is default. FullNode same as in E2: --prune=hrtc +- Archive Node is default. Full Node: `--prune.mode=full`, Minimal Node: `--prune.mode=minimal` ### Known Problems of E3: diff --git a/cl/phase1/stages/clstages.go b/cl/phase1/stages/clstages.go index 596acc37496..63fe0379c22 100644 --- a/cl/phase1/stages/clstages.go +++ b/cl/phase1/stages/clstages.go @@ -302,11 +302,25 @@ func ConsensusClStages(ctx context.Context, }) for i, block := range blocks { + blockRoot, err := block.Block.HashSSZ() + if err != nil { + logger.Warn("failed to hash block", "err", err) + blocks = blocks[i:] + break + } + if err := processBlock(cfg.indiciesDB, block, false, true, false); err != nil { log.Warn("bad blocks segment received", "err", err) blocks = blocks[i:] break } + + st, err := cfg.forkChoice.GetStateAtBlockRoot(blockRoot, false) + if err == nil && block.Block.Slot%(cfg.beaconCfg.SlotsPerEpoch*2) == 0 { + if err := cfg.forkChoice.DumpBeaconStateOnDisk(st); err != nil { + logger.Warn("failed to dump state", "err", err) + } + } if shouldInsert && block.Version() >= clparams.BellatrixVersion { if err := cfg.blockCollector.AddBlock(block.Block); err != nil { logger.Warn("failed to add block to collector", "err", err) diff --git a/cl/sentinel/discovery.go b/cl/sentinel/discovery.go index c95e4ab5dc3..53448906d0f 100644 --- a/cl/sentinel/discovery.go +++ b/cl/sentinel/discovery.go @@ -122,9 +122,12 @@ func (s *Sentinel) listenForPeers() { continue } - if err := s.ConnectWithPeer(s.ctx, *peerInfo); err != nil { - log.Trace("[Sentinel] Could not connect with peer", "err", err) - } + go func() { + if err := s.ConnectWithPeer(s.ctx, *peerInfo); err != nil { + log.Trace("[Sentinel] Could not connect with peer", "err", err) + } + }() + } } diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index fc1bc1e028b..8e552ffe1aa 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -1017,9 +1017,6 @@ func stageExec(db kv.RwDB, ctx context.Context, logger log.Logger) error { chainConfig, pm := fromdb.ChainConfig(db), fromdb.PruneMode(db) if pruneTo > 0 { pm.History = prune.Distance(s.BlockNumber - pruneTo) - pm.Receipts = prune.Distance(s.BlockNumber - pruneTo) - pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo) - pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo) } syncCfg := ethconfig.Defaults.Sync @@ -1123,9 +1120,6 @@ func stageCustomTrace(db kv.RwDB, ctx context.Context, logger log.Logger) error chainConfig, pm := fromdb.ChainConfig(db), fromdb.PruneMode(db) if pruneTo > 0 { pm.History = prune.Distance(s.BlockNumber - pruneTo) - pm.Receipts = prune.Distance(s.BlockNumber - pruneTo) - pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo) - pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo) } syncCfg := ethconfig.Defaults.Sync @@ -1244,9 +1238,6 @@ func stageTxLookup(db kv.RwDB, ctx context.Context, logger log.Logger) error { s := stage(sync, tx, nil, stages.TxLookup) if pruneTo > 0 { pm.History = prune.Distance(s.BlockNumber - pruneTo) - pm.Receipts = prune.Distance(s.BlockNumber - pruneTo) - pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo) - pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo) } logger.Info("Stage", "name", s.ID, "progress", s.BlockNumber) @@ -1547,8 +1538,7 @@ func stage(st *stagedsync.Sync, tx kv.Tx, db kv.RoDB, stage stages.SyncStage) *s func overrideStorageMode(db kv.RwDB, logger log.Logger) error { chainConfig := fromdb.ChainConfig(db) - pm, err := prune.FromCli(chainConfig.ChainID.Uint64(), pruneFlag, pruneB, pruneH, pruneR, pruneT, pruneC, - pruneHBefore, pruneRBefore, pruneTBefore, pruneCBefore, pruneBBefore, experiments) + pm, err := prune.FromCli(chainConfig.ChainID.Uint64(), pruneB, pruneH, experiments) if err != nil { return err } diff --git a/core/rawdb/rawdbhelpers/rawdbhelpers.go b/core/rawdb/rawdbhelpers/rawdbhelpers.go index c5af8c22080..e731db8a616 100644 --- a/core/rawdb/rawdbhelpers/rawdbhelpers.go +++ b/core/rawdb/rawdbhelpers/rawdbhelpers.go @@ -24,8 +24,8 @@ import ( ) func IdxStepsCountV3(tx kv.Tx) float64 { - fst, _ := kv.FirstKey(tx, kv.TblTracesToKeys) - lst, _ := kv.LastKey(tx, kv.TblTracesToKeys) + fst, _ := kv.FirstKey(tx, kv.TblAccountHistoryKeys) + lst, _ := kv.LastKey(tx, kv.TblAccountHistoryKeys) if len(fst) > 0 && len(lst) > 0 { fstTxNum := binary.BigEndian.Uint64(fst) lstTxNum := binary.BigEndian.Uint64(lst) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 8e656c404a9..9aac864c8c6 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -213,18 +213,30 @@ func (rs *StateV3) ApplyLogsAndTraces4(txTask *TxTask, domains *libstate.SharedD if dbg.DiscardHistory() { return nil } + shouldPruneNonEssentials := txTask.PruneNonEssentials && txTask.Config != nil for addr := range txTask.TraceFroms { + if shouldPruneNonEssentials && addr != txTask.Config.DepositContract { + continue + } if err := domains.IndexAdd(kv.TblTracesFromIdx, addr[:]); err != nil { return err } } + for addr := range txTask.TraceTos { + if shouldPruneNonEssentials && addr != txTask.Config.DepositContract { + continue + } if err := domains.IndexAdd(kv.TblTracesToIdx, addr[:]); err != nil { return err } } + for _, lg := range txTask.Logs { + if shouldPruneNonEssentials && lg.Address != txTask.Config.DepositContract { + continue + } if err := domains.IndexAdd(kv.TblLogAddressIdx, lg.Address[:]); err != nil { return err } diff --git a/core/state/txtask.go b/core/state/txtask.go index b82ef4bff4d..8b7707b626a 100644 --- a/core/state/txtask.go +++ b/core/state/txtask.go @@ -36,24 +36,25 @@ import ( // which is processed by a single thread that writes into the ReconState1 and // flushes to the database type TxTask struct { - TxNum uint64 - BlockNum uint64 - Rules *chain.Rules - Header *types.Header - Txs types.Transactions - Uncles []*types.Header - Coinbase libcommon.Address - Withdrawals types.Withdrawals - BlockHash libcommon.Hash - Sender *libcommon.Address - SkipAnalysis bool - TxIndex int // -1 for block initialisation - Final bool - Failed bool - Tx types.Transaction - GetHashFn func(n uint64) libcommon.Hash - TxAsMessage types.Message - EvmBlockContext evmtypes.BlockContext + TxNum uint64 + BlockNum uint64 + Rules *chain.Rules + Header *types.Header + Txs types.Transactions + Uncles []*types.Header + Coinbase libcommon.Address + Withdrawals types.Withdrawals + BlockHash libcommon.Hash + Sender *libcommon.Address + SkipAnalysis bool + PruneNonEssentials bool + TxIndex int // -1 for block initialisation + Final bool + Failed bool + Tx types.Transaction + GetHashFn func(n uint64) libcommon.Hash + TxAsMessage types.Message + EvmBlockContext evmtypes.BlockContext HistoryExecution bool // use history reader for that txn instead of state reader @@ -79,6 +80,7 @@ type TxTask struct { BlockReceipts types.Receipts Requests types.Requests + Config *chain.Config } func (t *TxTask) CreateReceipt(cumulativeGasUsed uint64) *types.Receipt { diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 9690613758b..0fc52e5e285 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -381,6 +381,7 @@ func (a *Aggregator) DiscardHistory(name kv.Domain) *Aggregator { a.d[name].historyDisabled = true return a } + func (a *Aggregator) EnableHistory(name kv.Domain) *Aggregator { a.d[name].historyDisabled = false return a diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index e7fed829849..47bae813cce 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1781,11 +1781,8 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT } var k, v []byte - if prunedKey != nil { + if prunedKey != nil && limit < 100_000 { k, v, err = valsCursor.Seek(prunedKey) - if err != nil { - return stat, err - } } else { k, v, err = valsCursor.First() } @@ -1818,10 +1815,12 @@ func (dt *DomainRoTx) Prune(ctx context.Context, rwTx kv.RwTx, step, txFrom, txT return stat, nil } limit-- + stat.Values++ if err := ancientDomainValsCollector.Collect(k, v); err != nil { return nil, err } - + stat.MinStep = min(stat.MinStep, is) + stat.MaxStep = max(stat.MaxStep, is) select { case <-ctx.Done(): // consider ctx exiting as incorrect outcome, error is returned diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 4aa51a78eea..68e1607646f 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -192,15 +192,13 @@ func ExecV3(ctx context.Context, if initialCycle { agg.SetCollateAndBuildWorkers(min(2, estimate.StateV3Collate.Workers())) agg.SetCompressWorkers(estimate.CompressSnapshot.Workers()) - //if blockNum < cfg.blockReader.FrozenBlocks() { - //defer agg.DiscardHistory(kv.CommitmentDomain).EnableHistory(kv.CommitmentDomain) - //defer agg.LimitRecentHistoryWithoutFiles(0).LimitRecentHistoryWithoutFiles(agg.StepSize() / 10) - //} } else { agg.SetCompressWorkers(1) agg.SetCollateAndBuildWorkers(1) } + pruneNonEssentials := cfg.prune.History.Enabled() && cfg.prune.History.PruneTo(execStage.BlockNumber) == execStage.BlockNumber + var err error inMemExec := txc.Doms != nil var doms *state2.SharedDomains @@ -718,26 +716,28 @@ Loop: for txIndex := -1; txIndex <= len(txs); txIndex++ { // Do not oversend, wait for the result heap to go under certain size txTask := &state.TxTask{ - BlockNum: blockNum, - Header: header, - Coinbase: b.Coinbase(), - Uncles: b.Uncles(), - Rules: rules, - Txs: txs, - TxNum: inputTxNum, - TxIndex: txIndex, - BlockHash: b.Hash(), - SkipAnalysis: skipAnalysis, - Final: txIndex == len(txs), - GetHashFn: getHashFn, - EvmBlockContext: blockContext, - Withdrawals: b.Withdrawals(), - Requests: b.Requests(), + BlockNum: blockNum, + Header: header, + Coinbase: b.Coinbase(), + Uncles: b.Uncles(), + Rules: rules, + Txs: txs, + TxNum: inputTxNum, + TxIndex: txIndex, + BlockHash: b.Hash(), + SkipAnalysis: skipAnalysis, + Final: txIndex == len(txs), + GetHashFn: getHashFn, + EvmBlockContext: blockContext, + Withdrawals: b.Withdrawals(), + Requests: b.Requests(), + PruneNonEssentials: pruneNonEssentials, // use history reader instead of state reader to catch up to the tx where we left off HistoryExecution: offsetFromBlockBeginning > 0 && txIndex < int(offsetFromBlockBeginning), BlockReceipts: receipts, + Config: cfg.genesis.Config, } if txTask.TxNum <= txNumInDB && txTask.TxNum > 0 { inputTxNum++ @@ -884,7 +884,8 @@ Loop: // } //} // If we skip post evaluation, then we should compute root hash ASAP for fail-fast - if !skipPostEvaluation && (rs.SizeEstimate() < commitThreshold || inMemExec) { + aggregatorRo := applyTx.(state2.HasAggTx).AggTx().(*state2.AggregatorRoTx) + if !skipPostEvaluation && (rs.SizeEstimate() < commitThreshold || inMemExec) && !aggregatorRo.CanPrune(applyTx, outputTxNum.Load()) { break } var ( @@ -903,7 +904,7 @@ Loop: t1 = time.Since(tt) + ts tt = time.Now() - if _, err := applyTx.(state2.HasAggTx).AggTx().(*state2.AggregatorRoTx).PruneSmallBatches(ctx, 10*time.Hour, applyTx); err != nil { + if _, err := aggregatorRo.PruneSmallBatches(ctx, 10*time.Hour, applyTx); err != nil { return err } t3 = time.Since(tt) diff --git a/eth/stagedsync/stage_call_traces.go b/eth/stagedsync/stage_call_traces.go index 5c5daa4114b..f145696011b 100644 --- a/eth/stagedsync/stage_call_traces.go +++ b/eth/stagedsync/stage_call_traces.go @@ -393,8 +393,8 @@ func PruneCallTraces(s *PruneState, tx kv.RwTx, cfg CallTracesCfg, ctx context.C defer tx.Rollback() } - if cfg.prune.CallTraces.Enabled() { - if err = pruneCallTraces(tx, logPrefix, cfg.prune.CallTraces.PruneTo(s.ForwardProgress), ctx, cfg.tmpdir, logger); err != nil { + if cfg.prune.History.Enabled() { + if err = pruneCallTraces(tx, logPrefix, cfg.prune.History.PruneTo(s.ForwardProgress), ctx, cfg.tmpdir, logger); err != nil { return err } } diff --git a/eth/stagedsync/stage_log_index.go b/eth/stagedsync/stage_log_index.go index e29e59ab490..3f3f64b03eb 100644 --- a/eth/stagedsync/stage_log_index.go +++ b/eth/stagedsync/stage_log_index.go @@ -102,7 +102,7 @@ func SpawnLogIndex(s *StageState, tx kv.RwTx, cfg LogIndexCfg, ctx context.Conte } startBlock := s.BlockNumber - pruneTo := cfg.prune.Receipts.PruneTo(endBlock) //endBlock - prune.r.older + pruneTo := cfg.prune.History.PruneTo(endBlock) //endBlock - prune.r.older // if startBlock < pruneTo { // startBlock = pruneTo // } @@ -435,7 +435,7 @@ func pruneOldLogChunks(tx kv.RwTx, bucket string, inMem *etl.Collector, pruneTo // Call pruneLogIndex with the current sync progresses and commit the data to db func PruneLogIndex(s *PruneState, tx kv.RwTx, cfg LogIndexCfg, ctx context.Context, logger log.Logger) (err error) { - if !cfg.prune.Receipts.Enabled() { + if !cfg.prune.History.Enabled() { return nil } logPrefix := s.LogPrefix() @@ -449,7 +449,7 @@ func PruneLogIndex(s *PruneState, tx kv.RwTx, cfg LogIndexCfg, ctx context.Conte defer tx.Rollback() } - pruneTo := cfg.prune.Receipts.PruneTo(s.ForwardProgress) + pruneTo := cfg.prune.History.PruneTo(s.ForwardProgress) if err = pruneLogIndex(logPrefix, tx, cfg.tmpdir, s.PruneProgress, pruneTo, ctx, logger, cfg.depositContract); err != nil { return err } diff --git a/eth/stagedsync/stage_senders.go b/eth/stagedsync/stage_senders.go index b8b4fdfd800..2a4b381a5b4 100644 --- a/eth/stagedsync/stage_senders.go +++ b/eth/stagedsync/stage_senders.go @@ -415,8 +415,8 @@ func PruneSendersStage(s *PruneState, tx kv.RwTx, cfg SendersCfg, ctx context.Co } if cfg.blockReader.FreezingCfg().Enabled { // noop. in this case senders will be deleted by BlockRetire.PruneAncientBlocks after data-freezing. - } else if cfg.prune.TxIndex.Enabled() { - to := cfg.prune.TxIndex.PruneTo(s.ForwardProgress) + } else if cfg.prune.History.Enabled() { + to := cfg.prune.History.PruneTo(s.ForwardProgress) if err = rawdb.PruneTable(tx, kv.Senders, to, ctx, 100); err != nil { return err } diff --git a/eth/stagedsync/stage_txlookup.go b/eth/stagedsync/stage_txlookup.go index 06b215f0208..3e3c39f274d 100644 --- a/eth/stagedsync/stage_txlookup.go +++ b/eth/stagedsync/stage_txlookup.go @@ -87,8 +87,8 @@ func SpawnTxLookup(s *StageState, tx kv.RwTx, toBlock uint64, cfg TxLookupCfg, c } startBlock := s.BlockNumber - if cfg.prune.TxIndex.Enabled() { - pruneTo := cfg.prune.TxIndex.PruneTo(endBlock) + if cfg.prune.History.Enabled() { + pruneTo := cfg.prune.History.PruneTo(endBlock) if startBlock < pruneTo { startBlock = pruneTo if err = s.UpdatePrune(tx, pruneTo); err != nil { // prune func of this stage will use this value to prevent all ancient blocks traversal @@ -248,8 +248,8 @@ func PruneTxLookup(s *PruneState, tx kv.RwTx, cfg TxLookupCfg, ctx context.Conte var pruneBor bool // Forward stage doesn't write anything before PruneTo point - if cfg.prune.TxIndex.Enabled() { - blockTo = cfg.prune.TxIndex.PruneTo(s.ForwardProgress) + if cfg.prune.History.Enabled() { + blockTo = cfg.prune.History.PruneTo(s.ForwardProgress) pruneBor = true } else if cfg.blockReader.FreezingCfg().Enabled { blockTo = cfg.blockReader.CanPruneTo(s.ForwardProgress) diff --git a/ethdb/prune/storage_mode.go b/ethdb/prune/storage_mode.go index 2b120484ef3..c889bbc4c96 100644 --- a/ethdb/prune/storage_mode.go +++ b/ethdb/prune/storage_mode.go @@ -17,7 +17,6 @@ package prune import ( - "bytes" "encoding/binary" "errors" "fmt" @@ -26,16 +25,12 @@ import ( "strings" "github.com/erigontech/erigon-lib/kv" - "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/params" ) var DefaultMode = Mode{ Initialised: true, - History: Distance(math.MaxUint64), // all off - Receipts: Distance(math.MaxUint64), - TxIndex: Distance(math.MaxUint64), - CallTraces: Distance(math.MaxUint64), + History: Distance(math.MaxUint64), Blocks: Distance(math.MaxUint64), Experiments: Experiments{}, // all off } @@ -43,60 +38,11 @@ var DefaultMode = Mode{ type Experiments struct { } -func FromCli(chainId uint64, flags string, exactBlocks, exactHistory, exactReceipts, exactTxIndex, exactCallTraces, - beforeB, beforeH, beforeR, beforeT, beforeC uint64, experiments []string) (Mode, error) { +func FromCli(chainId uint64, distanceHistory, distanceBlocks uint64, experiments []string) (Mode, error) { mode := DefaultMode - if flags != "default" && flags != "disabled" { - for _, flag := range flags { - switch flag { - case 'h': - mode.History = Distance(params.FullImmutabilityThreshold) - case 'r': - mode.Receipts = Distance(params.FullImmutabilityThreshold) - case 't': - mode.TxIndex = Distance(params.FullImmutabilityThreshold) - case 'c': - mode.CallTraces = Distance(params.FullImmutabilityThreshold) - case 'b': - mode.Blocks = Distance(params.FullImmutabilityThreshold) - default: - return DefaultMode, fmt.Errorf("unexpected flag found: %c", flag) - } - } - } - - if exactBlocks > 0 { - mode.Blocks = Distance(exactBlocks) - } - if exactHistory > 0 { - mode.History = Distance(exactHistory) - } - if exactReceipts > 0 { - mode.Receipts = Distance(exactReceipts) - } - if exactTxIndex > 0 { - mode.TxIndex = Distance(exactTxIndex) - } - if exactCallTraces > 0 { - mode.CallTraces = Distance(exactCallTraces) - } - - if beforeH > 0 { - mode.History = Before(beforeH) - } - if beforeR > 0 { - mode.Receipts = Before(beforeR) - } - if beforeT > 0 { - mode.TxIndex = Before(beforeT) - } - if beforeC > 0 { - mode.CallTraces = Before(beforeC) - } - if beforeB > 0 { - mode.Blocks = Before(beforeB) - } + mode.History = Distance(distanceHistory) + mode.Blocks = Distance(distanceBlocks) for _, ex := range experiments { switch ex { @@ -121,30 +67,6 @@ func Get(db kv.Getter) (Mode, error) { prune.History = blockAmount } - blockAmount, err = get(db, kv.PruneReceipts) - if err != nil { - return prune, err - } - if blockAmount != nil { - prune.Receipts = blockAmount - } - - blockAmount, err = get(db, kv.PruneTxIndex) - if err != nil { - return prune, err - } - if blockAmount != nil { - prune.TxIndex = blockAmount - } - - blockAmount, err = get(db, kv.PruneCallTraces) - if err != nil { - return prune, err - } - if blockAmount != nil { - prune.CallTraces = blockAmount - } - blockAmount, err = get(db, kv.PruneBlocks) if err != nil { return prune, err @@ -159,9 +81,6 @@ func Get(db kv.Getter) (Mode, error) { type Mode struct { Initialised bool // Set when the values are initialised (not default) History BlockAmount - Receipts BlockAmount - TxIndex BlockAmount - CallTraces BlockAmount Blocks BlockAmount Experiments Experiments } @@ -189,31 +108,12 @@ func (p Distance) useDefaultValue() bool { return uint64(p) == params.FullImmuta func (p Distance) dbType() []byte { return kv.PruneTypeOlder } func (p Distance) PruneTo(stageHead uint64) uint64 { - if p == 0 { - panic("pruning distance were not set") - } if uint64(p) > stageHead { return 0 } return stageHead - uint64(p) } -// Before number after which keep in DB -type Before uint64 - -func (b Before) Enabled() bool { return b > 0 } -func (b Before) toValue() uint64 { return uint64(b) } -func (b Before) useDefaultValue() bool { return uint64(b) == 0 } -func (b Before) dbType() []byte { return kv.PruneTypeBefore } - -func (b Before) PruneTo(uint64) uint64 { - if b == 0 { - return uint64(b) - } - - return uint64(b) - 1 -} - func (m Mode) String() string { if !m.Initialised { return "default" @@ -235,27 +135,6 @@ func (m Mode) String() string { long += fmt.Sprintf(" --prune.b.%s=%d", m.Blocks.dbType(), m.Blocks.toValue()) } } - if m.Receipts.Enabled() { - if m.Receipts.useDefaultValue() { - short += fmt.Sprintf(" --prune.r.older=%d", defaultVal) - } else { - long += fmt.Sprintf(" --prune.r.%s=%d", m.Receipts.dbType(), m.Receipts.toValue()) - } - } - if m.TxIndex.Enabled() { - if m.TxIndex.useDefaultValue() { - short += fmt.Sprintf(" --prune.t.older=%d", defaultVal) - } else { - long += fmt.Sprintf(" --prune.t.%s=%d", m.TxIndex.dbType(), m.TxIndex.toValue()) - } - } - if m.CallTraces.Enabled() { - if m.CallTraces.useDefaultValue() { - short += fmt.Sprintf(" --prune.c.older=%d", defaultVal) - } else { - long += fmt.Sprintf(" --prune.c.%s=%d", m.CallTraces.dbType(), m.CallTraces.toValue()) - } - } return strings.TrimLeft(short+long, " ") } @@ -270,21 +149,6 @@ func Override(db kv.RwTx, sm Mode) error { return err } - err = set(db, kv.PruneReceipts, sm.Receipts) - if err != nil { - return err - } - - err = set(db, kv.PruneTxIndex, sm.TxIndex) - if err != nil { - return err - } - - err = set(db, kv.PruneCallTraces, sm.CallTraces) - if err != nil { - return err - } - err = set(db, kv.PruneBlocks, sm.Blocks) if err != nil { return err @@ -306,17 +170,9 @@ func EnsureNotChanged(tx kv.GetPut, pruneMode Mode) (Mode, error) { } if pruneMode.Initialised { - // Don't change from previous default as default for Receipts pruning has now changed - if pruneMode.Receipts.useDefaultValue() { - pruneMode.Receipts = pm.Receipts - } // If storage mode is not explicitly specified, we take whatever is in the database if !reflect.DeepEqual(pm, pruneMode) { - if bytes.Equal(pm.Receipts.dbType(), kv.PruneTypeOlder) && bytes.Equal(pruneMode.Receipts.dbType(), kv.PruneTypeBefore) { - log.Error("--prune=r flag has been changed to mean pruning of receipts before the Beacon Chain genesis. Please re-sync Erigon from scratch. " + - "Alternatively, enforce the old behaviour explicitly by --prune.r.older=90000 flag at the risk of breaking the Consensus Layer.") - } return pm, errors.New("not allowed change of --prune flag, last time you used: " + pm.String()) } } @@ -332,11 +188,8 @@ func setIfNotExist(db kv.GetPut, pm Mode) error { } pruneDBData := map[string]BlockAmount{ - string(kv.PruneHistory): pm.History, - string(kv.PruneReceipts): pm.Receipts, - string(kv.PruneTxIndex): pm.TxIndex, - string(kv.PruneCallTraces): pm.CallTraces, - string(kv.PruneBlocks): pm.Blocks, + string(kv.PruneHistory): pm.History, + string(kv.PruneBlocks): pm.Blocks, } for key, value := range pruneDBData { @@ -355,8 +208,6 @@ func createBlockAmount(pruneType []byte, v []byte) (BlockAmount, error) { switch string(pruneType) { case string(kv.PruneTypeOlder): blockAmount = Distance(binary.BigEndian.Uint64(v)) - case string(kv.PruneTypeBefore): - blockAmount = Before(binary.BigEndian.Uint64(v)) default: return nil, fmt.Errorf("unexpected block amount type: %s", string(pruneType)) } diff --git a/ethdb/prune/storage_mode_test.go b/ethdb/prune/storage_mode_test.go index 29abec1f968..8e1066b0a83 100644 --- a/ethdb/prune/storage_mode_test.go +++ b/ethdb/prune/storage_mode_test.go @@ -17,7 +17,6 @@ package prune import ( - "math/rand" "strconv" "testing" @@ -30,17 +29,14 @@ func TestSetStorageModeIfNotExist(t *testing.T) { _, tx := memdb.NewTestTx(t) prune, err := Get(tx) assert.NoError(t, err) - assert.Equal(t, Mode{true, Distance(math.MaxUint64), Distance(math.MaxUint64), - Distance(math.MaxUint64), Distance(math.MaxUint64), Distance(math.MaxUint64), Experiments{}}, prune) + assert.Equal(t, Mode{true, Distance(math.MaxUint64), Distance(math.MaxUint64), Experiments{}}, prune) - err = setIfNotExist(tx, Mode{true, Distance(1), Distance(2), - Before(3), Before(4), Before(100), Experiments{}}) + err = setIfNotExist(tx, Mode{true, Distance(1), Distance(2), Experiments{}}) assert.NoError(t, err) prune, err = Get(tx) assert.NoError(t, err) - assert.Equal(t, Mode{true, Distance(1), Distance(2), - Before(3), Before(4), Before(100), Experiments{}}, prune) + assert.Equal(t, Mode{true, Distance(1), Distance(2), Experiments{}}, prune) } var distanceTests = []struct { @@ -67,25 +63,3 @@ func TestDistancePruneTo(t *testing.T) { }) } } - -var beforeTests = []struct { - pruneTo uint64 - expected uint64 -}{ - {0, 0}, - {1_000_000, 999_999}, -} - -func TestBeforePruneTo(t *testing.T) { - for _, tt := range beforeTests { - t.Run(strconv.FormatUint(tt.pruneTo, 10), func(t *testing.T) { - stageHead := uint64(rand.Int63n(10_000_000)) - b := Before(tt.pruneTo) - pruneTo := b.PruneTo(stageHead) - - if pruneTo != tt.expected { - t.Errorf("got %d, want %d", pruneTo, tt.expected) - } - }) - } -} diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go index adc992e752f..0eb2490483a 100644 --- a/turbo/cli/default_flags.go +++ b/turbo/cli/default_flags.go @@ -43,17 +43,9 @@ var DefaultFlags = []cli.Flag{ &utils.TxPoolLifetimeFlag, &utils.TxPoolTraceSendersFlag, &utils.TxPoolCommitEveryFlag, - &PruneFlag, - &PruneBlocksFlag, - &PruneHistoryFlag, - &PruneReceiptFlag, - &PruneTxIndexFlag, - &PruneCallTracesFlag, - &PruneBlocksBeforeFlag, - &PruneHistoryBeforeFlag, - &PruneReceiptBeforeFlag, - &PruneTxIndexBeforeFlag, - &PruneCallTracesBeforeFlag, + &PruneDistanceFlag, + &PruneBlocksDistanceFlag, + &PruneModeFlag, &BatchSizeFlag, &BodyCacheLimitFlag, &DatabaseVerbosityFlag, diff --git a/turbo/cli/flags.go b/turbo/cli/flags.go index fa81781ff40..66c8413bf23 100644 --- a/turbo/cli/flags.go +++ b/turbo/cli/flags.go @@ -82,62 +82,22 @@ var ( Value: kv.ReadersLimit - 128, } - PruneFlag = cli.StringFlag{ - Name: "prune", - Usage: `Choose which ancient data delete from DB: - h - prune history (ChangeSets, HistoryIndices - used by historical state access, like eth_getStorageAt, eth_getBalanceAt, debug_traceTransaction, trace_block, trace_transaction, etc.) - r - prune receipts (Receipts, Logs, LogTopicIndex, LogAddressIndex - used by eth_getLogs and similar RPC methods) - t - prune transaction by it's hash index - c - prune call traces (used by trace_filter method) - Does delete data older than 90K blocks, --prune=h is shortcut for: --prune.h.older=90000. - Similarly, --prune=t is shortcut for: --prune.t.older=90000 and --prune=c is shortcut for: --prune.c.older=90000. - However, --prune=r means to prune receipts before the Beacon Chain genesis (Consensus Layer might need receipts after that). - If an item is NOT on the list - means NO pruning for this data. - Example: --prune=htc`, - Value: "disabled", - } - PruneBlocksFlag = cli.Uint64Flag{ - Name: "prune.b.older", - Usage: `Prune data older than this number of blocks from the tip of the chain (if --prune flag has 'b', then default is 90K)`, - } - PruneHistoryFlag = cli.Uint64Flag{ - Name: "prune.h.older", - Usage: `Prune data older than this number of blocks from the tip of the chain (if --prune flag has 'h', then default is 90K)`, - } - PruneReceiptFlag = cli.Uint64Flag{ - Name: "prune.r.older", - Usage: `Prune data older than this number of blocks from the tip of the chain`, - } - PruneTxIndexFlag = cli.Uint64Flag{ - Name: "prune.t.older", - Usage: `Prune data older than this number of blocks from the tip of the chain (if --prune flag has 't', then default is 90K)`, - } - PruneCallTracesFlag = cli.Uint64Flag{ - Name: "prune.c.older", - Usage: `Prune data older than this number of blocks from the tip of the chain (if --prune flag has 'c', then default is 90K)`, - } - - PruneHistoryBeforeFlag = cli.Uint64Flag{ - Name: "prune.h.before", - Usage: `Prune data before this block`, - } - PruneReceiptBeforeFlag = cli.Uint64Flag{ - Name: "prune.r.before", - Usage: `Prune data before this block`, - } - PruneTxIndexBeforeFlag = cli.Uint64Flag{ - Name: "prune.t.before", - Usage: `Prune data before this block`, - } - PruneCallTracesBeforeFlag = cli.Uint64Flag{ - Name: "prune.c.before", - Usage: `Prune data before this block`, - } - PruneBlocksBeforeFlag = cli.Uint64Flag{ - Name: "prune.b.before", - Usage: `Prune data before this block`, + PruneModeFlag = cli.StringFlag{ + Name: "prune.mode", + Usage: `Choose a pruning preset to run onto. Avaiable values: "archive","full","minimal". + Archive: Keep the entire indexed database, aka. no pruning. (Pruning is flexible), + Full: Keep only blocks and latest state (Pruning is not flexible) + Minimal: Keep only latest state (Pruning is not flexible)`, + Value: "archive", + } + PruneDistanceFlag = cli.Uint64Flag{ + Name: "prune.distance", + Usage: `Keep state history for the latest N blocks (default: everything)`, + } + PruneBlocksDistanceFlag = cli.Uint64Flag{ + Name: "prune.distance.blocks", + Usage: `Keep block history for the latest N blocks (default: everything)`, } - ExperimentsFlag = cli.StringFlag{ Name: "experiments", Usage: `Enable some experimental stages: @@ -312,29 +272,38 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config, logger log. if cfg.Genesis != nil { chainId = cfg.Genesis.Config.ChainID.Uint64() } - minimal := ctx.String(PruneFlag.Name) == "minimal" - pruneFlagString := ctx.String(PruneFlag.Name) - if minimal { - pruneFlagString = "htrcb" + // Sanitize prune flag + if ctx.String(PruneModeFlag.Name) != "archive" && (ctx.IsSet(PruneBlocksDistanceFlag.Name) || ctx.IsSet(PruneDistanceFlag.Name)) { + utils.Fatalf(fmt.Sprintf("error: --prune.distance and --prune.distance.blocks are only allowed with --prune.mode=archive")) + } + distance := ctx.Uint64(PruneDistanceFlag.Name) + blockDistance := ctx.Uint64(PruneBlocksDistanceFlag.Name) + + if !ctx.IsSet(PruneBlocksDistanceFlag.Name) { + blockDistance = math.MaxUint64 + } + if !ctx.IsSet(PruneDistanceFlag.Name) { + distance = math.MaxUint64 } mode, err := prune.FromCli( chainId, - pruneFlagString, - ctx.Uint64(PruneBlocksFlag.Name), - ctx.Uint64(PruneHistoryFlag.Name), - ctx.Uint64(PruneReceiptFlag.Name), - ctx.Uint64(PruneTxIndexFlag.Name), - ctx.Uint64(PruneCallTracesFlag.Name), - ctx.Uint64(PruneHistoryBeforeFlag.Name), - ctx.Uint64(PruneReceiptBeforeFlag.Name), - ctx.Uint64(PruneTxIndexBeforeFlag.Name), - ctx.Uint64(PruneCallTracesBeforeFlag.Name), - ctx.Uint64(PruneBlocksBeforeFlag.Name), + distance, + blockDistance, libcommon.CliString2Array(ctx.String(ExperimentsFlag.Name)), ) if err != nil { utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) } + // Full mode prunes all but the latest state + if ctx.String(PruneModeFlag.Name) == "full" { + mode.Blocks = prune.Distance(math.MaxUint64) + mode.History = prune.Distance(0) + } + // Minimal mode prunes all but the latest state including blocks + if ctx.String(PruneModeFlag.Name) == "minimal" { + mode.Blocks = prune.Distance(2048) // 2048 is just some blocks to allow reorgs + mode.History = prune.Distance(0) + } if err != nil { utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) @@ -357,15 +326,6 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config, logger log. etl.BufferOptimalSize = *size } - if minimal { - // Prune them all. - cfg.Prune.Blocks = prune.Before(math.MaxUint64) - cfg.Prune.History = prune.Before(math.MaxUint64) - cfg.Prune.Receipts = prune.Before(math.MaxUint64) - cfg.Prune.TxIndex = prune.Before(math.MaxUint64) - cfg.Prune.CallTraces = prune.Before(math.MaxUint64) - } - cfg.StateStream = !ctx.Bool(StateStreamDisableFlag.Name) if ctx.String(BodyCacheLimitFlag.Name) != "" { err := cfg.Sync.BodyCacheLimit.UnmarshalText([]byte(ctx.String(BodyCacheLimitFlag.Name))) @@ -433,56 +393,49 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config, logger log. } func ApplyFlagsForEthConfigCobra(f *pflag.FlagSet, cfg *ethconfig.Config) { - if v := f.String(PruneFlag.Name, PruneFlag.Value, PruneFlag.Usage); v != nil { - var experiments []string - if exp := f.StringSlice(ExperimentsFlag.Name, nil, ExperimentsFlag.Usage); exp != nil { - experiments = *exp - } - var exactB, exactH, exactR, exactT, exactC uint64 - if v := f.Uint64(PruneBlocksFlag.Name, PruneBlocksFlag.Value, PruneBlocksFlag.Usage); v != nil { - exactB = *v - } - if v := f.Uint64(PruneHistoryFlag.Name, PruneHistoryFlag.Value, PruneHistoryFlag.Usage); v != nil { - exactH = *v - } - if v := f.Uint64(PruneReceiptFlag.Name, PruneReceiptFlag.Value, PruneReceiptFlag.Usage); v != nil { - exactR = *v - } - if v := f.Uint64(PruneTxIndexFlag.Name, PruneTxIndexFlag.Value, PruneTxIndexFlag.Usage); v != nil { - exactT = *v - } - if v := f.Uint64(PruneCallTracesFlag.Name, PruneCallTracesFlag.Value, PruneCallTracesFlag.Usage); v != nil { - exactC = *v - } - - var beforeB, beforeH, beforeR, beforeT, beforeC uint64 - if v := f.Uint64(PruneBlocksBeforeFlag.Name, PruneBlocksBeforeFlag.Value, PruneBlocksBeforeFlag.Usage); v != nil { - beforeB = *v - } - if v := f.Uint64(PruneHistoryBeforeFlag.Name, PruneHistoryBeforeFlag.Value, PruneHistoryBeforeFlag.Usage); v != nil { - beforeH = *v - } - if v := f.Uint64(PruneReceiptBeforeFlag.Name, PruneReceiptBeforeFlag.Value, PruneReceiptBeforeFlag.Usage); v != nil { - beforeR = *v - } - if v := f.Uint64(PruneTxIndexBeforeFlag.Name, PruneTxIndexBeforeFlag.Value, PruneTxIndexBeforeFlag.Usage); v != nil { - beforeT = *v - } - if v := f.Uint64(PruneCallTracesBeforeFlag.Name, PruneCallTracesBeforeFlag.Value, PruneCallTracesBeforeFlag.Usage); v != nil { - beforeC = *v - } + pruneMode := f.String(PruneModeFlag.Name, PruneModeFlag.DefaultText, PruneModeFlag.Usage) + pruneBlockDistance := f.Uint64(PruneBlocksDistanceFlag.Name, PruneBlocksDistanceFlag.Value, PruneBlocksDistanceFlag.Usage) + pruneDistance := f.Uint64(PruneDistanceFlag.Name, PruneDistanceFlag.Value, PruneDistanceFlag.Usage) - chainId := cfg.NetworkID - if cfg.Genesis != nil { - chainId = cfg.Genesis.Config.ChainID.Uint64() - } + chainId := cfg.NetworkID + if *pruneMode != "archive" && (pruneBlockDistance != nil || pruneDistance != nil) { + utils.Fatalf(fmt.Sprintf("error: --prune.distance and --prune.distance.blocks are only allowed with --prune.mode=archive")) + } + var distance, blockDistance uint64 = math.MaxUint64, math.MaxUint64 + if pruneBlockDistance != nil { + blockDistance = *pruneBlockDistance + } + if pruneDistance != nil { + distance = *pruneDistance + } - mode, err := prune.FromCli(chainId, *v, exactB, exactH, exactR, exactT, exactC, beforeH, beforeR, beforeT, beforeC, beforeB, experiments) - if err != nil { - utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) - } - cfg.Prune = mode + experiments := f.String(ExperimentsFlag.Name, ExperimentsFlag.Value, ExperimentsFlag.Usage) + experimentsVal := "" + if experiments != nil { + experimentsVal = *experiments } + mode, err := prune.FromCli( + chainId, + distance, + blockDistance, + libcommon.CliString2Array(experimentsVal), + ) + if err != nil { + utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) + } + switch *pruneMode { + case "archive": + case "full": + mode.Blocks = prune.Distance(math.MaxUint64) + mode.History = prune.Distance(0) + case "minimal": + mode.Blocks = prune.Distance(2048) // 2048 is just some blocks to allow reorgs + mode.History = prune.Distance(0) + default: + utils.Fatalf("error: --prune.mode must be one of archive, full, minimal") + } + cfg.Prune = mode + if v := f.String(BatchSizeFlag.Name, BatchSizeFlag.Value, BatchSizeFlag.Usage); v != nil { err := cfg.BatchSize.UnmarshalText([]byte(*v)) if err != nil { diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index ce42163aea7..653917e1e15 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -22,7 +22,6 @@ import ( "fmt" "math" "runtime" - "sort" "strconv" "strings" "time" @@ -106,15 +105,10 @@ func adjustBlockPrune(blocks, minBlocksToDownload uint64) uint64 { if minBlocksToDownload < snaptype.Erigon2MergeLimit { minBlocksToDownload = snaptype.Erigon2MergeLimit } - if blocks < minBlocksToDownload { + if blocks > minBlocksToDownload { blocks = minBlocksToDownload } - if blocks%snaptype.Erigon2MergeLimit == 0 { - return blocks - } - ret := blocks + snaptype.Erigon2MergeLimit - // round to nearest multiple of 64. if less than 64, round to 64 - return ret - ret%snaptype.Erigon2MergeLimit + return blocks - blocks%snaptype.Erigon2MergeLimit } func shouldUseStepsForPruning(name string) bool { @@ -126,96 +120,65 @@ func canSnapshotBePruned(name string) bool { } func buildBlackListForPruning(pruneMode bool, stepPrune, minBlockToDownload, blockPrune uint64, preverified snapcfg.Preverified) (map[string]struct{}, error) { - type snapshotFileData struct { - from, to uint64 - stepBased bool - name string - } + blackList := make(map[string]struct{}) if !pruneMode { return blackList, nil } stepPrune = adjustStepPrune(stepPrune) blockPrune = adjustBlockPrune(blockPrune, minBlockToDownload) - snapshotKindToNames := make(map[string][]snapshotFileData) for _, p := range preverified { name := p.Name // Don't prune unprunable files if !canSnapshotBePruned(name) { continue } - var from, to uint64 + var _, to uint64 var err error - var kind string if shouldUseStepsForPruning(name) { // parse "from" (0) and "to" (64) from the name // parse the snapshot "kind". e.g kind of 'idx/v1-accounts.0-64.ef' is "idx/v1-accounts" rangeString := strings.Split(name, ".")[1] rangeNums := strings.Split(rangeString, "-") // convert the range to uint64 - from, err = strconv.ParseUint(rangeNums[0], 10, 64) - if err != nil { - return nil, err - } to, err = strconv.ParseUint(rangeNums[1], 10, 64) if err != nil { return nil, err } - kind = strings.Split(name, ".")[0] + if stepPrune < to { + continue + } + blackList[name] = struct{}{} } else { // e.g 'v1-000000-000100-beaconblocks.seg' // parse "from" (000000) and "to" (000100) from the name. 100 is 100'000 blocks - minusSplit := strings.Split(name, "-") s, _, ok := snaptype.ParseFileName("", name) if !ok { continue } - from = s.From to = s.To - kind = minusSplit[3] - } - blackList[p.Name] = struct{}{} // Add all of them to the blacklist and remove the ones that are not blacklisted later. - snapshotKindToNames[kind] = append(snapshotKindToNames[kind], snapshotFileData{ - from: from, - to: to, - stepBased: shouldUseStepsForPruning(name), - name: name, - }) - } - // sort the snapshots by "from" and "to" in ascending order - for _, snapshots := range snapshotKindToNames { - prunedDistance := uint64(0) // keep track of pruned distance for snapshots - // sort the snapshots by "from" and "to" in descending order - sort.Slice(snapshots, func(i, j int) bool { - if snapshots[i].from == snapshots[j].from { - return snapshots[i].to > snapshots[j].to - } - return snapshots[i].from > snapshots[j].from - }) - for _, snapshot := range snapshots { - if snapshot.stepBased { - if prunedDistance >= stepPrune { - break - } - } else if prunedDistance >= blockPrune { - break + if blockPrune < to { + continue } - delete(blackList, snapshot.name) - prunedDistance += snapshot.to - snapshot.from + blackList[name] = struct{}{} } } + return blackList, nil } // getMinimumBlocksToDownload - get the minimum number of blocks to download -func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, minStep uint64, expectedPruneBlockAmount, expectedPruneHistoryAmount uint64) (uint64, uint64, error) { +func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, minStep uint64, blockPruneTo, historyPruneTo uint64) (uint64, uint64, error) { frozenBlocks := blockReader.Snapshots().SegmentsMax() minToDownload := uint64(math.MaxUint64) - minStepToDownload := minStep + minStepToDownload := uint64(math.MaxUint32) stateTxNum := minStep * config3.HistoryV3AggregationStep if err := blockReader.IterateFrozenBodies(func(blockNum, baseTxNum, txAmount uint64) error { - if blockNum == frozenBlocks-expectedPruneHistoryAmount { - minStepToDownload = (baseTxNum / config3.HistoryV3AggregationStep) - 1 + if blockNum == historyPruneTo { + minStepToDownload = (baseTxNum - (config3.HistoryV3AggregationStep - 1)) / config3.HistoryV3AggregationStep + if baseTxNum < (config3.HistoryV3AggregationStep - 1) { + minStepToDownload = 0 + } } if stateTxNum <= baseTxNum { // only cosnider the block if it return nil @@ -231,11 +194,9 @@ func getMinimumBlocksToDownload(tx kv.Tx, blockReader services.FullBlockReader, }); err != nil { return 0, 0, err } - if expectedPruneBlockAmount == 0 { - return minToDownload, 0, nil - } + // return the minimum number of blocks to download and the minimum step. - return minToDownload, minStep - minStepToDownload, nil + return frozenBlocks - minToDownload, minStepToDownload, nil } func getMaxStepRangeInSnapshots(preverified snapcfg.Preverified) (uint64, error) { @@ -261,15 +222,7 @@ func getMaxStepRangeInSnapshots(preverified snapcfg.Preverified) (uint64, error) func computeBlocksToPrune(blockReader services.FullBlockReader, p prune.Mode) (blocksToPrune uint64, historyToPrune uint64) { frozenBlocks := blockReader.Snapshots().SegmentsMax() - blocksPruneTo := p.Blocks.PruneTo(frozenBlocks) - historyPruneTo := p.History.PruneTo(frozenBlocks) - if blocksPruneTo <= frozenBlocks { - blocksToPrune = frozenBlocks - blocksPruneTo - } - if historyPruneTo <= frozenBlocks { - historyToPrune = frozenBlocks - historyPruneTo - } - return blocksToPrune, historyToPrune + return p.Blocks.PruneTo(frozenBlocks), p.History.PruneTo(frozenBlocks) } // WaitForDownloader - wait for Downloader service to download all expected snapshots @@ -316,11 +269,12 @@ func WaitForDownloader(ctx context.Context, logPrefix string, headerchain, blobs if err != nil { return err } - minBlockAmountToDownload, minStepToDownload, err := getMinimumBlocksToDownload(tx, blockReader, minStep, blockPrune, historyPrune) + minBlockToDownload, minStepToDownload, err := getMinimumBlocksToDownload(tx, blockReader, minStep, blockPrune, historyPrune) if err != nil { return err } - blackListForPruning, err = buildBlackListForPruning(wantToPrune, minStepToDownload, minBlockAmountToDownload, blockPrune, preverifiedBlockSnapshots) + + blackListForPruning, err = buildBlackListForPruning(wantToPrune, minStepToDownload, minBlockToDownload, blockPrune, preverifiedBlockSnapshots) if err != nil { return err } diff --git a/turbo/stages/blockchain_test.go b/turbo/stages/blockchain_test.go index d153d4cdcde..9141009c422 100644 --- a/turbo/stages/blockchain_test.go +++ b/turbo/stages/blockchain_test.go @@ -24,13 +24,14 @@ import ( "encoding/binary" "errors" "fmt" - protosentry "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" - "github.com/erigontech/erigon/eth/protocols/eth" - "github.com/erigontech/erigon/rlp" "math" "math/big" "testing" + protosentry "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" + "github.com/erigontech/erigon/eth/protocols/eth" + "github.com/erigontech/erigon/rlp" + "github.com/erigontech/erigon-lib/common/hexutil" "github.com/erigontech/erigon-lib/log/v3" @@ -769,18 +770,8 @@ func TestModes(t *testing.T) { ) } -func TestBeforeModes(t *testing.T) { - t.Parallel() - mode := prune.DefaultMode - mode.History = prune.Before(0) - mode.Receipts = prune.Before(1) - mode.TxIndex = prune.Before(2) - mode.CallTraces = prune.Before(3) - doModesTest(t, mode) -} - func doModesTest(t *testing.T, pm prune.Mode) error { - fmt.Printf("h=%v, r=%v, t=%v\n", pm.History.Enabled(), pm.Receipts.Enabled(), pm.TxIndex.Enabled()) + fmt.Printf("h=%v\n", pm.History.Enabled()) require := require.New(t) // Configure and generate a sample block chain var ( @@ -849,32 +840,6 @@ func doModesTest(t *testing.T, pm prune.Mode) error { require.NoError(err) defer tx.Rollback() - if pm.Receipts.Enabled() { - receiptsAvailable, err := rawdb.ReceiptsAvailableFrom(tx) - require.NoError(err) - found := uint64(0) - err = tx.ForEach(kv.Receipts, nil, func(k, v []byte) error { - found++ - return nil - }) - require.NoError(err) - if m.HistoryV3 { - // receipts are not stored in erigon3 - } else { - require.GreaterOrEqual(receiptsAvailable, pm.Receipts.PruneTo(head)) - require.Greater(found, uint64(0)) - } - } else { - if m.HistoryV3 { - // receipts are not stored in erigon3 - } else { - receiptsAvailable, err := rawdb.ReceiptsAvailableFrom(tx) - require.NoError(err) - require.Equal(uint64(0), receiptsAvailable) - } - - } - if m.HistoryV3 { //TODO: e3 not implemented Prune feature yet /* @@ -916,7 +881,7 @@ func doModesTest(t *testing.T, pm prune.Mode) error { } } - if pm.TxIndex.Enabled() { + if pm.History.Enabled() { b, err := m.BlockReader.BlockByNumber(m.Ctx, tx, 1) require.NoError(err) for _, txn := range b.Transactions() { @@ -977,7 +942,7 @@ func runWithModesPermuations(t *testing.T, testFunc func(*testing.T, prune.Mode) } func runPermutation(t *testing.T, testFunc func(*testing.T, prune.Mode) error, current int, pm prune.Mode) error { - if current == 3 { + if current == 1 { return testFunc(t, pm) } if err := runPermutation(t, testFunc, current+1, pm); err != nil { @@ -992,10 +957,6 @@ func runPermutation(t *testing.T, testFunc func(*testing.T, prune.Mode) error, c switch current { case 0: pm.History = invert(pm.History) - case 1: - pm.Receipts = invert(pm.Receipts) - case 2: - pm.TxIndex = invert(pm.TxIndex) default: panic("unexpected current item") }