ADR 101: Implement pruning mechanism (cometbft#1150)

* Added pruning mechanism for blocks and abci block results
* Added support for datacompanion and application retain heights
---------

Signed-off-by: Thane Thomson <connect@thanethomson.com>
Co-authored-by: Thane Thomson <connect@thanethomson.com>

.changelog/unreleased/breaking-changes/1096-state-store-adr101-methods.md

@@ -0,0 +1,3 @@
+- `[state]` The `state.Store` interface has been expanded
+  to accommodate the data pull companion API of ADR 101
+  ([\#1096](https://github.com/cometbft/cometbft/issues/1096))

.changelog/unreleased/enhancements/1096-config-pruner.md

@@ -0,0 +1,3 @@
+- `[config]` Added `[storage.pruning]` and `[storage.pruning.data_companion]`
+  sections to facilitate background pruning and data companion (ADR 101)
+  operations ([\#1096](https://github.com/cometbft/cometbft/issues/1096))

.changelog/unreleased/enhancements/1096-node-pruner.md

@@ -0,0 +1,3 @@
+- `[node]` The `node.Node` struct now manages a
+  `state.Pruner` service to facilitate background pruning
+  ([\#1096](https://github.com/cometbft/cometbft/issues/1096))

.changelog/unreleased/enhancements/1096-state-abcires-pruning.md

@@ -0,0 +1,2 @@
+- `[state]` ABCI response pruning has been added for use by the data companion
+  ([\#1096](https://github.com/cometbft/cometbft/issues/1096))

.changelog/unreleased/enhancements/1096-state-background-pruning.md

@@ -0,0 +1,2 @@
+- `[state]` Block pruning has been moved from the block executor into a
+  background process ([\#1096](https://github.com/cometbft/cometbft/issues/1096))

config/config.go

@@ -28,6 +28,7 @@ const (
 	// Default is v0.
 	MempoolV0 = "v0"
 	MempoolV1 = "v1"
+	DefaultPruningInterval = 10 * time.Second
 )
 
 // NOTE: Most of the structs & relevant comments + the
@@ -148,6 +149,9 @@ func (cfg *Config) ValidateBasic() error {
 	if err := cfg.Consensus.ValidateBasic(); err != nil {
 		return fmt.Errorf("error in [consensus] section: %w", err)
 	}
+	if err := cfg.Storage.ValidateBasic(); err != nil {
+		return fmt.Errorf("error in [storage] section: %w", err)
+	}
 	if err := cfg.Instrumentation.ValidateBasic(); err != nil {
 		return fmt.Errorf("error in [instrumentation] section: %w", err)
 	}
@@ -1134,13 +1138,16 @@ type StorageConfig struct {
 	// required for `/block_results` RPC queries, and to reindex events in the
 	// command-line tool.
 	DiscardABCIResponses bool `mapstructure:"discard_abci_responses"`
+	// Configuration related to storage pruning.
+	Pruning *PruningConfig `mapstructure:"pruning"`
 }
 
 // DefaultStorageConfig returns the default configuration options relating to
 // CometBFT storage optimization.
 func DefaultStorageConfig() *StorageConfig {
 	return &StorageConfig{
 		DiscardABCIResponses: false,
+		Pruning:              DefaultPruningConfig(),
 	}
 }
 
@@ -1149,9 +1156,17 @@ func DefaultStorageConfig() *StorageConfig {
 func TestStorageConfig() *StorageConfig {
 	return &StorageConfig{
 		DiscardABCIResponses: false,
+		Pruning:              TestPruningConfig(),
 	}
 }
 
+func (cfg *StorageConfig) ValidateBasic() error {
+	if err := cfg.Pruning.ValidateBasic(); err != nil {
+		return fmt.Errorf("error in [pruning] section: %w", err)
+	}
+	return nil
+}
+
 // -----------------------------------------------------------------------------
 // TxIndexConfig
 // Remember that Event has the following structure:
@@ -1266,3 +1281,89 @@ func getDefaultMoniker() string {
 	}
 	return moniker
 }
+
+//-----------------------------------------------------------------------------
+// PruningConfig
+
+type PruningConfig struct {
+	// The time period between automated background pruning operations.
+	Interval time.Duration `mapstructure:"period"`
+	// Data companion-related pruning configuration.
+	DataCompanion *DataCompanionPruningConfig `mapstructure:"data_companion"`
+}
+
+func DefaultPruningConfig() *PruningConfig {
+	return &PruningConfig{
+		Interval:      DefaultPruningInterval,
+		DataCompanion: DefaultDataCompanionPruningConfig(),
+	}
+}
+
+func TestPruningConfig() *PruningConfig {
+	return &PruningConfig{
+		Interval:      DefaultPruningInterval,
+		DataCompanion: TestDataCompanionPruningConfig(),
+	}
+}
+
+func (cfg *PruningConfig) ValidateBasic() error {
+	if cfg.Interval <= 0 {
+		return errors.New("interval must be > 0")
+	}
+	if err := cfg.DataCompanion.ValidateBasic(); err != nil {
+		return fmt.Errorf("error in [data_companion] section: %w", err)
+	}
+	return nil
+}
+
+//-----------------------------------------------------------------------------
+// DataCompanionPruningConfig
+
+type DataCompanionPruningConfig struct {
+	// Whether automatic pruning respects values set by the data companion.
+	// Disabled by default. All other parameters in this section are ignored
+	// when this is disabled.
+	//
+	// If disabled, only the application retain height will influence block
+	// pruning (but not block results pruning). Only enabling this at a later
+	// stage will potentially mean that blocks below the application-set retain
+	// height at the time will not be available to the data companion.
+	Enabled bool `mapstructure:"enabled"`
+	// The initial value for the data companion block retain height if the data
+	// companion has not yet explicitly set one. If the data companion has
+	// already set a block retain height, this is ignored.
+	InitialBlockRetainHeight int64 `mapstructure:"initial_block_retain_height"`
+	// The initial value for the data companion block results retain height if
+	// the data companion has not yet explicitly set one. If the data companion
+	// has already set a block results retain height, this is ignored.
+	InitialBlockResultsRetainHeight int64 `mapstructure:"initial_block_results_retain_height"`
+}
+
+func DefaultDataCompanionPruningConfig() *DataCompanionPruningConfig {
+	return &DataCompanionPruningConfig{
+		Enabled:                         false,
+		InitialBlockRetainHeight:        0,
+		InitialBlockResultsRetainHeight: 0,
+	}
+}
+
+func TestDataCompanionPruningConfig() *DataCompanionPruningConfig {
+	return &DataCompanionPruningConfig{
+		Enabled:                         false,
+		InitialBlockRetainHeight:        0,
+		InitialBlockResultsRetainHeight: 0,
+	}
+}
+
+func (cfg *DataCompanionPruningConfig) ValidateBasic() error {
+	if !cfg.Enabled {
+		return nil
+	}
+	if cfg.InitialBlockRetainHeight < 0 {
+		return errors.New("initial_block_retain_height cannot be negative")
+	}
+	if cfg.InitialBlockResultsRetainHeight < 0 {
+		return errors.New("initial_block_results_retain_height cannot be negative")
+	}
+	return nil
+}

config/toml.go

@@ -491,6 +491,36 @@ peer_query_maj23_sleep_duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}"
 # reindex events in the command-line tool.
 discard_abci_responses = {{ .Storage.DiscardABCIResponses}}
 
+[storage.pruning]
+
+# The time period between automated background pruning operations.
+interval = "{{ .Storage.Pruning.Interval }}"
+
+#
+# Storage pruning configuration relating only to the data companion.
+#
+[storage.pruning.data_companion]
+
+# Whether automatic pruning respects values set by the data companion. Disabled
+# by default. All other parameters in this section are ignored when this is
+# disabled.
+#
+# If disabled, only the application retain height will influence block pruning
+# (but not block results pruning). Only enabling this at a later stage will
+# potentially mean that blocks below the application-set retain height at the
+# time will not be available to the data companion.
+enabled = {{ .Storage.Pruning.DataCompanion.Enabled }}
+
+# The initial value for the data companion block retain height if the data
+# companion has not yet explicitly set one. If the data companion has already
+# set a block retain height, this is ignored.
+initial_block_retain_height = {{ .Storage.Pruning.DataCompanion.InitialBlockRetainHeight }}
+
+# The initial value for the data companion block results retain height if the
+# data companion has not yet explicitly set one. If the data companion has
+# already set a block results retain height, this is ignored.
+initial_block_results_retain_height = {{ .Storage.Pruning.DataCompanion.InitialBlockResultsRetainHeight }}
+
 #######################################################
 ###   Transaction Indexer Configuration Options     ###
 #######################################################

consensus/replay_test.go

@@ -890,7 +890,21 @@ func buildTMStateFromChain(
 		for _, block := range chain[:len(chain)-1] {
 			state = applyBlock(t, stateStore, state, block, proxyApp)
 		}
-
+/*  TODO: THIS WAS NOT PRESENT AT ALL IN 0.37.x Check whether it works w/o it
+		dummyStateStore := &smmocks.Store{}
+		lastHeight := int64(len(chain))
+		penultimateHeight := int64(len(chain) - 1)
+		vals, _ := stateStore.LoadValidators(penultimateHeight)
+		dummyStateStore.On("LoadValidators", penultimateHeight).Return(vals, nil)
+		dummyStateStore.On("Save", mock.Anything).Return(nil)
+		dummyStateStore.On("SaveFinalizeBlockResponse", lastHeight, mock.MatchedBy(func(response *abci.ResponseFinalizeBlock) bool {
+			require.NoError(t, stateStore.SaveFinalizeBlockResponse(lastHeight, response))
+			return true
+		})).Return(nil)
+		dummyStateStore.On("GetApplicationRetainHeight", mock.Anything).Return(int64(0), nil)
+		dummyStateStore.On("GetCompanionBlockRetainHeight", mock.Anything).Return(int64(0), nil)
+		dummyStateStore.On("GetABCIResRetainHeight", mock.Anything).Return(int64(0), nil)
+*/
 		// apply the final block to a state copy so we can
 		// get the right next appHash but keep the state back
 		applyBlock(t, stateStore, state, chain[len(chain)-1], proxyApp)

docs/core/configuration.md

@@ -440,6 +440,36 @@ peer_query_maj23_sleep_duration = "2s"
 # reindex events in the command-line tool.
 discard_abci_responses = false
 
+[storage.pruning]
+
+# The time period between automated background pruning operations.
+interval = "10s"
+
+#
+# Storage pruning configuration relating only to the data companion.
+#
+[storage.pruning.data_companion]
+
+# Whether automatic pruning respects values set by the data companion. Disabled
+# by default. All other parameters in this section are ignored when this is
+# disabled.
+#
+# If disabled, only the application retain height will influence block pruning
+# (but not block results pruning). Only enabling this at a later stage will
+# potentially mean that blocks below the application-set retain height at the
+# time will not be available to the data companion.
+enabled = false
+
+# The initial value for the data companion block retain height if the data
+# companion has not yet explicitly set one. If the data companion has already
+# set a block retain height, this is ignored.
+initial_block_retain_height = 0
+
+# The initial value for the data companion block results retain height if the
+# data companion has not yet explicitly set one. If the data companion has
+# already set a block results retain height, this is ignored.
+initial_block_results_retain_height = 0
+
 #######################################################
 ###   Transaction Indexer Configuration Options     ###
 #######################################################

node/node.go

@@ -212,6 +212,7 @@ type Node struct {
 	eventBus          *types.EventBus // pub/sub for services
 	stateStore        sm.Store
 	blockStore        *store.BlockStore // store the blockchain to disk
+	pruner            *sm.Pruner
 	bcReactor         p2p.Reactor       // for block-syncing
 	mempoolReactor    p2p.Reactor       // for gossipping transactions
 	mempool           mempl.Mempool
@@ -821,13 +822,35 @@ func NewNodeWithContext(ctx context.Context,
 		return nil, err
 	}
 
-	// make block executor for consensus and blockchain reactors to execute blocks
+	err = initApplicationRetainHeight(stateStore)
+	if err != nil {
+		return nil, err
+	}
+
+	err = initCompanionBlockRetainHeight(
+		stateStore,
+		config.Storage.Pruning.DataCompanion.Enabled,
+		config.Storage.Pruning.DataCompanion.InitialBlockRetainHeight,
+	)
+	if err != nil {
+		return nil, err
+	}
+	pruner := sm.NewPruner(
+		stateStore,
+		blockStore,
+		logger,
+		sm.WithPrunerInterval(config.Storage.Pruning.Interval),
+	)
+
+	// make block executor for consensus and blocksync reactors to execute blocks
 	blockExec := sm.NewBlockExecutor(
 		stateStore,
 		logger.With("module", "state"),
 		proxyApp.Consensus(),
 		mempool,
 		evidencePool,
+		blockStore,
+		sm.BlockExecutorWithPruner(pruner),
 		sm.BlockExecutorWithMetrics(smMetrics),
 	)
 
@@ -929,6 +952,7 @@ func NewNodeWithContext(ctx context.Context,
 
 		stateStore:       stateStore,
 		blockStore:       blockStore,
+		pruner:           pruner,
 		bcReactor:        bcReactor,
 		mempoolReactor:   mempoolReactor,
 		mempool:          mempool,
@@ -1017,6 +1041,11 @@ func (n *Node) OnStart() error {
 		}
 	}
 
+	// Start background pruning
+	if err := n.pruner.Start(); err != nil {
+		return fmt.Errorf("failed to start background pruning routine: %w", err)
+	}
+
 	return nil
 }
 
@@ -1027,6 +1056,9 @@ func (n *Node) OnStop() {
 	n.Logger.Info("Stopping Node")
 
 	// first stop the non-reactor services
+	if err := n.pruner.Stop(); err != nil {
+		n.Logger.Error("Error stopping the pruning service", "err", err)
+	}
 	if err := n.eventBus.Stop(); err != nil {
 		n.Logger.Error("Error closing eventBus", "err", err)
 	}
@@ -1509,3 +1541,42 @@ func splitAndTrimEmpty(s, sep, cutset string) []string {
 	}
 	return nonEmptyStrings
 }
+
+// Set the initial application retain height to 0 to avoid the data companion pruning blocks
+// before the application indicates it is ok
+// We set this to 0 only if the retain height was not set before by the application
+func initApplicationRetainHeight(stateStore sm.Store) error {
+	if _, err := stateStore.GetApplicationRetainHeight(); err != nil {
+		if errors.Is(err, sm.ErrKeyNotFound) {
+			return stateStore.SaveApplicationRetainHeight(0)
+		}
+		return err
+	}
+	return nil
+}
+
+func initCompanionBlockRetainHeight(stateStore sm.Store, companionEnabled bool, initialRetainHeight int64) error {
+	if _, err := stateStore.GetCompanionBlockRetainHeight(); err != nil {
+		// If the data companion block retain height has not yet been set in
+		// the database
+		if errors.Is(err, sm.ErrKeyNotFound) {
+			if companionEnabled && initialRetainHeight > 0 {
+				// This will set the data companion retain height into the
+				// database. We bypass the sanity checks by
+				// pruner.SetCompanionBlockRetainHeight. These checks do not
+				// allow a retain height below the current blockstore height or
+				// above the blockstore height  to be set. But this is a retain
+				// height that can be set before the chain starts to indicate
+				// potentially that no pruning should be done before the data
+				// companion comes online.
+				err = stateStore.SaveCompanionBlockRetainHeight(initialRetainHeight)
+				if err != nil {
+					return fmt.Errorf("failed to set initial data companion block retain height: %w", err)
+				}
+			}
+		} else {
+			return fmt.Errorf("failed to obtain companion retain height: %w", err)
+		}
+	}
+	return nil
+}