Backport v0.38.5: IAVL Bump (RC) & Pruning Refactor

CHANGELOG.md

@@ -35,7 +35,50 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 # Changelog
 
-## [Unreleased]
+## [v0.38.5] - TBD
+
+The pruning changes introduced in 0.38.0 has bugs that may cause data loss, even after upgrading
+to 0.38.5. When upgrading from 0.38.x it is important to follow the instructions below, to prevent 
+data loss and database corruption.
+
+Do not modify pruning settings with <=0.38.4 as that may cause data corruption - the following 
+assumes pruning settings have not been modified since the node started using 0.38.x. The default 
+pruning setting `syncable` used `KeepEvery:100`.
+
+* If using `KeepEvery:1` (pruning settings `nothing` or `everything`), upgrading to 0.38.5 is safe.
+
+* Otherwise, halt block processing with `--halt-height` after committing a height divisible by
+  `KeepEvery` - e.g. at block 147600 with `KeepEvery:100`. The node must _never_ have processed a
+  height beyond that at any time in its past. Upgrading to 0.38.5 is then safe.
+
+* Otherwise, set the 0.38.5 `KeepEvery` setting to the same as the previous `KeepEvery` setting
+  (<=0.38.4 and 0.38.5 both default to `KeepEvery:100`). Upgrading to 0.38.5 is then safe as long
+  as you wait one `KeepEvery` interval plus one `KeepRecent` interval plus one pruning `Interval`
+  before changing pruning settings or deleting the last <=0.38.4 height (so wait 210 heights with
+  the default configuration).
+
+* Otherwise, make sure the last version persisted with <=0.38.4 is never deleted after upgrading to 
+  0.38.5, as doing so may cause data loss and data corruption.
+
+* Otherwise, consider syncing the node from scratch with 0.38.5.
+
+### Improvements
+
+* (deps) Bump Tendermint version to [v0.33.5](https://github.com/tendermint/tendermint/releases/tag/v0.33.5)
+* (deps) Bump IAVL version to [v0.14.0](https://github.com/cosmos/iavl/releases/tag/v0.14.0)
+
+### Bug Fixes
+
+* (store) Revert IAVL pruning functionality introduced in [v0.13.0](https://github.com/cosmos/iavl/releases/tag/v0.13.0),
+where the IAVL no longer keeps states in-memory in which it flushes periodically. IAVL now commits and
+flushes every state to disk as it did pre-v0.13.0. The SDK's multi-store will track and ensure the proper
+heights are pruned. The operator can set the pruning options via a `pruning` config via the CLI or
+through `app.toml`. The `pruning` flag exposes `default|everything|nothing|custom` as options --
+see docs for further details. If the operator chooses `custom`, they may provide granular pruning
+options `pruning-keep-recent`, `pruning-keep-every`, and `pruning-interval`. The former two options
+dictate how many recent versions are kept on disk and the offset of what versions are kept after that
+respectively, and the latter defines the height interval in which versions are deleted in a batch.
+**Note, there are some client-facing API breaking changes with regard to IAVL, stores, and pruning settings.**
 
 ## [v0.38.4] - 2020-05-21
 

baseapp/baseapp_test.go

@@ -168,7 +168,7 @@ func initStore(t *testing.T, db dbm.DB, storeKey string, k, v []byte) {
 
 func checkStore(t *testing.T, db dbm.DB, ver int64, storeKey string, k, v []byte) {
 	rs := rootmulti.NewStore(db)
-	rs.SetPruning(store.PruneSyncable)
+	rs.SetPruning(store.PruneDefault)
 	key := sdk.NewKVStoreKey(storeKey)
 	rs.MountStoreWithDB(key, store.StoreTypeIAVL, nil)
 	err := rs.LoadLatestVersion()
@@ -273,7 +273,7 @@ func TestSetLoader(t *testing.T) {
 
 func TestAppVersionSetterGetter(t *testing.T) {
 	logger := defaultLogger()
-	pruningOpt := SetPruning(store.PruneSyncable)
+	pruningOpt := SetPruning(store.PruneDefault)
 	db := dbm.NewMemDB()
 	name := t.Name()
 	app := NewBaseApp(name, logger, db, nil, pruningOpt)
@@ -329,8 +329,9 @@ func TestLoadVersionInvalid(t *testing.T) {
 func TestLoadVersionPruning(t *testing.T) {
 	logger := log.NewNopLogger()
 	pruningOptions := store.PruningOptions{
-		KeepEvery:     2,
-		SnapshotEvery: 6,
+		KeepRecent: 2,
+		KeepEvery:  3,
+		Interval:   1,
 	}
 	pruningOpt := SetPruning(pruningOptions)
 	db := dbm.NewMemDB()
@@ -351,61 +352,33 @@ func TestLoadVersionPruning(t *testing.T) {
 	require.Equal(t, int64(0), lastHeight)
 	require.Equal(t, emptyCommitID, lastID)
 
-	// execute a block
-	header := abci.Header{Height: 1}
-	app.BeginBlock(abci.RequestBeginBlock{Header: header})
-	res := app.Commit()
-
-	// execute a block, collect commit ID
-	header = abci.Header{Height: 2}
-	app.BeginBlock(abci.RequestBeginBlock{Header: header})
-	res = app.Commit()
-	commitID2 := sdk.CommitID{Version: 2, Hash: res.Data}
-
-	// execute a block
-	header = abci.Header{Height: 3}
-	app.BeginBlock(abci.RequestBeginBlock{Header: header})
-	res = app.Commit()
-	commitID3 := sdk.CommitID{Version: 3, Hash: res.Data}
-
-	// reload with LoadLatestVersion, check it loads last flushed version
-	app = NewBaseApp(name, logger, db, nil, pruningOpt)
-	app.MountStores(capKey)
-	err = app.LoadLatestVersion(capKey)
-	require.Nil(t, err)
-	testLoadVersionHelper(t, app, int64(2), commitID2)
+	var lastCommitID sdk.CommitID
 
-	// re-execute block 3 and check it is same CommitID
-	header = abci.Header{Height: 3}
-	app.BeginBlock(abci.RequestBeginBlock{Header: header})
-	res = app.Commit()
-	recommitID3 := sdk.CommitID{Version: 3, Hash: res.Data}
-	require.Equal(t, commitID3, recommitID3, "Commits of identical blocks not equal after reload")
+	// Commit seven blocks, of which 7 (latest) is kept in addition to 6, 5
+	// (keep recent) and 3 (keep every).
+	for i := int64(1); i <= 7; i++ {
+		app.BeginBlock(abci.RequestBeginBlock{Header: abci.Header{Height: i}})
+		res := app.Commit()
+		lastCommitID = sdk.CommitID{Version: i, Hash: res.Data}
+	}
 
-	// execute a block, collect commit ID
-	header = abci.Header{Height: 4}
-	app.BeginBlock(abci.RequestBeginBlock{Header: header})
-	res = app.Commit()
-	commitID4 := sdk.CommitID{Version: 4, Hash: res.Data}
+	for _, v := range []int64{1, 2, 4} {
+		_, err = app.cms.CacheMultiStoreWithVersion(v)
+		require.Error(t, err)
+	}
 
-	// execute a block
-	header = abci.Header{Height: 5}
-	app.BeginBlock(abci.RequestBeginBlock{Header: header})
-	res = app.Commit()
+	for _, v := range []int64{3, 5, 6, 7} {
+		_, err = app.cms.CacheMultiStoreWithVersion(v)
+		require.NoError(t, err)
+	}
 
-	// reload with LoadLatestVersion, check it loads last flushed version
+	// reload with LoadLatestVersion, check it loads last version
 	app = NewBaseApp(name, logger, db, nil, pruningOpt)
 	app.MountStores(capKey)
+
 	err = app.LoadLatestVersion(capKey)
 	require.Nil(t, err)
-	testLoadVersionHelper(t, app, int64(4), commitID4)
-
-	// reload with LoadVersion of previous flushed version
-	// and check it fails since previous flush should be pruned
-	app = NewBaseApp(name, logger, db, nil, pruningOpt)
-	app.MountStores(capKey)
-	err = app.LoadVersion(2, capKey)
-	require.NotNil(t, err)
+	testLoadVersionHelper(t, app, int64(7), lastCommitID)
 }
 
 func testLoadVersionHelper(t *testing.T, app *BaseApp, expectedHeight int64, expectedID sdk.CommitID) {

client/context/context.go

@@ -12,6 +12,7 @@ import (
 	"github.com/tendermint/tendermint/libs/cli"
 	tmlite "github.com/tendermint/tendermint/lite"
 	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	rpchttp "github.com/tendermint/tendermint/rpc/client/http"
 
 	"github.com/cosmos/cosmos-sdk/client/flags"
 	"github.com/cosmos/cosmos-sdk/codec"
@@ -65,7 +66,7 @@ func NewCLIContextWithInputAndFrom(input io.Reader, from string) CLIContext {
 	if !genOnly {
 		nodeURI = viper.GetString(flags.FlagNode)
 		if nodeURI != "" {
-			rpc, err = rpcclient.NewHTTP(nodeURI, "/websocket")
+			rpc, err = rpchttp.New(nodeURI, "/websocket")
 			if err != nil {
 				fmt.Printf("failted to get client: %v\n", err)
 				os.Exit(1)
@@ -157,7 +158,7 @@ func (ctx CLIContext) WithTrustNode(trustNode bool) CLIContext {
 // WithNodeURI returns a copy of the context with an updated node URI.
 func (ctx CLIContext) WithNodeURI(nodeURI string) CLIContext {
 	ctx.NodeURI = nodeURI
-	client, err := rpcclient.NewHTTP(nodeURI, "/websocket")
+	client, err := rpchttp.New(nodeURI, "/websocket")
 	if err != nil {
 		panic(err)
 	}

client/context/verifier.go

@@ -7,7 +7,7 @@ import (
 	"github.com/tendermint/tendermint/libs/log"
 	tmlite "github.com/tendermint/tendermint/lite"
 	tmliteproxy "github.com/tendermint/tendermint/lite/proxy"
-	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	rpchttp "github.com/tendermint/tendermint/rpc/client/http"
 )
 
 const (
@@ -43,7 +43,7 @@ func CreateVerifier(ctx CLIContext, cacheSize int) (tmlite.Verifier, error) {
 	// create an RPC client based off of the RPC URI if no RPC client exists
 	client := ctx.Client
 	if client == nil {
-		client, err = rpcclient.NewHTTP(ctx.NodeURI, "/websocket")
+		client, err = rpchttp.New(ctx.NodeURI, "/websocket")
 		if err != nil {
 			return nil, err
 		}

client/lcd/root.go

@@ -12,7 +12,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/tendermint/tendermint/libs/log"
-	rpcserver "github.com/tendermint/tendermint/rpc/lib/server"
+	tmrpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server"
 
 	"github.com/cosmos/cosmos-sdk/client/context"
 	"github.com/cosmos/cosmos-sdk/client/flags"
@@ -52,12 +52,12 @@ func (rs *RestServer) Start(listenAddr string, maxOpen int, readTimeout, writeTi
 		rs.log.Error("error closing listener", "err", err)
 	})
 
-	cfg := rpcserver.DefaultConfig()
+	cfg := tmrpcserver.DefaultConfig()
 	cfg.MaxOpenConnections = maxOpen
 	cfg.ReadTimeout = time.Duration(readTimeout) * time.Second
 	cfg.WriteTimeout = time.Duration(writeTimeout) * time.Second
 
-	rs.listener, err = rpcserver.Listen(listenAddr, cfg)
+	rs.listener, err = tmrpcserver.Listen(listenAddr, cfg)
 	if err != nil {
 		return
 	}
@@ -68,7 +68,7 @@ func (rs *RestServer) Start(listenAddr string, maxOpen int, readTimeout, writeTi
 		),
 	)
 
-	return rpcserver.StartHTTPServer(rs.listener, rs.Mux, rs.log, cfg)
+	return tmrpcserver.Serve(rs.listener, rs.Mux, rs.log, cfg)
 }
 
 // ServeCommand will start the application REST service as a blocking process. It

go.mod

@@ -1,10 +1,12 @@
+go 1.13
+
 module github.com/cosmos/cosmos-sdk
 
 require (
 	github.com/99designs/keyring v1.1.3
 	github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d
 	github.com/bgentry/speakeasy v0.1.0
-	github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d
+	github.com/btcsuite/btcd v0.20.1-beta
 	github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d
 	github.com/cosmos/ledger-cosmos-go v0.11.1
 	github.com/gogo/protobuf v1.3.1
@@ -16,20 +18,19 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/rakyll/statik v0.1.6
 	github.com/spf13/afero v1.2.1 // indirect
-	github.com/spf13/cobra v0.0.6
+	github.com/spf13/cobra v1.0.0
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.6.2
-	github.com/stretchr/testify v1.5.1
+	github.com/spf13/viper v1.6.3
+	github.com/stretchr/testify v1.6.1
 	github.com/tendermint/btcd v0.1.1
 	github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15
 	github.com/tendermint/go-amino v0.15.1
-	github.com/tendermint/iavl v0.13.2
-	github.com/tendermint/tendermint v0.33.3
-	github.com/tendermint/tm-db v0.5.0
-	gopkg.in/yaml.v2 v2.2.8
+	github.com/tendermint/iavl v0.14.0-rc2
+	github.com/tendermint/tendermint v0.33.5
+	github.com/tendermint/tm-db v0.5.1
+	google.golang.org/grpc v1.30.0 // indirect
+	gopkg.in/yaml.v2 v2.3.0
 )
 
-go 1.13
-
 replace github.com/keybase/go-keychain => github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4