Merge branch 'main' into adam/module-hash-by-height

CHANGELOG.md

@@ -57,7 +57,6 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i
 * (baseapp) [#18499](https://github.com/cosmos/cosmos-sdk/pull/18499) Add `MsgRouter` response type from message name function.
 * (types) [#18768](https://github.com/cosmos/cosmos-sdk/pull/18768) Add MustValAddressFromBech32 function.
 * (gRPC) [#19049](https://github.com/cosmos/cosmos-sdk/pull/19049) Add debug log prints for each gRPC request.
-* (x/consensus) [#19483](https://github.com/cosmos/cosmos-sdk/pull/19483) Add consensus messages registration to consensus module.
 * (types) [#19759](https://github.com/cosmos/cosmos-sdk/pull/19759) Align SignerExtractionAdapter in PriorityNonceMempool Remove.
 * (client) [#19870](https://github.com/cosmos/cosmos-sdk/pull/19870) Add new query command `wait-tx`. Alias `event-query-tx-for` to `wait-tx` for backward compatibility.
 * (crypto/keyring) [#20212](https://github.com/cosmos/cosmos-sdk/pull/20212) Expose the db keyring used in the keystore.
@@ -145,7 +144,6 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i
     * The module manager now can do everything that the basic manager was doing.
     * When using runtime, just inject the module manager when needed using your app config.
     * All `AppModuleBasic` structs have been removed.
-* (x/consensus) [#19488](https://github.com/cosmos/cosmos-sdk/pull/19488) Consensus module creation takes `appmodule.Environment` instead of individual services.
 * (server) [#18303](https://github.com/cosmos/cosmos-sdk/pull/18303) `x/genutil` now handles the application export. `server.AddCommands` does not take an `AppExporter` but instead `genutilcli.Commands` does.
 * (x/gov/testutil) [#17986](https://github.com/cosmos/cosmos-sdk/pull/18036) `MsgDeposit` has been removed because of AutoCLI migration.
 * (x/staking/testutil) [#17986](https://github.com/cosmos/cosmos-sdk/pull/17986) `MsgRedelegateExec`, `MsgUnbondExec` has been removed because of AutoCLI migration.
@@ -175,7 +173,6 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i
 * (x/staking) [#18257](https://github.com/cosmos/cosmos-sdk/pull/18257) Staking module was moved to its own go.mod `cosmossdk.io/x/staking`
 * (x/authz) [#18265](https://github.com/cosmos/cosmos-sdk/pull/18265) Authz module was moved to its own go.mod `cosmossdk.io/x/authz`
 * (x/mint) [#18283](https://github.com/cosmos/cosmos-sdk/pull/18283) Mint module was moved to its own go.mod `cosmossdk.io/x/mint`
-* (x/consensus) [#18041](https://github.com/cosmos/cosmos-sdk/pull/18041) `ToProtoConsensusParams()` returns an error
 * (x/slashing) [#18115](https://github.com/cosmos/cosmos-sdk/pull/18115) `NewValidatorSigningInfo` takes strings instead of `sdk.AccAddress`
 * (types) [#18268](https://github.com/cosmos/cosmos-sdk/pull/18268) Remove global setting of basedenom. Use the staking module parameter instead
 * (x/auth) [#18351](https://github.com/cosmos/cosmos-sdk/pull/18351) Auth module was moved to its own go.mod `cosmossdk.io/x/auth`

store/v2/commitment/store_test_suite.go

@@ -221,6 +221,83 @@ func (s *CommitStoreTestSuite) TestStore_Pruning() {
 	}
 }
 
+func (s *CommitStoreTestSuite) TestStore_GetProof() {
+	storeKeys := []string{storeKey1, storeKey2}
+	commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger())
+	s.Require().NoError(err)
+
+	toVersion := uint64(10)
+	keyCount := 5
+
+	// commit some changes
+	for version := uint64(1); version <= toVersion; version++ {
+		cs := corestore.NewChangeset()
+		for _, storeKey := range storeKeys {
+			for i := 0; i < keyCount; i++ {
+				cs.Add([]byte(storeKey), []byte(fmt.Sprintf("key-%d-%d", version, i)), []byte(fmt.Sprintf("value-%d-%d", version, i)), false)
+			}
+		}
+		err := commitStore.WriteChangeset(cs)
+		s.Require().NoError(err)
+		_, err = commitStore.Commit(version)
+		s.Require().NoError(err)
+	}
+
+	// get proof
+	for version := uint64(1); version <= toVersion; version++ {
+		for _, storeKey := range storeKeys {
+			for i := 0; i < keyCount; i++ {
+				_, err := commitStore.GetProof([]byte(storeKey), version, []byte(fmt.Sprintf("key-%d-%d", version, i)))
+				s.Require().NoError(err)
+			}
+		}
+	}
+
+	// prune version 1
+	s.Require().NoError(commitStore.Prune(1))
+
+	// check if proof for version 1 is pruned
+	_, err = commitStore.GetProof([]byte(storeKeys[0]), 1, []byte(fmt.Sprintf("key-%d-%d", 1, 0)))
+	s.Require().Error(err)
+	// check the commit info
+	commit, _ := commitStore.GetCommitInfo(1)
+	s.Require().Nil(commit)
+}
+
+func (s *CommitStoreTestSuite) TestStore_Get() {
+	storeKeys := []string{storeKey1, storeKey2}
+	commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger())
+	s.Require().NoError(err)
+
+	toVersion := uint64(10)
+	keyCount := 5
+
+	// commit some changes
+	for version := uint64(1); version <= toVersion; version++ {
+		cs := corestore.NewChangeset()
+		for _, storeKey := range storeKeys {
+			for i := 0; i < keyCount; i++ {
+				cs.Add([]byte(storeKey), []byte(fmt.Sprintf("key-%d-%d", version, i)), []byte(fmt.Sprintf("value-%d-%d", version, i)), false)
+			}
+		}
+		err := commitStore.WriteChangeset(cs)
+		s.Require().NoError(err)
+		_, err = commitStore.Commit(version)
+		s.Require().NoError(err)
+	}
+
+	// get proof
+	for version := uint64(1); version <= toVersion; version++ {
+		for _, storeKey := range storeKeys {
+			for i := 0; i < keyCount; i++ {
+				val, err := commitStore.Get([]byte(storeKey), version, []byte(fmt.Sprintf("key-%d-%d", version, i)))
+				s.Require().NoError(err)
+				s.Require().Equal([]byte(fmt.Sprintf("value-%d-%d", version, i)), val)
+			}
+		}
+	}
+}
+
 func (s *CommitStoreTestSuite) TestStore_Upgrades() {
 	storeKeys := []string{storeKey1, storeKey2, storeKey3}
 	commitDB := dbm.NewMemDB()

store/v2/migration/manager_test.go

@@ -1,6 +1,7 @@
 package migration
 
 import (
+	"encoding/binary"
 	"fmt"
 	"testing"
 
@@ -117,3 +118,103 @@ func TestMigrateState(t *testing.T) {
 		})
 	}
 }
+
+func TestStartMigrateState(t *testing.T) {
+	for _, noCommitStore := range []bool{false, true} {
+		t.Run(fmt.Sprintf("Migrate noCommitStore=%v", noCommitStore), func(t *testing.T) {
+			m, orgCommitStore := setupMigrationManager(t, noCommitStore)
+
+			chDone := make(chan struct{})
+			chChangeset := make(chan *VersionedChangeset, 1)
+
+			// apply changeset
+			toVersion := uint64(10)
+			keyCount := 5
+			changesets := []corestore.Changeset{}
+
+			for version := uint64(1); version <= toVersion; version++ {
+				cs := corestore.NewChangeset()
+				for _, storeKey := range storeKeys {
+					for i := 0; i < keyCount; i++ {
+						cs.Add([]byte(storeKey), []byte(fmt.Sprintf("key-%d-%d", version, i)), []byte(fmt.Sprintf("value-%d-%d", version, i)), false)
+					}
+				}
+				changesets = append(changesets, *cs)
+				require.NoError(t, orgCommitStore.WriteChangeset(cs))
+				_, err := orgCommitStore.Commit(version)
+				require.NoError(t, err)
+			}
+
+			// feed changesets to channel
+			go func() {
+				for version := uint64(1); version <= toVersion; version++ {
+					chChangeset <- &VersionedChangeset{
+						Version:   version,
+						Changeset: &changesets[version-1],
+					}
+				}
+			}()
+
+			// check if migrate process complete
+			go func() {
+				for {
+					migrateVersion := m.GetMigratedVersion()
+					if migrateVersion == toVersion-1 {
+						break
+					}
+				}
+
+				chDone <- struct{}{}
+			}()
+
+			err := m.Start(toVersion-1, chChangeset, chDone)
+			require.NoError(t, err)
+
+			// expecting error for conflicting process, since Migrate trigger snapshotter create migration,
+			// which start a snapshot process already.
+			_, err = m.snapshotsManager.Create(toVersion - 1)
+			require.Error(t, err)
+
+			if m.stateCommitment != nil {
+				// check the migrated state
+				for version := uint64(1); version < toVersion; version++ {
+					for _, storeKey := range storeKeys {
+						for i := 0; i < keyCount; i++ {
+							val, err := m.stateCommitment.Get([]byte(storeKey), toVersion-1, []byte(fmt.Sprintf("key-%d-%d", version, i)))
+							require.NoError(t, err)
+							require.Equal(t, []byte(fmt.Sprintf("value-%d-%d", version, i)), val)
+						}
+					}
+				}
+				// check the latest state
+				val, err := m.stateCommitment.Get([]byte("store1"), toVersion-1, []byte("key-100-1"))
+				require.NoError(t, err)
+				require.Nil(t, val)
+				val, err = m.stateCommitment.Get([]byte("store2"), toVersion-1, []byte("key-100-0"))
+				require.NoError(t, err)
+				require.Nil(t, val)
+			}
+
+			// check the storage
+			for version := uint64(1); version < toVersion; version++ {
+				for _, storeKey := range storeKeys {
+					for i := 0; i < keyCount; i++ {
+						val, err := m.stateStorage.Get([]byte(storeKey), toVersion-1, []byte(fmt.Sprintf("key-%d-%d", version, i)))
+						require.NoError(t, err)
+						require.Equal(t, []byte(fmt.Sprintf("value-%d-%d", version, i)), val)
+					}
+				}
+			}
+
+			// check if migration db write change set to storage
+			for version := uint64(1); version < toVersion; version++ {
+				buf := make([]byte, 8)
+				binary.BigEndian.PutUint64(buf, version)
+				csKey := []byte(fmt.Sprintf(migrateChangesetKeyFmt, buf))
+				csVal, err := m.db.Get(csKey)
+				require.NoError(t, err)
+				require.NotEmpty(t, csVal)
+			}
+		})
+	}
+}

store/v2/pruning/manager_test.go

@@ -152,3 +152,107 @@ func TestPruningOption(t *testing.T) {
 		})
 	}
 }
+
+func (s *PruningManagerTestSuite) TestSignalCommit() {
+	// commit version 1
+	cs := corestore.NewChangeset()
+	for _, storeKey := range storeKeys {
+		cs.Add([]byte(storeKey), []byte(fmt.Sprintf("key-%d-%d", 1, 0)), []byte(fmt.Sprintf("value-%d-%d", 1, 0)), false)
+	}
+
+	s.Require().NoError(s.sc.WriteChangeset(cs))
+	_, err := s.sc.Commit(1)
+	s.Require().NoError(err)
+
+	s.Require().NoError(s.ss.ApplyChangeset(1, cs))
+
+	// commit version 2
+	for _, storeKey := range storeKeys {
+		cs.Add([]byte(storeKey), []byte(fmt.Sprintf("key-%d-%d", 2, 0)), []byte(fmt.Sprintf("value-%d-%d", 2, 0)), false)
+	}
+
+	// signaling commit has started
+	s.Require().NoError(s.manager.SignalCommit(true, 2))
+
+	s.Require().NoError(s.sc.WriteChangeset(cs))
+	_, err = s.sc.Commit(2)
+	s.Require().NoError(err)
+
+	s.Require().NoError(s.ss.ApplyChangeset(2, cs))
+
+	// try prune before signaling commit has finished
+	s.Require().NoError(s.manager.Prune(2))
+
+	// proof is removed no matter SignalCommit has not yet inform that commit process has finish
+	// since commitInfo is remove async with tree data
+	checkSCPrune := func() bool {
+		count := 0
+		for _, storeKey := range storeKeys {
+			_, err := s.sc.GetProof([]byte(storeKey), 1, []byte(fmt.Sprintf("key-%d-%d", 1, 0)))
+			if err != nil {
+				count++
+			}
+		}
+
+		return count == len(storeKeys)
+	}
+	s.Require().Eventually(checkSCPrune, 10*time.Second, 1*time.Second)
+
+	// data from state commitment should not be pruned since we haven't signal the commit process has finished
+	val, err := s.sc.Get([]byte(storeKeys[0]), 1, []byte(fmt.Sprintf("key-%d-%d", 1, 0)))
+	s.Require().NoError(err)
+	s.Require().Equal(val, []byte(fmt.Sprintf("value-%d-%d", 1, 0)))
+
+	// signaling commit has finished, version 1 should be pruned
+	s.Require().NoError(s.manager.SignalCommit(false, 2))
+
+	checkSCPrune = func() bool {
+		count := 0
+		for _, storeKey := range storeKeys {
+			_, err := s.sc.GetProof([]byte(storeKey), 1, []byte(fmt.Sprintf("key-%d-%d", 1, 0)))
+			if err != nil {
+				count++
+			}
+		}
+
+		return count == len(storeKeys)
+	}
+	s.Require().Eventually(checkSCPrune, 10*time.Second, 1*time.Second)
+
+	// try with signal commit start and finish accordingly
+	// commit changesets with pruning
+	toVersion := uint64(100)
+	keyCount := 10
+	for version := uint64(3); version <= toVersion; version++ {
+		cs := corestore.NewChangeset()
+		for _, storeKey := range storeKeys {
+			for i := 0; i < keyCount; i++ {
+				cs.Add([]byte(storeKey), []byte(fmt.Sprintf("key-%d-%d", version, i)), []byte(fmt.Sprintf("value-%d-%d", version, i)), false)
+			}
+		}
+		s.Require().NoError(s.manager.SignalCommit(true, version))
+
+		s.Require().NoError(s.sc.WriteChangeset(cs))
+		_, err := s.sc.Commit(version)
+		s.Require().NoError(err)
+
+		s.Require().NoError(s.ss.ApplyChangeset(version, cs))
+
+		s.Require().NoError(s.manager.SignalCommit(false, version))
+
+	}
+
+	// wait for the pruning to finish in the commitment store
+	checkSCPrune = func() bool {
+		count := 0
+		for _, storeKey := range storeKeys {
+			_, err := s.sc.GetProof([]byte(storeKey), toVersion-1, []byte(fmt.Sprintf("key-%d-%d", toVersion-1, 0)))
+			if err != nil {
+				count++
+			}
+		}
+
+		return count == len(storeKeys)
+	}
+	s.Require().Eventually(checkSCPrune, 10*time.Second, 1*time.Second)
+}

store/v2/root/store_test.go

@@ -643,7 +643,7 @@ func (s *RootStoreTestSuite) TestMultiStore_PruningRestart() {
 			return false
 		}
 		// wait for async pruning process to finish
-		s.Require().Eventually(checkErr, 5*time.Second, 100*time.Millisecond, "expected error when loading height: %d", v)
+		s.Require().Eventually(checkErr, 10*time.Second, 1*time.Second, "expected error when loading height: %d", v)
 	}
 }