diff --git a/tests/integration/staking/keeper/determinstic_test.go b/tests/integration/staking/keeper/determinstic_test.go index 932d2a488402..1c928d00b719 100644 --- a/tests/integration/staking/keeper/determinstic_test.go +++ b/tests/integration/staking/keeper/determinstic_test.go @@ -414,7 +414,7 @@ func TestGRPCValidatorDelegations(t *testing.T) { ValidatorAddr: validator.OperatorAddress, } - testdata.DeterministicIterations(f.ctx, t, req, f.queryClient.ValidatorDelegations, 17484, false) + testdata.DeterministicIterations(f.ctx, t, req, f.queryClient.ValidatorDelegations, 14475, false) } func TestGRPCValidatorUnbondingDelegations(t *testing.T) { diff --git a/x/bank/keeper/send.go b/x/bank/keeper/send.go index f22f99734a8d..ec1293cb6cf6 100644 --- a/x/bank/keeper/send.go +++ b/x/bank/keeper/send.go @@ -165,6 +165,19 @@ func (k BaseSendKeeper) InputOutputCoins(ctx context.Context, input types.Input, return err } + for _, out := range outputs { + outAddress, err := k.ak.AddressCodec().StringToBytes(out.Address) + if err != nil { + return err + } + + if err := k.BlockBeforeSend(ctx, inAddress, outAddress, out.Coins); err != nil { + return err + } + + k.TrackBeforeSend(ctx, inAddress, outAddress, out.Coins) + } + err = k.subUnlockedCoins(ctx, inAddress, input.Coins) if err != nil { return err diff --git a/x/staking/keeper/abci.go b/x/staking/keeper/abci.go index f65a5c1460b9..41016241da21 100644 --- a/x/staking/keeper/abci.go +++ b/x/staking/keeper/abci.go @@ -6,7 +6,6 @@ import ( abci "github.com/cometbft/cometbft/abci/types" "github.com/cosmos/cosmos-sdk/telemetry" - sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/staking/types" ) @@ -21,18 +20,5 @@ func (k *Keeper) BeginBlocker(ctx context.Context) error { func (k *Keeper) EndBlocker(ctx context.Context) ([]abci.ValidatorUpdate, error) { defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.Now(), telemetry.MetricKeyEndBlocker) - // TODO: Remove migration code and panic catch in the next upgrade - // Wrap the migration call in a function that can recover from panics - func() { - defer func() { - if r := recover(); r != nil { - k.Logger(sdk.UnwrapSDKContext(ctx)).Error("Panic in MigrateDelegationsByValidatorIndex", "recover", r) - } - }() - - // Only migrate 10000 items per block to make the migration as fast as possible - k.MigrateDelegationsByValidatorIndex(sdk.UnwrapSDKContext(ctx), 10000) - }() - return k.BlockValidatorUpdates(ctx) } diff --git a/x/staking/keeper/grpc_query.go b/x/staking/keeper/grpc_query.go index 551ad16163bc..890ca859c12b 100644 --- a/x/staking/keeper/grpc_query.go +++ b/x/staking/keeper/grpc_query.go @@ -2,7 +2,6 @@ package keeper import ( "context" - "fmt" "strings" "google.golang.org/grpc/codes" @@ -109,12 +108,6 @@ func (k Querier) ValidatorDelegations(ctx context.Context, req *types.QueryValid pageRes *query.PageResponse ) pageRes, err = query.Paginate(delStore, req.Pagination, func(delAddr, value []byte) error { - // Check the store to see if there is a value stored under the key - key := store.Get(types.NextMigrateDelegationsByValidatorIndexKey) - if key != nil { - // Users will never see this error as if there is an error the function defaults to the legacy implementation below - return fmt.Errorf("store migration is not finished, try again later") - } bz := store.Get(types.GetDelegationKey(delAddr, valAddr)) var delegation types.Delegation diff --git a/x/staking/keeper/validator_index.go b/x/staking/keeper/validator_index.go deleted file mode 100644 index 16061b947b6a..000000000000 --- a/x/staking/keeper/validator_index.go +++ /dev/null @@ -1,86 +0,0 @@ -package keeper - -import ( - "fmt" - - "cosmossdk.io/store/prefix" - - "github.com/cosmos/cosmos-sdk/runtime" - sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/cosmos/cosmos-sdk/x/staking/types" -) - -// MigrateDelegationsByValidatorIndex is a migration that runs over multiple blocks, -// this is necessary as to build the reverse index we need to iterate over a large set -// of delegations. -func (k Keeper) MigrateDelegationsByValidatorIndex(ctx sdk.Context, iterationLimit int) error { - store := runtime.KVStoreAdapter(k.storeService.OpenKVStore(ctx)) - valStore := prefix.NewStore(store, types.DelegationKey) - - // Check the store to see if there is a value stored under the key - key := store.Get(types.NextMigrateDelegationsByValidatorIndexKey) - if key == nil { - return nil - } - - // Initialize the counter to 0 - iterationCounter := 0 - - // Start the iterator from the key that is in the store - iterator := valStore.Iterator(key, nil) - defer iterator.Close() - - for ; iterator.Valid(); iterator.Next() { - key := iterator.Key() - - // Parse the index to use setting the reverse index - del, val, err := ParseDelegationKey(key) - if err != nil { - return err - } - - // Set the reverse index in the store - store.Set(types.GetDelegationsByValKey(val, del), []byte{}) - - iterationCounter++ - if iterationCounter >= iterationLimit { - ctx.Logger().Info(fmt.Sprintf("Migrated %d delegations, next key %x", iterationLimit, key)) - - // Set the key in the store after it has been processed - store.Set(types.NextMigrateDelegationsByValidatorIndexKey, key) - break - } - } - - // If the iterator is invalid we have processed the full store - if !iterator.Valid() { - ctx.Logger().Info("Migration completed") - store.Delete(types.NextMigrateDelegationsByValidatorIndexKey) - } - - return nil -} - -// ParseDelegationKey parses given key and returns delagator, validator address bytes -func ParseDelegationKey(bz []byte) (sdk.AccAddress, sdk.ValAddress, error) { - delAddrLen := bz[0] - bz = bz[1:] // remove the length byte of delegator address. - if len(bz) == 0 { - return nil, nil, fmt.Errorf("no bytes left to parse delegator address: %X", bz) - } - - del := bz[:int(delAddrLen)] - bz = bz[int(delAddrLen):] // remove the length byte of a delegator address - if len(bz) == 0 { - return nil, nil, fmt.Errorf("no bytes left to parse delegator address: %X", bz) - } - - bz = bz[1:] // remove the validator address bytes. - if len(bz) == 0 { - return nil, nil, fmt.Errorf("no bytes left to parse validator address: %X", bz) - } - - val := bz - - return del, val, nil -} diff --git a/x/staking/keeper/validator_index_test.go b/x/staking/keeper/validator_index_test.go deleted file mode 100644 index 13c4c26dc8c6..000000000000 --- a/x/staking/keeper/validator_index_test.go +++ /dev/null @@ -1,150 +0,0 @@ -package keeper_test - -import ( - "cosmossdk.io/core/store" - sdkmath "cosmossdk.io/math" - storetypes "cosmossdk.io/store/types" - - "github.com/cosmos/cosmos-sdk/codec" - "github.com/cosmos/cosmos-sdk/runtime" - "github.com/cosmos/cosmos-sdk/testutil/sims" - sdk "github.com/cosmos/cosmos-sdk/types" - moduletestutil "github.com/cosmos/cosmos-sdk/types/module/testutil" - "github.com/cosmos/cosmos-sdk/x/staking" - "github.com/cosmos/cosmos-sdk/x/staking/types" -) - -// TestDelegationsByValidatorMigration tests the multi block migration of the reverse delegation index -func (s *KeeperTestSuite) TestDelegationsByValidatorMigration() { - require := s.Require() - ctx, keeper := s.ctx, s.stakingKeeper - store := s.storeService.OpenKVStore(ctx) - storeInit := runtime.KVStoreAdapter(store) - cdc := moduletestutil.MakeTestEncodingConfig(staking.AppModuleBasic{}).Codec - - accAddrs := sims.CreateIncrementalAccounts(15) - valAddrs := sims.ConvertAddrsToValAddrs(accAddrs[0:1]) - var addedDels []types.Delegation - - // start at 1 as 0 addr is the validator addr - for i := 1; i < 11; i++ { - del1 := types.NewDelegation(accAddrs[i].String(), valAddrs[0].String(), sdkmath.LegacyNewDec(100)) - store.Set(types.GetDelegationKey(accAddrs[i], valAddrs[0]), types.MustMarshalDelegation(cdc, del1)) - addedDels = append(addedDels, del1) - } - - // number of items we migrate per migration - migrationCadence := 6 - - // set the key in the store, this happens on the original migration - iterator := storetypes.KVStorePrefixIterator(storeInit, types.DelegationKey) - for ; iterator.Valid(); iterator.Next() { - key := iterator.Key() - store.Set(types.NextMigrateDelegationsByValidatorIndexKey, key[1:]) - break - } - - // before migration the state of delegations by val index should be empty - dels := getValDelegations(cdc, store, valAddrs[0]) - require.Equal(len(dels), 0) - - // run the first round of migrations first 6, 10 in store - err := keeper.MigrateDelegationsByValidatorIndex(ctx, migrationCadence) - require.NoError(err) - - // after migration the state of delegations by val index should not be empty - dels = getValDelegations(cdc, store, valAddrs[0]) - require.Equal(len(dels), migrationCadence) - require.NotEqual(len(dels), len(addedDels)) - - // check that the next value needed from the store is present - next, err := store.Get(types.NextMigrateDelegationsByValidatorIndexKey) - require.NoError(err) - require.NotNil(next) - - // delegate to a validator while the migration is in progress - delagationWhileMigrationInProgress := types.NewDelegation(accAddrs[12].String(), valAddrs[0].String(), sdkmath.LegacyNewDec(100)) - keeper.SetDelegation(ctx, delagationWhileMigrationInProgress) - addedDels = append(addedDels, delagationWhileMigrationInProgress) - - // remove a delegation from a validator while the migration is in progress that has been processed - removeDelagationWhileMigrationInProgress := types.NewDelegation(accAddrs[3].String(), valAddrs[0].String(), sdkmath.LegacyNewDec(100)) - keeper.RemoveDelegation(ctx, removeDelagationWhileMigrationInProgress) - // index in the array is 2 - addedDels = deleteElement(addedDels, 2) - - // remove the index on the off chance this happens during the migration - removeDelagationWhileMigrationInProgressNextIndex := types.NewDelegation(accAddrs[6].String(), valAddrs[0].String(), sdkmath.LegacyNewDec(100)) - keeper.RemoveDelegation(ctx, removeDelagationWhileMigrationInProgressNextIndex) - // index in the array is 4, as we've removed one item - addedDels = deleteElement(addedDels, 4) - - // remove a delegation from a validator while the migration is in progress that has not been processed - removeDelagationWhileMigrationInProgressNotProcessed := types.NewDelegation(accAddrs[10].String(), valAddrs[0].String(), sdkmath.LegacyNewDec(100)) - keeper.RemoveDelegation(ctx, removeDelagationWhileMigrationInProgressNotProcessed) - // index in the array is 7, as we've removed 2 items - addedDels = deleteElement(addedDels, 7) - - // while migrating get state of delegations by val index should be increased by 1 - delagationWhileMigrationInProgressCount := getValDelegations(cdc, store, valAddrs[0]) - require.Equal(len(delagationWhileMigrationInProgressCount), migrationCadence-1) - - // run the second round of migrations - err = keeper.MigrateDelegationsByValidatorIndex(ctx, migrationCadence) - require.NoError(err) - - // after migration the state of delegations by val index equal all delegations - dels = getValDelegations(cdc, store, valAddrs[0]) - require.Equal(len(dels), len(addedDels)) - require.Equal(dels, addedDels) - - // check that the next value needed from the store is empty - next, err = store.Get(types.NextMigrateDelegationsByValidatorIndexKey) - require.NoError(err) - require.Nil(next) - - // Iterate over the store by delegation key - delKeyCount := 0 - iteratorDel := storetypes.KVStorePrefixIterator(storeInit, types.DelegationKey) - for ; iteratorDel.Valid(); iteratorDel.Next() { - delKeyCount++ - } - - // Iterate over the store by validator key - valKeyCount := 0 - iteratorVal := storetypes.KVStorePrefixIterator(storeInit, types.DelegationByValIndexKey) - for ; iteratorVal.Valid(); iteratorVal.Next() { - valKeyCount++ - } - - // Make sure the store count is the same - require.Equal(valKeyCount, delKeyCount) -} - -// deleteElement is a simple helper function to remove items from a slice -func deleteElement(slice []types.Delegation, index int) []types.Delegation { - return append(slice[:index], slice[index+1:]...) -} - -// getValidatorDelegations is a helper function to get all delegations using the new v5 staking reverse index -func getValDelegations(cdc codec.Codec, keeperStore store.KVStore, valAddr sdk.ValAddress) []types.Delegation { - var delegations []types.Delegation - - store := runtime.KVStoreAdapter(keeperStore) - iterator := storetypes.KVStorePrefixIterator(store, types.GetDelegationsByValPrefixKey(valAddr)) - - for ; iterator.Valid(); iterator.Next() { - var delegation types.Delegation - valAddr, delAddr, err := types.ParseDelegationsByValKey(iterator.Key()) - if err != nil { - panic(err) - } - - bz := store.Get(types.GetDelegationKey(delAddr, valAddr)) - cdc.MustUnmarshal(bz, &delegation) - - delegations = append(delegations, delegation) - } - - return delegations -} diff --git a/x/staking/types/keys.go b/x/staking/types/keys.go index 4afa81994a8a..598aa332b312 100644 --- a/x/staking/types/keys.go +++ b/x/staking/types/keys.go @@ -56,8 +56,6 @@ var ( ParamsKey = []byte{0x51} // prefix for parameters for module x/staking DelegationByValIndexKey = []byte{0x71} // key for delegations by a validator - - NextMigrateDelegationsByValidatorIndexKey = []byte{0x81} // key used to migrate to the new validator index ) // UnbondingType defines the type of unbonding operation