fix: removed potential sources of non-determinism in upgrades (backport

cosmos#10189) (cosmos#10253)

* fix: removed potential sources of non-determinism in upgrades (cosmos#10189)

forced deterministic iteration order in upgrade migrations, x/upgrade and store during upgrades

Co-authored-by: Robert Zaremba <robert@zaremba.ch>
(cherry picked from commit f757c90)

# Conflicts:
#	CHANGELOG.md

* Update CHANGELOG.md

Co-authored-by: Tomas Tauber <2410580+tomtau@users.noreply.github.com>
Co-authored-by: Robert Zaremba <robert@zaremba.ch>

CHANGELOG.md

@@ -72,6 +72,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 ### Bug Fixes
 
 * [\#9965](https://github.com/cosmos/cosmos-sdk/pull/9965) Fixed `simd version` command output to report the right release tag.
+* (x/upgrade) [\#10189](https://github.com/cosmos/cosmos-sdk/issues/10189) Removed potential sources of non-determinism in upgrades.
 
 ### Client Breaking Changes
 

store/rootmulti/store.go

@@ -239,6 +239,20 @@ func (rs *Store) loadVersion(ver int64, upgrades *types.StoreUpgrades) error {
 		})
 	}
 
+	storesKeys := make([]types.StoreKey, 0, len(rs.storesParams))
+
+	for key := range rs.storesParams {
+		storesKeys = append(storesKeys, key)
+	}
+	if upgrades != nil {
+		// deterministic iteration order for upgrades
+		// (as the underlying store may change and
+		// upgrades make store changes where the execution order may matter)
+		sort.Slice(storesKeys, func(i, j int) bool {
+			return storesKeys[i].Name() < storesKeys[j].Name()
+		})
+	}
+
 	for _, key := range storesKeys {
 		storeParams := rs.storesParams[key]
 		commitID := rs.getCommitID(infos, key.Name())

types/module/module.go

@@ -22,10 +22,6 @@ package module
 import (
 	"context"
 	"encoding/json"
-	"errors"
-	"fmt"
-	"maps"
-	"slices"
 	"sort"
 
 	abci "github.com/cometbft/cometbft/api/cometbft/abci/v1"
@@ -670,9 +666,18 @@ func (m Manager) RunMigrations(ctx context.Context, cfg Configurator, fromVM app
 		modules = DefaultMigrationsOrder(m.ModuleNames())
 	}
 
-	sdkCtx := sdk.UnwrapSDKContext(ctx)
-	updatedVM := appmodule.VersionMap{}
-	for _, moduleName := range modules {
+	updatedVM := make(VersionMap)
+	// for deterministic iteration order
+	// (as some migrations depend on other modules
+	// and the order of executing migrations matters)
+	// TODO: make the order user-configurable?
+	sortedModNames := make([]string, 0, len(m.Modules))
+	for key := range m.Modules {
+		sortedModNames = append(sortedModNames, key)
+	}
+	sort.Strings(sortedModNames)
+
+	for _, moduleName := range sortedModNames {
 		module := m.Modules[moduleName]
 		fromVersion, exists := fromVM[moduleName]
 		toVersion := uint64(0)

x/upgrade/keeper/keeper.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strconv"
 
 	"github.com/hashicorp/go-metrics"