Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Problem: there's no compact historical state storage #722

Closed
wants to merge 17 commits into from
Closed

Problem: there's no compact historical state storage #722

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
3bcb198
Problem: there's no compact historical state storage
yihuang Sep 27, 2022
7d1201a
fix streamer config
yihuang Nov 28, 2022
66e50d4
fix streamer decoding
yihuang Nov 28, 2022
850ed66
fix linters
yihuang Nov 28, 2022
1f6eebb
home path
yihuang Nov 28, 2022
856b18f
fix integration test
yihuang Nov 28, 2022
7b93785
fix unit test
yihuang Nov 28, 2022
234ad32
fix upgrade test
yihuang Nov 29, 2022
14be5b6
update to upstream
yihuang Dec 7, 2022
3af8413
Merge branch 'main' into version-store
yihuang Dec 7, 2022
b686422
fix gravity test
yihuang Dec 7, 2022
8e981c4
Merge remote-tracking branch 'fork/version-store' into version-store
yihuang Dec 7, 2022
eef8781
revert mdbx from this PR
yihuang Dec 9, 2022
611e2f3
remove FIXME
yihuang Dec 9, 2022
bde66c1
remove unused functions
yihuang Dec 9, 2022
659a562
Update integration_tests/test_benchmark_storage.py
yihuang Dec 9, 2022
f46daef
remove TODO
yihuang Dec 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 47 additions & 28 deletions app/app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
package app

import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sync"

"github.com/crypto-org-chain/cronos/x/cronos"
"github.com/crypto-org-chain/cronos/x/cronos/middleware"

"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/codec/types"
"github.com/cosmos/cosmos-sdk/server"
"github.com/gorilla/mux"
"github.com/rakyll/statik/fs"
"github.com/spf13/cast"
Expand All @@ -20,13 +22,12 @@ import (
dbm "github.com/tendermint/tm-db"

"github.com/cosmos/cosmos-sdk/baseapp"
"github.com/cosmos/cosmos-sdk/client/flags"
"github.com/cosmos/cosmos-sdk/client/grpc/tmservice"
"github.com/cosmos/cosmos-sdk/codec"
"github.com/cosmos/cosmos-sdk/server/api"
"github.com/cosmos/cosmos-sdk/server/config"
servertypes "github.com/cosmos/cosmos-sdk/server/types"
"github.com/cosmos/cosmos-sdk/store/streaming/file"
"github.com/cosmos/cosmos-sdk/store/streaming"
storetypes "github.com/cosmos/cosmos-sdk/store/types"
"github.com/cosmos/cosmos-sdk/testutil/testdata"
sdk "github.com/cosmos/cosmos-sdk/types"
Expand Down Expand Up @@ -121,8 +122,9 @@ import (
gravitytypes "github.com/peggyjv/gravity-bridge/module/v2/x/gravity/types"

// this line is used by starport scaffolding # stargate/app/moduleImport
cronosappclient "github.com/crypto-org-chain/cronos/client"
"github.com/crypto-org-chain/cronos/x/cronos"

"github.com/crypto-org-chain/cronos/versiondb"
"github.com/crypto-org-chain/cronos/versiondb/tmdb"
cronosclient "github.com/crypto-org-chain/cronos/x/cronos/client"
cronoskeeper "github.com/crypto-org-chain/cronos/x/cronos/keeper"
evmhandlers "github.com/crypto-org-chain/cronos/x/cronos/keeper/evmhandlers"
Expand All @@ -146,8 +148,6 @@ const (
//
// NOTE: In the SDK, the default value is 255.
AddrLen = 20

FileStreamerDirectory = "file_streamer"
)

// this line is used by starport scaffolding # stargate/wasm/app/enabledProposals
Expand Down Expand Up @@ -347,29 +347,48 @@ func New(
tkeys := sdk.NewTransientStoreKeys(paramstypes.TStoreKey, evmtypes.TransientKey, feemarkettypes.TransientKey)
memKeys := sdk.NewMemoryStoreKeys(capabilitytypes.MemStoreKey)

// load state streaming if enabled
if _, _, err := streaming.LoadStreamingServices(bApp, appOpts, appCodec, keys); err != nil {
fmt.Printf("failed to load state streaming: %s", err)
os.Exit(1)
}

// configure state listening capabilities using AppOptions
// we are doing nothing with the returned streamingServices and waitGroup in this case
// Only support file streamer right now.
if cast.ToString(appOpts.Get(cronosappclient.FlagStreamers)) == "file" {
streamingDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", FileStreamerDirectory)
if err := os.MkdirAll(streamingDir, os.ModePerm); err != nil {
panic(err)
}

// default to exposing all
exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys))
for _, storeKey := range keys {
exposeStoreKeys = append(exposeStoreKeys, storeKey)
}
service, err := file.NewStreamingService(streamingDir, "", exposeStoreKeys, appCodec)
if err != nil {
panic(err)
}
bApp.SetStreamingService(service)

wg := new(sync.WaitGroup)
if err := service.Stream(wg); err != nil {
panic(err)
streamers := cast.ToStringSlice(appOpts.Get("store.streamers"))
for _, streamerName := range streamers {
if streamerName == "versiondb" {
dataDir := filepath.Join(homePath, "data", "versiondb")
if err := os.MkdirAll(dataDir, os.ModePerm); err != nil {
panic(err)
}
backendType := server.GetAppDBBackend(appOpts)
plainDB, err := dbm.NewDB("plain", backendType, dataDir)
if err != nil {
panic(err)
}
historyDB, err := dbm.NewDB("history", backendType, dataDir)
if err != nil {
panic(err)
}
changesetDB, err := dbm.NewDB("changeset", backendType, dataDir)
if err != nil {
panic(err)
}
versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB)

// default to exposing all
exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys))
for _, storeKey := range keys {
exposeStoreKeys = append(exposeStoreKeys, storeKey)
}
Comment on lines +382 to +384

Check warning

Code scanning / CodeQL

Iteration over map

Iteration over map may be a possible source of non-determinism
service := versiondb.NewStreamingService(versionDB, exposeStoreKeys)
bApp.SetStreamingService(service)
qms := versiondb.NewMultiStore(versionDB, exposeStoreKeys)
qms.MountTransientStores(tkeys)
qms.MountMemoryStores(memKeys)
bApp.SetQueryMultiStore(qms)
break
}
}

Expand Down
3 changes: 0 additions & 3 deletions client/flags.go
View file
Open in desktop

This file was deleted.

4 changes: 1 addition & 3 deletions cmd/cronosd/cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ import (
ethermint "github.com/evmos/ethermint/types"

"github.com/crypto-org-chain/cronos/app"
cronosclient "github.com/crypto-org-chain/cronos/client"
// this line is used by starport scaffolding # stargate/root/import
)

Expand Down Expand Up @@ -150,7 +149,6 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) {
func addModuleInitFlags(startCmd *cobra.Command) {
crisis.AddModuleInitFlags(startCmd)
cronos.AddModuleInitFlags(startCmd)
startCmd.Flags().String(cronosclient.FlagStreamers, "", "Enable streamers, only file streamer is supported right now")
// this line is used by starport scaffolding # stargate/root/initFlags
}

Expand Down Expand Up @@ -271,7 +269,7 @@ func (a appCreator) newApp(logger log.Logger, db dbm.DB, traceStore io.Writer, a
baseapp.SetIndexEvents(cast.ToStringSlice(appOpts.Get(server.FlagIndexEvents))),
baseapp.SetSnapshot(snapshotStore, snapshotOptions),
baseapp.SetIAVLCacheSize(cast.ToInt(appOpts.Get(server.FlagIAVLCacheSize))),
baseapp.SetIAVLDisableFastNode(cast.ToBool(appOpts.Get(server.FlagIAVLFastNode))),
baseapp.SetIAVLDisableFastNode(cast.ToBool(appOpts.Get(server.FlagDisableIAVLFastNode))),
)
}

Expand Down
1 change: 1 addition & 0 deletions default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ buildGoApplication rec {
"!/app/"
"!/cmd/"
"!/client/"
"!/versiondb/"
"!go.mod"
"!go.sum"
"!gomod2nix.toml"
Expand Down
38 changes: 21 additions & 17 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@ go 1.18

require (
cosmossdk.io/math v1.0.0-beta.3
github.com/RoaringBitmap/roaring v1.2.1
github.com/armon/go-metrics v0.4.1
github.com/cosmos/cosmos-sdk v0.46.3
github.com/cosmos/gogoproto v1.4.3
github.com/cosmos/ibc-go/v5 v5.0.0
github.com/ethereum/go-ethereum v1.10.19
github.com/evmos/ethermint v0.6.1-0.20221003153722-491c3da7ebd7
Expand All @@ -16,14 +18,14 @@ require (
github.com/peggyjv/gravity-bridge/module/v2 v2.0.0-20220420162017-838c0d25e974
github.com/rakyll/statik v0.1.7
github.com/spf13/cast v1.5.0
github.com/spf13/cobra v1.5.0
github.com/spf13/cobra v1.6.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.0
github.com/tendermint/tendermint v0.34.22
github.com/tendermint/tendermint v0.34.24
github.com/tendermint/tm-db v0.6.7
google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc
google.golang.org/grpc v1.50.0
google.golang.org/protobuf v1.28.1
google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a
google.golang.org/grpc v1.50.1
google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8
gopkg.in/yaml.v2 v2.4.0
)

Expand All @@ -45,6 +47,7 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
github.com/bgentry/speakeasy v0.1.0 // indirect
github.com/bits-and-blooms/bitset v1.2.0 // indirect
github.com/btcsuite/btcd v0.22.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
Expand All @@ -60,7 +63,7 @@ require (
github.com/cosmos/cosmos-proto v1.0.0-alpha7 // indirect
github.com/cosmos/go-bip39 v1.0.0 // indirect
github.com/cosmos/gorocksdb v1.2.0 // indirect
github.com/cosmos/iavl v0.19.3 // indirect
github.com/cosmos/iavl v0.19.4 // indirect
github.com/cosmos/ledger-cosmos-go v0.11.1 // indirect
github.com/cosmos/ledger-go v0.9.2 // indirect
github.com/creachadair/taskgroup v0.3.2 // indirect
Expand Down Expand Up @@ -91,8 +94,8 @@ require (
github.com/golang/glog v1.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/orderedcode v0.0.1 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
Expand Down Expand Up @@ -121,7 +124,7 @@ require (
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jmhodges/levigo v1.0.0 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/lib/pq v1.10.6 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
Expand All @@ -135,6 +138,7 @@ require (
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-testing-interface v1.0.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/mtibben/percent v0.2.1 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
Expand All @@ -145,7 +149,7 @@ require (
github.com/prometheus/client_golang v1.12.2 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.34.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/prometheus/tsdb v0.7.1 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/regen-network/cosmos-proto v0.3.1 // indirect
Expand All @@ -170,14 +174,14 @@ require (
github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect
golang.org/x/net v0.1.0 // indirect
golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 // indirect
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect
golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 // indirect
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/term v0.1.0 // indirect
golang.org/x/text v0.4.0 // indirect
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
google.golang.org/api v0.93.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand All @@ -189,7 +193,7 @@ require (
)

replace (
github.com/cosmos/cosmos-sdk => github.com/cosmos/cosmos-sdk v0.46.2
github.com/cosmos/cosmos-sdk => github.com/cosmos/cosmos-sdk v0.46.7-0.20221206163158-45d2f08e0e1d
github.com/ethereum/go-ethereum => github.com/ethereum/go-ethereum v1.10.19

// Fix upstream GHSA-h395-qcrw-5vmq vulnerability.
Expand Down
Loading