From c42912d9197dd287462daf3dea210bf38901660b Mon Sep 17 00:00:00 2001 From: Hleb Albau Date: Thu, 20 Dec 2018 18:17:56 +0300 Subject: [PATCH] Various bug fixes --- app/app.go | 37 +++++++++++----------- app/rpc.go | 10 ++++-- client/helpers.go | 27 ++++++++++++++++ client/http_client.go | 68 ++++++++++++++++++++++++++++++----------- client/spec.go | 26 ++++++++-------- daemon/rpc/bandwidth.go | 9 ++++++ daemon/rpc/routes.go | 9 +++--- docs/rpc.md | 10 ++++++ go.sum | 1 + testnet/checklist.md | 7 +++-- testnet/config.toml | 2 +- wiki/cid.go | 6 ++-- wiki/index.go | 17 +++++++---- x/link/handler.go | 21 ++----------- x/rank/calculate_gpu.go | 2 +- x/rank/state.go | 2 +- 16 files changed, 165 insertions(+), 89 deletions(-) create mode 100644 client/helpers.go create mode 100644 daemon/rpc/bandwidth.go diff --git a/app/app.go b/app/app.go index b1a5007f..d955f868 100644 --- a/app/app.go +++ b/app/app.go @@ -123,19 +123,20 @@ func NewCyberdApp( cdc := MakeCodec() dbKeys := CyberdAppDbKeys{ - main: sdk.NewKVStoreKey("main"), - acc: sdk.NewKVStoreKey("acc"), - cidNum: sdk.NewKVStoreKey("cid_index"), - links: sdk.NewKVStoreKey("links"), - rank: sdk.NewKVStoreKey("rank"), - stake: sdk.NewKVStoreKey("stake"), - fees: sdk.NewKVStoreKey("fee"), - tStake: sdk.NewTransientStoreKey("transient_stake"), - keyDistr: sdk.NewKVStoreKey("distr"), - slashing: sdk.NewKVStoreKey("slashing"), - params: sdk.NewKVStoreKey("params"), - tParams: sdk.NewTransientStoreKey("transient_params"), - accBandwidth: sdk.NewKVStoreKey("acc_bandwidth"), + main: sdk.NewKVStoreKey("main"), + acc: sdk.NewKVStoreKey("acc"), + cidNum: sdk.NewKVStoreKey("cid_index"), + cidNumReverse: sdk.NewKVStoreKey("cid_index_reverse"), + links: sdk.NewKVStoreKey("links"), + rank: sdk.NewKVStoreKey("rank"), + stake: sdk.NewKVStoreKey("stake"), + fees: sdk.NewKVStoreKey("fee"), + tStake: sdk.NewTransientStoreKey("transient_stake"), + keyDistr: sdk.NewKVStoreKey("distr"), + slashing: sdk.NewKVStoreKey("slashing"), + params: sdk.NewKVStoreKey("params"), + tParams: sdk.NewTransientStoreKey("transient_params"), + accBandwidth: sdk.NewKVStoreKey("acc_bandwidth"), } ms := store.NewMainKeeper(dbKeys.main) @@ -198,7 +199,7 @@ func NewCyberdApp( // register message routes app.Router(). AddRoute("bank", sdkbank.NewHandler(app.bankKeeper)). - AddRoute("link", link.NewLinksHandler(app.cidNumKeeper, app.linkIndexedKeeper, app.accountKeeper)). + AddRoute("link", link.NewLinksHandler(app.cidNumKeeper, &app.linkIndexedKeeper, app.accountKeeper)). AddRoute("stake", stake.NewHandler(app.stakeKeeper)). AddRoute("slashing", slashing.NewHandler(app.slashingKeeper)) @@ -213,7 +214,7 @@ func NewCyberdApp( // mount the multistore and load the latest state app.MountStores( - dbKeys.main, dbKeys.acc, dbKeys.cidNum, dbKeys.links, dbKeys.rank, dbKeys.stake, + dbKeys.main, dbKeys.acc, dbKeys.cidNum, dbKeys.cidNumReverse, dbKeys.links, dbKeys.rank, dbKeys.stake, dbKeys.slashing, dbKeys.params, dbKeys.keyDistr, dbKeys.fees, dbKeys.accBandwidth, ) app.MountStoresTransient(dbKeys.tParams, dbKeys.tStake) @@ -377,10 +378,8 @@ func (app *CyberdApp) DeliverTx(txBytes []byte) (res abci.ResponseDeliverTx) { } else { resp := app.BaseApp.DeliverTx(txBytes) - if resp.Code == 0 { - app.bandwidthMeter.ConsumeAccBandwidth(ctx, accBw, txCost) - app.curBlockSpentBandwidth = app.curBlockSpentBandwidth + uint64(txCost) - } + app.bandwidthMeter.ConsumeAccBandwidth(ctx, accBw, txCost) + app.curBlockSpentBandwidth = app.curBlockSpentBandwidth + uint64(txCost) return abci.ResponseDeliverTx{ Code: uint32(resp.Code), diff --git a/app/rpc.go b/app/rpc.go index 7cdce9db..b3351166 100644 --- a/app/rpc.go +++ b/app/rpc.go @@ -11,8 +11,8 @@ import ( ) type RankedCid struct { - cid cbdlink.Cid - rank float64 + Cid cbdlink.Cid `json:"cid"` + Rank float64 `amino:"unsafe" json:"rank"` } func (app *CyberdApp) RpcContext() sdk.Context { @@ -35,7 +35,7 @@ func (app *CyberdApp) Search(cid string, page, perPage int) ([]RankedCid, int, e result := make([]RankedCid, 0, len(rankedCidNumbers)) for _, c := range rankedCidNumbers { - result = append(result, RankedCid{cid: app.cidNumKeeper.GetCid(ctx, c.GetNumber()), rank: c.GetRank()}) + result = append(result, RankedCid{Cid: app.cidNumKeeper.GetCid(ctx, c.GetNumber()), Rank: c.GetRank()}) } return result, size, nil @@ -65,3 +65,7 @@ func (app *CyberdApp) IsLinkExist(from cbdlink.Cid, to cbdlink.Cid, address sdk. return false } + +func (app *CyberdApp) CurrentBandwidthPrice() float64 { + return app.currentCreditPrice +} diff --git a/client/helpers.go b/client/helpers.go new file mode 100644 index 00000000..f33dd280 --- /dev/null +++ b/client/helpers.go @@ -0,0 +1,27 @@ +package client + +import ( + cbdlink "github.com/cybercongress/cyberd/x/link/types" +) + +type CidsFilter map[cbdlink.Cid]map[cbdlink.Cid]struct{} + +func (cf CidsFilter) Put(from cbdlink.Cid, to cbdlink.Cid) { + + cidLinks := cf[from] + if cidLinks == nil { + cidLinks = make(map[cbdlink.Cid]struct{}) + } + cidLinks[to] = struct{}{} + cf[from] = cidLinks +} + +func (cf CidsFilter) Contains(from cbdlink.Cid, to cbdlink.Cid) bool { + + cidLinks := cf[from] + if cidLinks == nil { + return false + } + _, contains := cidLinks[to] + return contains +} diff --git a/client/http_client.go b/client/http_client.go index 991a157e..0a1596ad 100644 --- a/client/http_client.go +++ b/client/http_client.go @@ -9,8 +9,10 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" authtxb "github.com/cosmos/cosmos-sdk/x/auth/client/txbuilder" "github.com/cybercongress/cyberd/app" - cbd "github.com/cybercongress/cyberd/app/types" + "github.com/cybercongress/cyberd/daemon/rpc" + bwtps "github.com/cybercongress/cyberd/x/bandwidth/types" "github.com/cybercongress/cyberd/x/link" + cbdlink "github.com/cybercongress/cyberd/x/link/types" tdmClient "github.com/tendermint/tendermint/rpc/client" "github.com/tendermint/tendermint/rpc/lib/client" "os" @@ -29,8 +31,8 @@ type HttpCyberdClient struct { // fields used by local keys store to sing transactions passphrase string fromAddress sdk.AccAddress - cliCtx cli.CLIContext - txBuilder authtxb.TxBuilder + cliCtx *cli.CLIContext + txBuilder *authtxb.TxBuilder } func NewHttpCyberdClient(nodeUrl string, passphrase string, singAddr string) CyberdClient { @@ -58,11 +60,18 @@ func NewHttpCyberdClient(nodeUrl string, passphrase string, singAddr string) Cyb }.WithCodec(cdc).WithAccountDecoder(cdc) accountNumber, _ := cliCtx.GetAccountNumber(addr) + seq, err := cliCtx.GetAccountSequence(addr) + + if err != nil { + panic(err) + } + txBuilder := authtxb.TxBuilder{ Gas: 1000000, ChainID: status.NodeInfo.Network, AccountNumber: accountNumber, Codec: cdc, + Sequence: seq, } return HttpCyberdClient{ @@ -73,8 +82,8 @@ func NewHttpCyberdClient(nodeUrl string, passphrase string, singAddr string) Cyb passphrase: passphrase, fromAddress: addr, - cliCtx: cliCtx, - txBuilder: txBuilder, + cliCtx: &cliCtx, + txBuilder: &txBuilder, } } @@ -82,37 +91,59 @@ func (c HttpCyberdClient) GetChainId() string { return c.chainId } -/*func (c HttpCyberdClient) GetCurrentBandwidthCreditPrice() (float64, error) { - +func (c HttpCyberdClient) IsLinkExist(from cbdlink.Cid, to cbdlink.Cid, addr sdk.AccAddress) (result bool, err error) { + _, err = c.httpClient.Call("is_link_exist", + map[string]interface{}{"from": from, "to": to, "address": addr.String()}, + &result, + ) + return } -func (c HttpCyberdClient) GetAccount(address sdk.AccAddress) (auth.Account, error) { - +func (c HttpCyberdClient) GetCurrentBandwidthCreditPrice() (float64, error) { + result := &rpc.ResultBandwidthPrice{} + _, err := c.httpClient.Call("current_bandwidth_price", map[string]interface{}{}, &result) + return result.Price, err } -func (c HttpCyberdClient) GetAccountBandwidth(address sdk.AccAddress) (bdwth.AcсBandwidth, error) { - -}*/ +func (c HttpCyberdClient) GetAccountBandwidth() (result bwtps.AcсBandwidth, err error) { + _, err = c.httpClient.Call("account_bandwidth", + map[string]interface{}{"address": c.fromAddress.String()}, &result) + return +} func (c HttpCyberdClient) SubmitLinkSync(link Link) error { return c.SubmitLinksSync([]Link{link}) } func (c HttpCyberdClient) SubmitLinksSync(links []Link) error { - msges := make([]sdk.Msg, 0, len(links)) + + // used to remove duplicated items + var filter = make(CidsFilter) + msges := make([]sdk.Msg, 0) + for _, l := range links { - msges = append(msges, link.NewMsg(c.fromAddress, cbd.Cid(l.From), cbd.Cid(l.To))) + + if filter.Contains(l.From, l.To) { + continue + } + + exists, err := c.IsLinkExist(l.From, l.To, c.fromAddress) + if err != nil { + return err + } + if !exists { + msges = append(msges, link.NewMsg(c.fromAddress, l.From, l.To)) + } + filter.Put(l.From, l.To) } return c.BroadcastTx(msges) } func (c HttpCyberdClient) BroadcastTx(msgs []sdk.Msg) error { - seq, err := c.cliCtx.GetAccountSequence(c.fromAddress) - if err != nil { - return err + if len(msgs) == 0 { + return nil } - c.txBuilder.Sequence = seq txBytes, err := c.txBuilder.BuildAndSign(c.cliCtx.From, c.passphrase, msgs) if err != nil { @@ -130,6 +161,7 @@ func (c HttpCyberdClient) BroadcastTx(msgs []sdk.Msg) error { if result.Code != 0 { return errors.New(string(result.Log)) } + c.txBuilder.Sequence = c.txBuilder.Sequence + 1 return nil } diff --git a/client/spec.go b/client/spec.go index 66510448..44d3d108 100644 --- a/client/spec.go +++ b/client/spec.go @@ -1,12 +1,14 @@ package client import ( - cbd "github.com/cybercongress/cyberd/app/types" + sdk "github.com/cosmos/cosmos-sdk/types" + bwtps "github.com/cybercongress/cyberd/x/bandwidth/types" + cbdlink "github.com/cybercongress/cyberd/x/link/types" ) type Link struct { - From cbd.Cid - To cbd.Cid + From cbdlink.Cid + To cbdlink.Cid } type CyberdClient interface { @@ -15,17 +17,17 @@ type CyberdClient interface { // returns current connected node chain id GetChainId() string - /* // get current bandwidth credits price - // price 1 is price for situation, when all users use all their bandwidth (all blocks are filled for 100%) - // if price < 1, that means blocks filled partially, thus allow more active users to do more transactions - // if price > 1, that means network is under high load. - GetCurrentBandwidthCreditPrice() (float64, error) + // returns, if given link already exists + IsLinkExist(from cbdlink.Cid, to cbdlink.Cid, addr sdk.AccAddress) (result bool, err error) - // returns account for given address - GetAccount(address sdk.AccAddress) (auth.Account, error) + // get current bandwidth credits price + // price 1 is price for situation, when all users use all their bandwidth (all blocks are filled for 100%) + // if price < 1, that means blocks filled partially, thus allow more active users to do more transactions + // if price > 1, that means network is under high load. + GetCurrentBandwidthCreditPrice() (float64, error) - // returns account bandwidth information for given account - GetAccountBandwidth(address sdk.AccAddress) (bdwth.AcсBandwidth, error)*/ + // returns account bandwidth information for given account + GetAccountBandwidth() (bwtps.AcсBandwidth, error) // links two cids for given user // this method also should check, either cids are correct cids and given user is msg signer diff --git a/daemon/rpc/bandwidth.go b/daemon/rpc/bandwidth.go new file mode 100644 index 00000000..fe87a10c --- /dev/null +++ b/daemon/rpc/bandwidth.go @@ -0,0 +1,9 @@ +package rpc + +type ResultBandwidthPrice struct { + Price float64 `amino:"unsafe" json:"price"` +} + +func CurrentBandwidthPrice() (*ResultBandwidthPrice, error) { + return &ResultBandwidthPrice{cyberdApp.CurrentBandwidthPrice()}, nil +} diff --git a/daemon/rpc/routes.go b/daemon/rpc/routes.go index af4bab51..aebf59d7 100644 --- a/daemon/rpc/routes.go +++ b/daemon/rpc/routes.go @@ -13,10 +13,11 @@ func SetCyberdApp(cApp *app.CyberdApp) { } var Routes = map[string]*rpcserver.RPCFunc{ - "search": rpcserver.NewRPCFunc(Search, "cid,page,perPage"), - "account": rpcserver.NewRPCFunc(Account, "address"), - "account_bandwidth": rpcserver.NewRPCFunc(AccountBandwidth, "address"), - "is_link_exist": rpcserver.NewRPCFunc(IsLinkExist, "from, to, address"), + "search": rpcserver.NewRPCFunc(Search, "cid,page,perPage"), + "account": rpcserver.NewRPCFunc(Account, "address"), + "account_bandwidth": rpcserver.NewRPCFunc(AccountBandwidth, "address"), + "is_link_exist": rpcserver.NewRPCFunc(IsLinkExist, "from,to,address"), + "current_bandwidth_price": rpcserver.NewRPCFunc(CurrentBandwidthPrice, ""), } func init() { diff --git a/docs/rpc.md b/docs/rpc.md index 8c2b6d4f..8625609b 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -40,6 +40,7 @@ the method name for further details such as parameter and return information. |2|[account](#account)|Get account nonce, pubkey, number, and coins.| |3|[account_bandwidth](#account-bandwidth)|Get account bandwidth info for current height.| |4|[is_link_exist](#link-exist)|Return true, if given link exist.| +|5|[current_bandwidth_price](#current-bandwidth-price)|Returns current bandwidth credit price.| ### Method Details @@ -80,6 +81,15 @@ the method name for further details such as parameter and return information. |Description|Return true, if given link exist.| |[Return to Overview](#method-overview)
+ + +| | | +|---|---| +|Method|current_bandwidth_price| +|Parameters|| +|Description|Returns current bandwidth credit price.| +|[Return to Overview](#method-overview)
+ ***

diff --git a/go.sum b/go.sum index 8365dbaa..50bce205 100644 --- a/go.sum +++ b/go.sum @@ -128,6 +128,7 @@ github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2 h1:GnOzE5fEFN3b2z github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/tendermint/btcd v0.0.0-20180816174608-e5840949ff4f h1:R0wLxgASGMoRQTF/dCSk4N+M3j9DLyPDzDff2WtCg/I= github.com/tendermint/btcd v0.0.0-20180816174608-e5840949ff4f/go.mod h1:DC6/m53jtQzr/NFmMNEu0rxf18/ktVoVtMrnDD5pN+U= +github.com/tendermint/crypto v0.0.0-20180820045704-3764759f34a5 h1:u8i49c+BxloX3XQ55cvzFNXplizZP/q00i+IlttUjAU= github.com/tendermint/crypto v0.0.0-20180820045704-3764759f34a5/go.mod h1:z4YtwM70uOnk8h0pjJYlj3zdYwi9l03By6iAIF5j/Pk= github.com/tendermint/go-amino v0.14.1 h1:o2WudxNfdLNBwMyl2dqOJxiro5rfrEaU0Ugs6offJMk= github.com/tendermint/go-amino v0.14.1/go.mod h1:i/UKE5Uocn+argJJBb12qTZsCDBcAYMbR92AaJVmKso= diff --git a/testnet/checklist.md b/testnet/checklist.md index ce9c447f..17b396c7 100644 --- a/testnet/checklist.md +++ b/testnet/checklist.md @@ -4,8 +4,9 @@ 2. adjust genesis supply in x/mint 3. reset old state 4. change genesis file chain-id -5. generate new first validator keys -6. upadte genesis gen tx +5. upadte genesis gen tx +6. update readme.md testnet lable +7. Update docs/run_validator.md guide ## Useful commands @@ -21,7 +22,7 @@ scp -P 33324 /home/hlb/projects/cyberd/testnet/genesis.json earth@earth.cybernode.ai:/cyberdata/cyberd/config/ scp -P 33324 /home/hlb/projects/cyberd/testnet/config.toml earth@earth.cybernode.ai:/cyberdata/cyberd/config/ # Copy from earth -scp -P 33324 earth@earth.cybernode.ai:/path/file /host/path/file +scp -P 33324 earth@93.125.26.210:/cyberdata/cyberd/config/priv_validator.json ~/.cyberd/config/ ``` ```bash diff --git a/testnet/config.toml b/testnet/config.toml index f216e6b2..12c30597 100644 --- a/testnet/config.toml +++ b/testnet/config.toml @@ -22,7 +22,7 @@ db_backend = "leveldb" db_dir = "data" # Output level for logging, including package level options -log_level = "main:info,state:info,*:error" +log_level = "*:error" # Output format: 'plain' (colored text) or 'json' log_format = "plain" diff --git a/wiki/cid.go b/wiki/cid.go index 158a99a9..e736d7b6 100644 --- a/wiki/cid.go +++ b/wiki/cid.go @@ -1,7 +1,7 @@ package main import ( - cbd "github.com/cybercongress/cyberd/app/types" + links "github.com/cybercongress/cyberd/x/link/types" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" ) @@ -13,7 +13,7 @@ var pref = cid.Prefix{ MhLength: -1, // default length } -func Cid(data string) cbd.Cid { +func Cid(data string) links.Cid { result, _ := pref.Sum([]byte(data)) - return cbd.Cid(result.String()) + return links.Cid(result.String()) } diff --git a/wiki/index.go b/wiki/index.go index 01183ede..5c4932fa 100644 --- a/wiki/index.go +++ b/wiki/index.go @@ -2,11 +2,11 @@ package main import ( "bufio" + "fmt" "github.com/cybercongress/cyberd/client" "os" "regexp" "strings" - "time" "unicode" ) @@ -58,11 +58,16 @@ func ContinueIndex(cbdClient client.CyberdClient) { links = append(links, client.Link{From: Cid(id), To: Cid(page)}) counter++ - if len(links) == 1000 { - println(split[1]) - println(counter) - time.Sleep(500 * time.Millisecond) - err := cbdClient.SubmitLinksSync(links) + if len(links) == 100 { + fmt.Printf("%d %s\n", counter, split[1]) + + accBw, err := cbdClient.GetAccountBandwidth() + if err == nil { + per := int64(100 * float64(accBw.RemainedValue) / float64(accBw.MaxValue)) + fmt.Printf("Remaining acc bw: %d %v%%\n", accBw.RemainedValue, per) + } + + err = cbdClient.SubmitLinksSync(links) if err != nil { panic(err.Error()) } diff --git a/x/link/handler.go b/x/link/handler.go index 50f7e589..fcfd75c2 100644 --- a/x/link/handler.go +++ b/x/link/handler.go @@ -5,7 +5,6 @@ import ( "github.com/cosmos/cosmos-sdk/x/auth" cbd "github.com/cybercongress/cyberd/types" "github.com/cybercongress/cyberd/x/link/keeper" - "github.com/cybercongress/cyberd/x/link/types" cbdlink "github.com/cybercongress/cyberd/x/link/types" ) @@ -14,15 +13,13 @@ import ( // ils - links storage // as - account storage // imms - in-memory storage -func NewLinksHandler(cis keeper.CidNumberKeeper, ls keeper.LinkIndexedKeeper, as auth.AccountKeeper) sdk.Handler { - - getCidNumber := GetCidNumberFunc(cis) +func NewLinksHandler(cis keeper.CidNumberKeeper, ls *keeper.LinkIndexedKeeper, as auth.AccountKeeper) sdk.Handler { return func(ctx sdk.Context, msg sdk.Msg) sdk.Result { linkMsg := msg.(Msg) - fromCidNumber := getCidNumber(ctx, linkMsg.From) - toCidNumber := getCidNumber(ctx, linkMsg.To) + fromCidNumber := cis.GetOrPutCidNumber(ctx, linkMsg.From) + toCidNumber := cis.GetOrPutCidNumber(ctx, linkMsg.To) accNumber := cbd.AccNumber(as.GetAccount(ctx, linkMsg.Address).GetAccountNumber()) link := cbdlink.NewLink(fromCidNumber, toCidNumber, accNumber) @@ -38,15 +35,3 @@ func NewLinksHandler(cis keeper.CidNumberKeeper, ls keeper.LinkIndexedKeeper, as return sdk.Result{Code: cbd.CodeOK, Codespace: cbd.CodespaceCbd} } } - -func GetCidNumberFunc(cis keeper.CidNumberKeeper) func(sdk.Context, cbdlink.Cid) cbdlink.CidNumber { - - return func(ctx sdk.Context, cid types.Cid) types.CidNumber { - - index, exist := cis.GetCidNumber(ctx, cid) - if !exist { // new cid - index = cis.GetOrPutCidNumber(ctx, cid) - } - return index - } -} diff --git a/x/rank/calculate_gpu.go b/x/rank/calculate_gpu.go index 6cd01ea9..6fec78b1 100644 --- a/x/rank/calculate_gpu.go +++ b/x/rank/calculate_gpu.go @@ -44,7 +44,7 @@ func calculateRankGPU(ctx *CalculationContext, logger log.Logger) ([]float64, in /* Fill values */ // todo parallel this - for i := uint64(0); i < cidsCount; i++ { + for i := int64(0); i < cidsCount; i++ { if inLinks, sortedCids, ok := ctx.GetSortedInLinks(types.CidNumber(i)); ok { for _, cid := range sortedCids { diff --git a/x/rank/state.go b/x/rank/state.go index f6a9fc0e..e3d00e60 100644 --- a/x/rank/state.go +++ b/x/rank/state.go @@ -64,7 +64,7 @@ func (s *RankState) buildCidRankedLinksIndex(cidsCount int64) { for cidNumber := CidNumber(0); cidNumber < CidNumber(cidsCount); cidNumber++ { cidOutLinks := s.linkIndex.GetOutLinks()[cidNumber] cidSortedByRankLinkedCids := s.getLinksSortedByRank(cidOutLinks) - s.cidRankedLinksIndex[cidNumber] = cidSortedByRankLinkedCids + newIndex[cidNumber] = cidSortedByRankLinkedCids } s.cidRankedLinksIndex = newIndex