diff --git a/.gitignore b/.gitignore index f55eafc6..2c5b3323 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ node_modules/ initial-rank initial_guess_impact_results +*enwiki* diff --git a/cosmos/poc/testnet/config.toml b/cosmos/poc/testnet/config.toml index 5d4ad1d2..7955c87c 100644 --- a/cosmos/poc/testnet/config.toml +++ b/cosmos/poc/testnet/config.toml @@ -143,7 +143,7 @@ private_peer_ids = "" ##### mempool configuration options ##### [mempool] -recheck = false +recheck = true recheck_empty = true broadcast = true wal_dir = "data/mempool.wal" diff --git a/cosmos/poc/wiki/client.go b/cosmos/poc/wiki/client.go new file mode 100644 index 00000000..77a61df7 --- /dev/null +++ b/cosmos/poc/wiki/client.go @@ -0,0 +1,134 @@ +package main + +import ( + "fmt" + "github.com/cosmos/cosmos-sdk/client/context" + cli "github.com/cosmos/cosmos-sdk/client/context" + "github.com/cosmos/cosmos-sdk/client/keys" + cskeys "github.com/cosmos/cosmos-sdk/crypto/keys" + sdk "github.com/cosmos/cosmos-sdk/types" + authcmd "github.com/cosmos/cosmos-sdk/x/auth/client/cli" + authtxb "github.com/cosmos/cosmos-sdk/x/auth/client/txbuilder" + "github.com/cybercongress/cyberd/cosmos/poc/app" + "github.com/cybercongress/cyberd/cosmos/poc/app/storage" + "github.com/cybercongress/cyberd/cosmos/poc/claim/common" + "github.com/spf13/viper" + rpcclient "github.com/tendermint/tendermint/rpc/client" + "os" + "time" +) + +func InitAddLink() func([]Link) { + + chainId := viper.GetString(common.FlagChainId) + address := viper.GetString(common.FlagAddress) + addr, name := accountFromAddress(address) + + cdc := app.MakeCodec() + cliCtx := newCLIContext(name, chainId). + WithCodec(cdc). + WithAccountDecoder(authcmd.GetAccountDecoder(cdc)) + + accountNumber, _ := cliCtx.GetAccountNumber(addr) + seq, _ := cliCtx.GetAccountSequence(addr) + + txCtx := authtxb.TxBuilder{ + ChainID: chainId, + Gas: 10000000, + AccountNumber: accountNumber, + Sequence: seq, + Fee: "", + Memo: "", + Codec: cdc, + } + + return func(links []Link) { + + msges := make([]sdk.Msg, 0, len(links)) + for _, link := range links { + msges = append(msges, app.NewMsgLink(addr, storage.Cid(link.from), storage.Cid(link.to))) + } + + sendTx(address, txCtx, cliCtx, msges) + txCtx.Sequence++ + } +} + +func sendTx(address string, txCtx authtxb.TxBuilder, cliCtx context.CLIContext, msgs []sdk.Msg) { + + passphrase := viper.GetString(common.FlagPassphrase) + txBytes, err := txCtx.BuildAndSign(cliCtx.From, passphrase, msgs) + if err != nil { + panic(err) + } + + result, err := cliCtx.BroadcastTxSync(txBytes) + + if err != nil { + println("Error during broadcasting tx. Rebrodcasting ...") + println(err.Error()) + sendTx(address, txCtx, cliCtx, msgs) + } + + if result.Code != 0 { + println("Error during broadcasting tx") + println(string(result.Log)) + time.Sleep(5 * time.Second) + addr, _ := accountFromAddress(address) + seq, _ := cliCtx.GetAccountSequence(addr) + txCtx.Sequence = seq + sendTx(address, txCtx, cliCtx, msgs) + } +} + +func newCLIContext(accName string, chainId string) cli.CLIContext { + + nodeUrl := viper.GetString(common.FlagNode) + node := rpcclient.NewHTTP(nodeUrl, "/websocket") + verifier := &common.NoopVerifier{ChainId: chainId} + + return cli.CLIContext{ + Client: node, + NodeURI: "", + AccountStore: "acc", + From: accName, + Height: 0, + TrustNode: true, + UseLedger: false, + Async: false, + JSON: false, + PrintResponse: true, + Verifier: verifier, + } +} + +func accountFromAddress(from string) (fromAddr sdk.AccAddress, fromName string) { + if from == "" { + return nil, "" + } + + keybase, err := keys.GetKeyBase() + if err != nil { + fmt.Println("no keybase found") + os.Exit(1) + } + + var info cskeys.Info + if addr, err := sdk.AccAddressFromBech32(from); err == nil { + info, err = keybase.GetByAddress(addr) + if err != nil { + fmt.Printf("could not find key %s\n", from) + os.Exit(1) + } + } else { + info, err = keybase.Get(from) + if err != nil { + fmt.Printf("could not find key %s\n", from) + os.Exit(1) + } + } + + fromAddr = info.GetAddress() + fromName = info.GetName() + return +} diff --git a/cosmos/poc/wiki/index.go b/cosmos/poc/wiki/index.go new file mode 100644 index 00000000..5139bf65 --- /dev/null +++ b/cosmos/poc/wiki/index.go @@ -0,0 +1,69 @@ +package main + +import ( + "bufio" + "os" + "regexp" + "strings" + "unicode" +) + +func Index() { + + startArticleId := int64(1) + + sendLinks := InitAddLink() + + f, err := os.OpenFile("enwiki-latest-all-titles", 0, 0) + if err != nil { + panic(err) + } + br := bufio.NewReader(f) + defer f.Close() + + reg, err := regexp.Compile("[^a-zA-Z0-9]+") + + counter := int64(0) + links := make([]Link, 0, 100) + for { + + line, err := br.ReadString('\n') + + if err != nil { + break + } + + if counter < startArticleId { + counter++ + continue + } + + split := strings.Split(strings.TrimSuffix(line, "\n"), "\t") + ids := strings.Split(split[1], "_") + + for _, id := range ids { + + id = reg.ReplaceAllString(id, "") + id = strings.ToLower(id) + + if len(id) == 0 || id == "" { + continue + } + + if len(id) == 1 && unicode.IsSymbol(rune(id[0])) { + continue + } + + page := ".wiki/wiki/" + split[1] + ".html" + links = append(links, Link{from: id, to: page}) + counter++ + + if len(links) == 1000 { + println(split[1]) + println(counter) + sendLinks(links) + links = make([]Link, 0, 100) + } + } + } +} diff --git a/cosmos/poc/wiki/wiki.go b/cosmos/poc/wiki/wiki.go new file mode 100644 index 00000000..b2988e9d --- /dev/null +++ b/cosmos/poc/wiki/wiki.go @@ -0,0 +1,77 @@ +package main + +import ( + "fmt" + "github.com/cybercongress/cyberd/cosmos/poc/claim/common" + "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "os" +) + +type Link struct { + from string + to string +} + +var ( + wiki = &cobra.Command{ + Use: "wiki", + Short: "Use to index wiki", + } +) + +/* +./wiki start +--node=127.0.0.1:34657 --passphrase=1q2w3e4r \ +--address=cosmos1g7e74lxpwlcsza8v0nca2hwahluqcv4r4d3p8p \ +--chain-id=test-chain-gRXWCL +*/ + +func main() { + + wiki.AddCommand(StartCmd()) + homeDir, err := getHomeDir() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + viper.SetDefault("home", homeDir+"/.cyberdwiki") + + if err := wiki.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func StartCmd() *cobra.Command { + + cmd := &cobra.Command{ + Use: "start", + Short: "Start wiki indexing", + RunE: func(cmd *cobra.Command, args []string) error { + Index() + return nil + }, + } + + cmd.Flags().String(common.FlagPassphrase, "", "Passphrase of account to claim from") + cmd.Flags().String(common.FlagChainId, "", "Chain Id") + cmd.Flags().String(common.FlagAddress, "", "ClaimFrom of account to claim from") + cmd.Flags().String(common.FlagNode, "127.0.0.1:26657", "Node url to connect") + + viper.BindPFlag(common.FlagPassphrase, cmd.Flags().Lookup(common.FlagPassphrase)) + viper.BindPFlag(common.FlagChainId, cmd.Flags().Lookup(common.FlagChainId)) + viper.BindPFlag(common.FlagAddress, cmd.Flags().Lookup(common.FlagAddress)) + viper.BindPFlag(common.FlagNode, cmd.Flags().Lookup(common.FlagNode)) + return cmd +} + +func getHomeDir() (string, error) { + home, err := homedir.Dir() + if err != nil { + return "", err + } + return home, nil +} diff --git a/docs/earth.md b/docs/earth.md index bdffdc13..f772a85f 100644 --- a/docs/earth.md +++ b/docs/earth.md @@ -5,7 +5,8 @@ # Copy from earth scp -P 33224 earth@earth.cybernode.ai:/path/file /host/path/file # Copy to earth -scp -P 33324 /host/path/file earth@earth.cybernode.ai:/path/file +scp -P 33324 testnet/genesis.json earth@earth.cybernode.ai:/cyberdata/cyberd/config/genesis.json +scp -P 33324 testnet/config.toml earth@earth.cybernode.ai:/cyberdata/cyberd/config/config.toml ``` ## Reset cyberd