Skip to content

Commit

Permalink
#71 Create basic wiki cyberd indexer
Browse files Browse the repository at this point in the history
  • Loading branch information
hleb-albau committed Oct 25, 2018
1 parent 8c72229 commit ac047cd
Show file tree
Hide file tree
Showing 6 changed files with 284 additions and 2 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,4 @@ node_modules/

initial-rank
initial_guess_impact_results
*enwiki*
2 changes: 1 addition & 1 deletion cosmos/poc/testnet/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ private_peer_ids = ""
##### mempool configuration options #####
[mempool]

recheck = false
recheck = true
recheck_empty = true
broadcast = true
wal_dir = "data/mempool.wal"
Expand Down
134 changes: 134 additions & 0 deletions cosmos/poc/wiki/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package main

import (
"fmt"
"github.com/cosmos/cosmos-sdk/client/context"
cli "github.com/cosmos/cosmos-sdk/client/context"
"github.com/cosmos/cosmos-sdk/client/keys"
cskeys "github.com/cosmos/cosmos-sdk/crypto/keys"
sdk "github.com/cosmos/cosmos-sdk/types"
authcmd "github.com/cosmos/cosmos-sdk/x/auth/client/cli"
authtxb "github.com/cosmos/cosmos-sdk/x/auth/client/txbuilder"
"github.com/cybercongress/cyberd/cosmos/poc/app"
"github.com/cybercongress/cyberd/cosmos/poc/app/storage"
"github.com/cybercongress/cyberd/cosmos/poc/claim/common"
"github.com/spf13/viper"
rpcclient "github.com/tendermint/tendermint/rpc/client"
"os"
"time"
)

func InitAddLink() func([]Link) {

chainId := viper.GetString(common.FlagChainId)
address := viper.GetString(common.FlagAddress)
addr, name := accountFromAddress(address)

cdc := app.MakeCodec()
cliCtx := newCLIContext(name, chainId).
WithCodec(cdc).
WithAccountDecoder(authcmd.GetAccountDecoder(cdc))

accountNumber, _ := cliCtx.GetAccountNumber(addr)
seq, _ := cliCtx.GetAccountSequence(addr)

txCtx := authtxb.TxBuilder{
ChainID: chainId,
Gas: 10000000,
AccountNumber: accountNumber,
Sequence: seq,
Fee: "",
Memo: "",
Codec: cdc,
}

return func(links []Link) {

msges := make([]sdk.Msg, 0, len(links))
for _, link := range links {
msges = append(msges, app.NewMsgLink(addr, storage.Cid(link.from), storage.Cid(link.to)))
}

sendTx(address, txCtx, cliCtx, msges)
txCtx.Sequence++
}
}

func sendTx(address string, txCtx authtxb.TxBuilder, cliCtx context.CLIContext, msgs []sdk.Msg) {

passphrase := viper.GetString(common.FlagPassphrase)
txBytes, err := txCtx.BuildAndSign(cliCtx.From, passphrase, msgs)
if err != nil {
panic(err)
}

result, err := cliCtx.BroadcastTxSync(txBytes)

if err != nil {
println("Error during broadcasting tx. Rebrodcasting ...")
println(err.Error())
sendTx(address, txCtx, cliCtx, msgs)
}

if result.Code != 0 {
println("Error during broadcasting tx")
println(string(result.Log))
time.Sleep(5 * time.Second)
addr, _ := accountFromAddress(address)
seq, _ := cliCtx.GetAccountSequence(addr)
txCtx.Sequence = seq
sendTx(address, txCtx, cliCtx, msgs)
}
}

func newCLIContext(accName string, chainId string) cli.CLIContext {

nodeUrl := viper.GetString(common.FlagNode)
node := rpcclient.NewHTTP(nodeUrl, "/websocket")
verifier := &common.NoopVerifier{ChainId: chainId}

return cli.CLIContext{
Client: node,
NodeURI: "",
AccountStore: "acc",
From: accName,
Height: 0,
TrustNode: true,
UseLedger: false,
Async: false,
JSON: false,
PrintResponse: true,
Verifier: verifier,
}
}

func accountFromAddress(from string) (fromAddr sdk.AccAddress, fromName string) {
if from == "" {
return nil, ""
}

keybase, err := keys.GetKeyBase()
if err != nil {
fmt.Println("no keybase found")
os.Exit(1)
}

var info cskeys.Info
if addr, err := sdk.AccAddressFromBech32(from); err == nil {
info, err = keybase.GetByAddress(addr)
if err != nil {
fmt.Printf("could not find key %s\n", from)
os.Exit(1)
}
} else {
info, err = keybase.Get(from)
if err != nil {
fmt.Printf("could not find key %s\n", from)
os.Exit(1)
}
}

fromAddr = info.GetAddress()
fromName = info.GetName()
return
}
69 changes: 69 additions & 0 deletions cosmos/poc/wiki/index.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package main

import (
"bufio"
"os"
"regexp"
"strings"
"unicode"
)

func Index() {

startArticleId := int64(1)

sendLinks := InitAddLink()

f, err := os.OpenFile("enwiki-latest-all-titles", 0, 0)
if err != nil {
panic(err)
}
br := bufio.NewReader(f)
defer f.Close()

reg, err := regexp.Compile("[^a-zA-Z0-9]+")

counter := int64(0)
links := make([]Link, 0, 100)
for {

line, err := br.ReadString('\n')

if err != nil {
break
}

if counter < startArticleId {
counter++
continue
}

split := strings.Split(strings.TrimSuffix(line, "\n"), "\t")
ids := strings.Split(split[1], "_")

for _, id := range ids {

id = reg.ReplaceAllString(id, "")
id = strings.ToLower(id)

if len(id) == 0 || id == "" {
continue
}

if len(id) == 1 && unicode.IsSymbol(rune(id[0])) {
continue
}

page := ".wiki/wiki/" + split[1] + ".html"
links = append(links, Link{from: id, to: page})
counter++

if len(links) == 1000 {
println(split[1])
println(counter)
sendLinks(links)
links = make([]Link, 0, 100)
}
}
}
}
77 changes: 77 additions & 0 deletions cosmos/poc/wiki/wiki.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package main

import (
"fmt"
"github.com/cybercongress/cyberd/cosmos/poc/claim/common"
"github.com/mitchellh/go-homedir"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"os"
)

type Link struct {
from string
to string
}

var (
wiki = &cobra.Command{
Use: "wiki",
Short: "Use to index wiki",
}
)

/*
./wiki start
--node=127.0.0.1:34657 --passphrase=1q2w3e4r \
--address=cosmos1g7e74lxpwlcsza8v0nca2hwahluqcv4r4d3p8p \
--chain-id=test-chain-gRXWCL
*/

func main() {

wiki.AddCommand(StartCmd())
homeDir, err := getHomeDir()
if err != nil {
fmt.Println(err)
os.Exit(1)
}

viper.SetDefault("home", homeDir+"/.cyberdwiki")

if err := wiki.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}

func StartCmd() *cobra.Command {

cmd := &cobra.Command{
Use: "start",
Short: "Start wiki indexing",
RunE: func(cmd *cobra.Command, args []string) error {
Index()
return nil
},
}

cmd.Flags().String(common.FlagPassphrase, "", "Passphrase of account to claim from")
cmd.Flags().String(common.FlagChainId, "", "Chain Id")
cmd.Flags().String(common.FlagAddress, "", "ClaimFrom of account to claim from")
cmd.Flags().String(common.FlagNode, "127.0.0.1:26657", "Node url to connect")

viper.BindPFlag(common.FlagPassphrase, cmd.Flags().Lookup(common.FlagPassphrase))
viper.BindPFlag(common.FlagChainId, cmd.Flags().Lookup(common.FlagChainId))
viper.BindPFlag(common.FlagAddress, cmd.Flags().Lookup(common.FlagAddress))
viper.BindPFlag(common.FlagNode, cmd.Flags().Lookup(common.FlagNode))
return cmd
}

func getHomeDir() (string, error) {
home, err := homedir.Dir()
if err != nil {
return "", err
}
return home, nil
}
3 changes: 2 additions & 1 deletion docs/earth.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
# Copy from earth
scp -P 33224 earth@earth.cybernode.ai:/path/file /host/path/file
# Copy to earth
scp -P 33324 /host/path/file earth@earth.cybernode.ai:/path/file
scp -P 33324 testnet/genesis.json earth@earth.cybernode.ai:/cyberdata/cyberd/config/genesis.json
scp -P 33324 testnet/config.toml earth@earth.cybernode.ai:/cyberdata/cyberd/config/config.toml
```

## Reset cyberd
Expand Down

0 comments on commit ac047cd

Please sign in to comment.