-
Notifications
You must be signed in to change notification settings - Fork 1.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
cmd/dbcmd: add inspect trie tool (#2082)
- Loading branch information
Showing
2 changed files
with
370 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,271 @@ | ||
| package trie | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "errors" | ||
| "fmt" | ||
| "math/big" | ||
| "os" | ||
| "runtime" | ||
| "sort" | ||
| "strconv" | ||
| "sync" | ||
| "sync/atomic" | ||
|
|
||
| "github.com/ethereum/go-ethereum/common" | ||
| "github.com/ethereum/go-ethereum/log" | ||
| "github.com/ethereum/go-ethereum/rlp" | ||
| "github.com/olekukonko/tablewriter" | ||
| ) | ||
|
|
||
| type Account struct { | ||
| Nonce uint64 | ||
| Balance *big.Int | ||
| Root common.Hash // merkle root of the storage trie | ||
| CodeHash []byte | ||
| } | ||
|
|
||
| type Inspector struct { | ||
| trie *Trie // traverse trie | ||
| db *Database | ||
| stateRootHash common.Hash | ||
| blocknum uint64 | ||
| root node // root of triedb | ||
| totalNum uint64 | ||
| concurrentQueue chan struct{} | ||
| wg sync.WaitGroup | ||
| statLock sync.RWMutex | ||
| result map[string]*TrieTreeStat | ||
| } | ||
|
|
||
| type TrieTreeStat struct { | ||
| is_account_trie bool | ||
| theNodeStatByLevel [15]NodeStat | ||
| totalNodeStat NodeStat | ||
| } | ||
|
|
||
| type NodeStat struct { | ||
| ShortNodeCnt uint64 | ||
| FullNodeCnt uint64 | ||
| ValueNodeCnt uint64 | ||
| } | ||
|
|
||
| func (trieStat *TrieTreeStat) AtomicAdd(theNode node, height uint32) { | ||
| switch (theNode).(type) { | ||
| case *shortNode: | ||
| atomic.AddUint64(&trieStat.totalNodeStat.ShortNodeCnt, 1) | ||
| atomic.AddUint64(&(trieStat.theNodeStatByLevel[height].ShortNodeCnt), 1) | ||
| case *fullNode: | ||
| atomic.AddUint64(&trieStat.totalNodeStat.FullNodeCnt, 1) | ||
| atomic.AddUint64(&trieStat.theNodeStatByLevel[height].FullNodeCnt, 1) | ||
| case valueNode: | ||
| atomic.AddUint64(&trieStat.totalNodeStat.ValueNodeCnt, 1) | ||
| atomic.AddUint64(&((trieStat.theNodeStatByLevel[height]).ValueNodeCnt), 1) | ||
| default: | ||
| panic(errors.New("Invalid node type to statistics")) | ||
| } | ||
| } | ||
|
|
||
| func (trieStat *TrieTreeStat) Display(ownerAddress string, treeType string) { | ||
| table := tablewriter.NewWriter(os.Stdout) | ||
| table.SetHeader([]string{"-", "Level", "ShortNodeCnt", "FullNodeCnt", "ValueNodeCnt"}) | ||
| if ownerAddress == "" { | ||
| table.SetCaption(true, fmt.Sprintf("%v", treeType)) | ||
| } else { | ||
| table.SetCaption(true, fmt.Sprintf("%v-%v", treeType, ownerAddress)) | ||
| } | ||
| table.SetAlignment(1) | ||
| for i := 0; i < len(trieStat.theNodeStatByLevel); i++ { | ||
| nodeStat := trieStat.theNodeStatByLevel[i] | ||
| if nodeStat.FullNodeCnt == 0 && nodeStat.ShortNodeCnt == 0 && nodeStat.ValueNodeCnt == 0 { | ||
| break | ||
| } | ||
| table.AppendBulk([][]string{ | ||
| {"-", strconv.Itoa(i), nodeStat.ShortNodeCount(), nodeStat.FullNodeCount(), nodeStat.ValueNodeCount()}, | ||
| }) | ||
| } | ||
| table.AppendBulk([][]string{ | ||
| {"Total", "-", trieStat.totalNodeStat.ShortNodeCount(), trieStat.totalNodeStat.FullNodeCount(), trieStat.totalNodeStat.ValueNodeCount()}, | ||
| }) | ||
| table.Render() | ||
| } | ||
|
|
||
| func Uint64ToString(cnt uint64) string { | ||
| return fmt.Sprintf("%v", cnt) | ||
| } | ||
|
|
||
| func (nodeStat *NodeStat) ShortNodeCount() string { | ||
| return Uint64ToString(nodeStat.ShortNodeCnt) | ||
| } | ||
|
|
||
| func (nodeStat *NodeStat) FullNodeCount() string { | ||
| return Uint64ToString(nodeStat.FullNodeCnt) | ||
| } | ||
| func (nodeStat *NodeStat) ValueNodeCount() string { | ||
| return Uint64ToString(nodeStat.ValueNodeCnt) | ||
| } | ||
|
|
||
| // NewInspector return a inspector obj | ||
| func NewInspector(tr *Trie, db *Database, stateRootHash common.Hash, blocknum uint64, jobnum uint64) (*Inspector, error) { | ||
| if tr == nil { | ||
| return nil, errors.New("trie is nil") | ||
| } | ||
|
|
||
| if tr.root == nil { | ||
| return nil, errors.New("trie root is nil") | ||
| } | ||
|
|
||
| ins := &Inspector{ | ||
| trie: tr, | ||
| db: db, | ||
| stateRootHash: stateRootHash, | ||
| blocknum: blocknum, | ||
| root: tr.root, | ||
| result: make(map[string]*TrieTreeStat), | ||
| totalNum: (uint64)(0), | ||
| concurrentQueue: make(chan struct{}, jobnum), | ||
| wg: sync.WaitGroup{}, | ||
| } | ||
|
|
||
| return ins, nil | ||
| } | ||
|
|
||
| // Run statistics, external call | ||
| func (inspect *Inspector) Run() { | ||
| accountTrieStat := new(TrieTreeStat) | ||
| path := make([]byte, 0) | ||
|
|
||
| if _, ok := inspect.result[""]; !ok { | ||
| inspect.result[""] = accountTrieStat | ||
| } | ||
| log.Info("Find Account Trie Tree", "rootHash: ", inspect.trie.Hash().String(), "BlockNum: ", inspect.blocknum) | ||
| inspect.ConcurrentTraversal(inspect.trie, accountTrieStat, inspect.root, 0, path) | ||
| inspect.wg.Wait() | ||
| } | ||
|
|
||
| func (inspect *Inspector) SubConcurrentTraversal(theTrie *Trie, theTrieTreeStat *TrieTreeStat, theNode node, height uint32, path []byte) { | ||
| inspect.concurrentQueue <- struct{}{} | ||
| inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, theNode, height, path) | ||
| <-inspect.concurrentQueue | ||
| inspect.wg.Done() | ||
| } | ||
|
|
||
| func (inspect *Inspector) ConcurrentTraversal(theTrie *Trie, theTrieTreeStat *TrieTreeStat, theNode node, height uint32, path []byte) { | ||
| // print process progress | ||
| total_num := atomic.AddUint64(&inspect.totalNum, 1) | ||
| if total_num%100000 == 0 { | ||
| fmt.Printf("Complete progress: %v, go routines Num: %v, inspect concurrentQueue: %v\n", total_num, runtime.NumGoroutine(), len(inspect.concurrentQueue)) | ||
| } | ||
|
|
||
| // nil node | ||
| if theNode == nil { | ||
| return | ||
| } | ||
|
|
||
| switch current := (theNode).(type) { | ||
| case *shortNode: | ||
| inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, current.Val, height, append(path, current.Key...)) | ||
| case *fullNode: | ||
| for idx, child := range current.Children { | ||
| if child == nil { | ||
| continue | ||
| } | ||
| childPath := append(path, byte(idx)) | ||
| if len(inspect.concurrentQueue)*2 < cap(inspect.concurrentQueue) { | ||
| inspect.wg.Add(1) | ||
| dst := make([]byte, len(childPath)) | ||
| copy(dst, childPath) | ||
| go inspect.SubConcurrentTraversal(theTrie, theTrieTreeStat, child, height+1, dst) | ||
| } else { | ||
| inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, child, height+1, childPath) | ||
| } | ||
| } | ||
| case hashNode: | ||
| n, err := theTrie.resloveWithoutTrack(current, path) | ||
| if err != nil { | ||
| fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, theTrie.Hash().String(), height+1, path) | ||
| return | ||
| } | ||
| inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, n, height, path) | ||
| return | ||
| case valueNode: | ||
| if !hasTerm(path) { | ||
| break | ||
| } | ||
| var account Account | ||
| if err := rlp.Decode(bytes.NewReader(current), &account); err != nil { | ||
| break | ||
| } | ||
| if account.Root == (common.Hash{}) { | ||
| break | ||
| } | ||
| ownerAddress := common.BytesToHash(hexToCompact(path)) | ||
| contractTrie, err := New(StorageTrieID(inspect.stateRootHash, ownerAddress, account.Root), inspect.db) | ||
| if err != nil { | ||
| // fmt.Printf("New contract trie node: %v, error: %v, Height: %v, Path: %v\n", theNode, err, height, path) | ||
| break | ||
| } | ||
| trieStat := new(TrieTreeStat) | ||
| trieStat.is_account_trie = false | ||
|
|
||
| inspect.statLock.Lock() | ||
| if _, ok := inspect.result[ownerAddress.String()]; !ok { | ||
| inspect.result[ownerAddress.String()] = trieStat | ||
| } | ||
| inspect.statLock.Unlock() | ||
|
|
||
| contractPath := make([]byte, 0) | ||
| // log.Info("Find Contract Trie Tree, rootHash: ", contractTrie.Hash().String(), "") | ||
| inspect.wg.Add(1) | ||
| go inspect.SubConcurrentTraversal(contractTrie, trieStat, contractTrie.root, 0, contractPath) | ||
| default: | ||
| panic(errors.New("Invalid node type to traverse.")) | ||
| } | ||
| theTrieTreeStat.AtomicAdd(theNode, height) | ||
| } | ||
|
|
||
| func (inspect *Inspector) DisplayResult() { | ||
| // display root hash | ||
| if _, ok := inspect.result[""]; !ok { | ||
| log.Info("Display result error", "missing account trie") | ||
| return | ||
| } | ||
| inspect.result[""].Display("", "AccountTrie") | ||
|
|
||
| type SortedTrie struct { | ||
| totalNum uint64 | ||
| ownerAddress string | ||
| } | ||
| // display contract trie | ||
| var sortedTriesByNums []SortedTrie | ||
| var totalContactsNodeStat NodeStat | ||
| var contractTrieCnt uint64 = 0 | ||
|
|
||
| for ownerAddress, stat := range inspect.result { | ||
| if ownerAddress == "" { | ||
| continue | ||
| } | ||
| contractTrieCnt++ | ||
| totalContactsNodeStat.ShortNodeCnt += stat.totalNodeStat.ShortNodeCnt | ||
| totalContactsNodeStat.FullNodeCnt += stat.totalNodeStat.FullNodeCnt | ||
| totalContactsNodeStat.ValueNodeCnt += stat.totalNodeStat.ValueNodeCnt | ||
| totalNodeCnt := stat.totalNodeStat.ShortNodeCnt + stat.totalNodeStat.ValueNodeCnt + stat.totalNodeStat.FullNodeCnt | ||
| sortedTriesByNums = append(sortedTriesByNums, SortedTrie{totalNum: totalNodeCnt, ownerAddress: ownerAddress}) | ||
| } | ||
| sort.Slice(sortedTriesByNums, func(i, j int) bool { | ||
| return sortedTriesByNums[i].totalNum > sortedTriesByNums[j].totalNum | ||
| }) | ||
| // only display top 5 | ||
| for i, t := range sortedTriesByNums { | ||
| if i > 5 { | ||
| break | ||
| } | ||
| if stat, ok := inspect.result[t.ownerAddress]; !ok { | ||
| log.Error("Storage trie stat not found", "ownerAddress", t.ownerAddress) | ||
| } else { | ||
| stat.Display(t.ownerAddress, "ContractTrie") | ||
| } | ||
| } | ||
| fmt.Printf("Contract Trie, total trie num: %v, ShortNodeCnt: %v, FullNodeCnt: %v, ValueNodeCnt: %v\n", | ||
| contractTrieCnt, totalContactsNodeStat.ShortNodeCnt, totalContactsNodeStat.FullNodeCnt, totalContactsNodeStat.ValueNodeCnt) | ||
| } |