From 0ca67fc99b86df24524a72ba5ebf43b22e264e2d Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 30 Aug 2023 13:30:04 +0200 Subject: [PATCH] added ability to search for execution layer block hashes / numbers --- db/db.go | 8 +- dbtypes/dbtypes.go | 6 -- dbtypes/search.go | 8 ++ handlers/search.go | 198 +++++++++++++++++++++++++++------- indexer/cache.go | 18 +++- indexer/cache_block.go | 16 +++ indexer/indexer.go | 46 ++++++++ services/beaconservice.go | 30 +----- static/js/explorer.js | 28 +++++ templates/_layout/header.html | 2 +- types/models/search.go | 9 ++ 11 files changed, 289 insertions(+), 80 deletions(-) diff --git a/db/db.go b/db/db.go index 6e169862..bde216f0 100644 --- a/db/db.go +++ b/db/db.go @@ -641,15 +641,15 @@ func InsertUnfinalizedBlock(block *dbtypes.UnfinalizedBlock, tx *sqlx.Tx) error return nil } -func GetUnfinalizedBlockHeader() []*dbtypes.UnfinalizedBlockHeader { - blockRefs := []*dbtypes.UnfinalizedBlockHeader{} +func GetUnfinalizedBlocks() []*dbtypes.UnfinalizedBlock { + blockRefs := []*dbtypes.UnfinalizedBlock{} err := ReaderDb.Select(&blockRefs, ` SELECT - root, slot, header + root, slot, header, block FROM unfinalized_blocks `) if err != nil { - logger.Errorf("Error while fetching unfinalized block refs: %v", err) + logger.Errorf("Error while fetching unfinalized blocks: %v", err) return nil } return blockRefs diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index 8ee7f1a7..7d1a967c 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -73,12 +73,6 @@ type UnfinalizedBlock struct { Block string `db:"block"` } -type UnfinalizedBlockHeader struct { - Root []byte `db:"root"` - Slot uint64 `db:"slot"` - Header string `db:"header"` -} - type UnfinalizedEpochDuty struct { Epoch uint64 `db:"epoch"` DependentRoot []byte `db:"dependent_root"` diff --git a/dbtypes/search.go b/dbtypes/search.go index a7f6dbff..5a603938 100644 --- a/dbtypes/search.go +++ b/dbtypes/search.go @@ -20,6 +20,14 @@ type SearchAheadSlotsResult []struct { Orphaned bool `db:"orphaned"` } +type SearchAheadExecBlocksResult []struct { + Slot uint64 `db:"slot"` + Root []byte `db:"root"` + ExecHash []byte `db:"eth_block_hash"` + ExecNumber uint64 `db:"eth_block_number"` + Orphaned bool `db:"orphaned"` +} + type SearchAheadGraffitiResult []struct { Graffiti string `db:"graffiti"` Count uint64 `db:"count"` diff --git a/handlers/search.go b/handlers/search.go index 54cc6bd9..abb1ec9e 100644 --- a/handlers/search.go +++ b/handlers/search.go @@ -104,12 +104,15 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { searchType := vars["type"] urlArgs := r.URL.Query() search := urlArgs.Get("q") + search = strings.Trim(search, " \t") search = strings.Replace(search, "0x", "", -1) search = strings.Replace(search, "0X", "", -1) var err error logger := logrus.WithField("searchType", searchType) var result interface{} + indexer := services.GlobalBeaconService.GetIndexer() + switch searchType { case "epochs": dbres := &dbtypes.SearchAheadEpochsResult{} @@ -124,61 +127,83 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { result = model } case "slots": - if len(search) <= 1 { + if len(search) == 0 { + break + } + if !searchLikeRE.MatchString(search) { break } - if searchLikeRE.MatchString(search) { - if len(search) == 64 { - blockHash, err := hex.DecodeString(search) + if len(search) == 64 { + blockHash, err := hex.DecodeString(search) + if err != nil { + logger.Errorf("error parsing blockHash to int: %v", err) + http.Error(w, "Internal server error", http.StatusServiceUnavailable) + return + } + + cachedBlock := indexer.GetCachedBlock(blockHash) + if cachedBlock == nil { + cachedBlock = indexer.GetCachedBlockByStateroot(blockHash) + } + if !cachedBlock.IsReady() { + cachedBlock = nil + } + if cachedBlock != nil { + header := cachedBlock.GetHeader() + result = &[]models.SearchAheadSlotsResult{ + { + Slot: fmt.Sprintf("%v", uint64(header.Message.Slot)), + Root: cachedBlock.Root, + Orphaned: !cachedBlock.IsCanonical(indexer, nil), + }, + } + } else { + dbres := &dbtypes.SearchAheadSlotsResult{} + err = db.ReaderDb.Select(dbres, ` + SELECT slot, root, orphaned + FROM blocks + WHERE root = $1 OR + state_root = $1 + ORDER BY slot LIMIT 1`, blockHash) if err != nil { - logger.Errorf("error parsing blockHash to int: %v", err) + logger.Errorf("error reading block root: %v", err) http.Error(w, "Internal server error", http.StatusServiceUnavailable) return } - - cachedBlock := services.GlobalBeaconService.GetCachedBlockByBlockroot(blockHash) - if cachedBlock == nil { - cachedBlock = services.GlobalBeaconService.GetCachedBlockByStateroot(blockHash) - } - if cachedBlock != nil { + if len(*dbres) > 0 { result = &[]models.SearchAheadSlotsResult{ { - Slot: fmt.Sprintf("%v", uint64(cachedBlock.Header.Message.Slot)), - Root: cachedBlock.Root, - Orphaned: cachedBlock.Orphaned, + Slot: fmt.Sprintf("%v", (*dbres)[0].Slot), + Root: (*dbres)[0].Root, + Orphaned: (*dbres)[0].Orphaned, }, } - } else { - dbres := &dbtypes.SearchAheadSlotsResult{} - err = db.ReaderDb.Select(dbres, ` - SELECT slot, root, orphaned - FROM blocks - WHERE root = $1 OR - state_root = $1 - ORDER BY slot LIMIT 1`, blockHash) - if err != nil { - logger.Errorf("error reading block root: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) - return - } - if len(*dbres) > 0 { - result = &[]models.SearchAheadSlotsResult{ - { - Slot: fmt.Sprintf("%v", (*dbres)[0].Slot), - Root: (*dbres)[0].Root, - Orphaned: (*dbres)[0].Orphaned, - }, - } - } + } + } + } else if blockNumber, convertErr := strconv.ParseUint(search, 10, 32); convertErr == nil { + cachedBlocks := indexer.GetCachedBlocks(blockNumber) + if len(cachedBlocks) > 0 { + res := make([]*models.SearchAheadSlotsResult, 0) + for _, cachedBlock := range cachedBlocks { + if !cachedBlock.IsReady() { + continue + } + header := cachedBlock.GetHeader() + res = append(res, &models.SearchAheadSlotsResult{ + Slot: fmt.Sprintf("%v", uint64(header.Message.Slot)), + Root: cachedBlock.Root, + Orphaned: !cachedBlock.IsCanonical(indexer, nil), + }) } - } else if _, convertErr := strconv.ParseInt(search, 10, 32); convertErr == nil { + result = res + } else { dbres := &dbtypes.SearchAheadSlotsResult{} err = db.ReaderDb.Select(dbres, ` SELECT slot, root, orphaned FROM blocks WHERE slot = $1 - ORDER BY slot LIMIT 10`, search) + ORDER BY slot LIMIT 10`, blockNumber) if err == nil { model := make([]models.SearchAheadSlotsResult, len(*dbres)) for idx, entry := range *dbres { @@ -192,6 +217,103 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { } } } + case "execblocks": + if len(search) == 0 { + break + } + if !searchLikeRE.MatchString(search) { + break + } + if len(search) == 64 { + blockHash, err := hex.DecodeString(search) + if err != nil { + logger.Errorf("error parsing blockHash to int: %v", err) + http.Error(w, "Internal server error", http.StatusServiceUnavailable) + return + } + + cachedBlocks := indexer.GetCachedBlocksByExecutionBlockHash(blockHash) + if len(cachedBlocks) > 0 { + res := make([]*models.SearchAheadExecBlocksResult, 0) + for idx, cachedBlock := range cachedBlocks { + if !cachedBlock.IsReady() { + continue + } + header := cachedBlock.GetHeader() + res[idx] = &models.SearchAheadExecBlocksResult{ + Slot: fmt.Sprintf("%v", uint64(header.Message.Slot)), + Root: cachedBlock.Root, + ExecHash: cachedBlock.Refs.ExecutionHash, + ExecNumber: cachedBlock.Refs.ExecutionNumber, + Orphaned: !cachedBlock.IsCanonical(indexer, nil), + } + } + result = res + } else { + dbres := &dbtypes.SearchAheadExecBlocksResult{} + err = db.ReaderDb.Select(dbres, ` + SELECT slot, root, eth_block_hash, eth_block_number, orphaned + FROM blocks + WHERE eth_block_hash = $1 + ORDER BY slot LIMIT 10`, blockHash) + if err != nil { + logger.Errorf("error reading block: %v", err) + http.Error(w, "Internal server error", http.StatusServiceUnavailable) + return + } + if len(*dbres) > 0 { + result = &[]models.SearchAheadExecBlocksResult{ + { + Slot: fmt.Sprintf("%v", (*dbres)[0].Slot), + Root: (*dbres)[0].Root, + ExecHash: (*dbres)[0].ExecHash, + ExecNumber: (*dbres)[0].ExecNumber, + Orphaned: (*dbres)[0].Orphaned, + }, + } + } + + } + } else if blockNumber, convertErr := strconv.ParseUint(search, 10, 32); convertErr == nil { + cachedBlocks := indexer.GetCachedBlocksByExecutionBlockNumber(blockNumber) + if len(cachedBlocks) > 0 { + res := make([]*models.SearchAheadExecBlocksResult, 0) + for _, cachedBlock := range cachedBlocks { + if !cachedBlock.IsReady() { + continue + } + header := cachedBlock.GetHeader() + res = append(res, &models.SearchAheadExecBlocksResult{ + Slot: fmt.Sprintf("%v", uint64(header.Message.Slot)), + Root: cachedBlock.Root, + ExecHash: cachedBlock.Refs.ExecutionHash, + ExecNumber: cachedBlock.Refs.ExecutionNumber, + Orphaned: !cachedBlock.IsCanonical(indexer, nil), + }) + } + result = res + } else { + dbres := &dbtypes.SearchAheadExecBlocksResult{} + err = db.ReaderDb.Select(dbres, ` + SELECT slot, root, eth_block_hash, eth_block_number, orphaned + FROM blocks + WHERE eth_block_number = $1 + ORDER BY slot LIMIT 10`, blockNumber) + if err == nil { + model := make([]models.SearchAheadExecBlocksResult, len(*dbres)) + for idx, entry := range *dbres { + model[idx] = models.SearchAheadExecBlocksResult{ + Slot: fmt.Sprintf("%v", entry.Slot), + Root: entry.Root, + ExecHash: entry.ExecHash, + ExecNumber: entry.ExecNumber, + Orphaned: entry.Orphaned, + } + } + result = model + } + } + } case "graffiti": graffiti := &dbtypes.SearchAheadGraffitiResult{} err = db.ReaderDb.Select(graffiti, db.EngineQuery(map[dbtypes.DBEngineType]string{ diff --git a/indexer/cache.go b/indexer/cache.go index ef8307cd..c56971b0 100644 --- a/indexer/cache.go +++ b/indexer/cache.go @@ -92,19 +92,27 @@ func (cache *indexerCache) setLastValidators(epoch uint64, validators *rpctypes. } func (cache *indexerCache) loadStoredUnfinalizedCache() error { - blockHeaders := db.GetUnfinalizedBlockHeader() - for _, blockHeader := range blockHeaders { + blocks := db.GetUnfinalizedBlocks() + for _, block := range blocks { var header rpctypes.SignedBeaconBlockHeader - err := json.Unmarshal([]byte(blockHeader.Header), &header) + err := json.Unmarshal([]byte(block.Header), &header) if err != nil { logger.Warnf("Error parsing unfinalized block header from db: %v", err) continue } - logger.Debugf("Restored unfinalized block header from db: %v", blockHeader.Slot) - cachedBlock, _ := cache.createOrGetCachedBlock(blockHeader.Root, blockHeader.Slot) + var body rpctypes.SignedBeaconBlock + err = json.Unmarshal([]byte(block.Block), &body) + if err != nil { + logger.Warnf("Error parsing unfinalized block body from db: %v", err) + continue + } + logger.Debugf("Restored unfinalized block header from db: %v", block.Slot) + cachedBlock, _ := cache.createOrGetCachedBlock(block.Root, block.Slot) cachedBlock.mutex.Lock() cachedBlock.header = &header + cachedBlock.block = &body cachedBlock.isInDb = true + cachedBlock.parseBlockRefs() cachedBlock.mutex.Unlock() } epochDuties := db.GetUnfinalizedEpochDutyRefs() diff --git a/indexer/cache_block.go b/indexer/cache_block.go index e50e75f2..196fc40e 100644 --- a/indexer/cache_block.go +++ b/indexer/cache_block.go @@ -17,6 +17,10 @@ type CacheBlock struct { isInDb bool header *rpctypes.SignedBeaconBlockHeader block *rpctypes.SignedBeaconBlock + Refs struct { + ExecutionHash []byte + ExecutionNumber uint64 + } dbBlockMutex sync.Mutex dbBlockCache *dbtypes.Block @@ -100,6 +104,17 @@ func (block *CacheBlock) buildOrphanedBlock() *dbtypes.OrphanedBlock { } } +func (block *CacheBlock) parseBlockRefs() { + if block.block == nil { + return + } + execPayload := block.block.Message.Body.ExecutionPayload + if execPayload != nil { + block.Refs.ExecutionHash = execPayload.BlockHash + block.Refs.ExecutionNumber = uint64(execPayload.BlockNumber) + } +} + func (block *CacheBlock) GetParentRoot() []byte { block.mutex.RLock() defer block.mutex.RUnlock() @@ -134,6 +149,7 @@ func (block *CacheBlock) GetBlockBody() *rpctypes.SignedBeaconBlock { return nil } block.block = &blockBody + block.parseBlockRefs() return block.block } diff --git a/indexer/indexer.go b/indexer/indexer.go index 6e7940d5..ab0e8f21 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -295,6 +295,52 @@ func (indexer *Indexer) GetCachedBlockByStateroot(stateroot []byte) *CacheBlock return nil } +func (indexer *Indexer) GetCachedBlocksByExecutionBlockHash(hash []byte) []*CacheBlock { + indexer.indexerCache.cacheMutex.RLock() + defer indexer.indexerCache.cacheMutex.RUnlock() + + resBlocks := make([]*CacheBlock, 0) + var lowestSlotIdx int64 + if indexer.indexerCache.finalizedEpoch >= 0 { + lowestSlotIdx = (indexer.indexerCache.finalizedEpoch + 1) * int64(utils.Config.Chain.Config.SlotsPerEpoch) + } else { + lowestSlotIdx = 0 + } + for slotIdx := int64(indexer.indexerCache.highestSlot); slotIdx >= lowestSlotIdx; slotIdx-- { + slot := uint64(slotIdx) + blocks := indexer.indexerCache.slotMap[slot] + for _, block := range blocks { + if block.IsReady() && bytes.Equal(block.Refs.ExecutionHash, hash) { + resBlocks = append(resBlocks, block) + } + } + } + return resBlocks +} + +func (indexer *Indexer) GetCachedBlocksByExecutionBlockNumber(number uint64) []*CacheBlock { + indexer.indexerCache.cacheMutex.RLock() + defer indexer.indexerCache.cacheMutex.RUnlock() + + resBlocks := make([]*CacheBlock, 0) + var lowestSlotIdx int64 + if indexer.indexerCache.finalizedEpoch >= 0 { + lowestSlotIdx = (indexer.indexerCache.finalizedEpoch + 1) * int64(utils.Config.Chain.Config.SlotsPerEpoch) + } else { + lowestSlotIdx = 0 + } + for slotIdx := int64(indexer.indexerCache.highestSlot); slotIdx >= lowestSlotIdx; slotIdx-- { + slot := uint64(slotIdx) + blocks := indexer.indexerCache.slotMap[slot] + for _, block := range blocks { + if block.IsReady() && block.Refs.ExecutionNumber == number { + resBlocks = append(resBlocks, block) + } + } + } + return resBlocks +} + func (indexer *Indexer) GetCachedBlocksByProposer(proposer uint64) []*CacheBlock { indexer.indexerCache.cacheMutex.RLock() defer indexer.indexerCache.cacheMutex.RUnlock() diff --git a/services/beaconservice.go b/services/beaconservice.go index 0a9de4a4..47befcf8 100644 --- a/services/beaconservice.go +++ b/services/beaconservice.go @@ -64,6 +64,10 @@ func StartBeaconService() error { return nil } +func (bs *BeaconService) GetIndexer() *indexer.Indexer { + return bs.indexer +} + func (bs *BeaconService) GetClients() []*indexer.IndexerClient { return bs.indexer.GetClients() } @@ -265,32 +269,6 @@ func (bs *BeaconService) GetOrphanedBlock(blockroot []byte) *rpctypes.CombinedBl } } -func (bs *BeaconService) GetCachedBlockByBlockroot(blockroot []byte) *rpctypes.CombinedBlockResponse { - cachedBlock := bs.indexer.GetCachedBlock(blockroot) - if cachedBlock == nil { - return nil - } - return &rpctypes.CombinedBlockResponse{ - Root: cachedBlock.Root, - Header: cachedBlock.GetHeader(), - Block: cachedBlock.GetBlockBody(), - Orphaned: !cachedBlock.IsCanonical(bs.indexer, nil), - } -} - -func (bs *BeaconService) GetCachedBlockByStateroot(stateroot []byte) *rpctypes.CombinedBlockResponse { - cachedBlock := bs.indexer.GetCachedBlockByStateroot(stateroot) - if cachedBlock == nil { - return nil - } - return &rpctypes.CombinedBlockResponse{ - Root: cachedBlock.Root, - Header: cachedBlock.GetHeader(), - Block: cachedBlock.GetBlockBody(), - Orphaned: !cachedBlock.IsCanonical(bs.indexer, nil), - } -} - func (bs *BeaconService) GetEpochAssignments(epoch uint64) (*rpctypes.EpochAssignments, error) { finalizedEpoch, _ := bs.indexer.GetFinalizedEpoch() diff --git a/static/js/explorer.js b/static/js/explorer.js index f9ff6d74..aba260f5 100644 --- a/static/js/explorer.js +++ b/static/js/explorer.js @@ -93,6 +93,18 @@ maxPendingRequests: requestNum, }, }); + var bhExecBlocks = new Bloodhound({ + datumTokenizer: Bloodhound.tokenizers.whitespace, + queryTokenizer: Bloodhound.tokenizers.whitespace, + identify: function (obj) { + return obj.slot + }, + remote: { + url: "/search/execblocks?q=", + prepare: prepareQueryFn, + maxPendingRequests: requestNum, + }, + }); var bhEpochs = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -142,6 +154,22 @@ }, }, }, + { + limit: 5, + name: "execblocks", + source: bhExecBlocks, + display: "root", + templates: { + header: '

Slots (by execution block):

', + suggestion: function (data) { + var status = ""; + if (data.orphaned) { + status = `Orphaned`; + } + return `
${data.slot}:Block ${data.exec_number} / ${data.exec_hash}${status}
`; + }, + }, + }, { limit: 5, name: "epoch", diff --git a/templates/_layout/header.html b/templates/_layout/header.html index a3a963cf..a349b806 100644 --- a/templates/_layout/header.html +++ b/templates/_layout/header.html @@ -49,7 +49,7 @@ diff --git a/types/models/search.go b/types/models/search.go index 049f75f4..b9f4e42d 100644 --- a/types/models/search.go +++ b/types/models/search.go @@ -26,6 +26,15 @@ type SearchAheadSlotsResult struct { Orphaned bool `json:"orphaned,omitempty"` } +// SearchAheadExecBlocksResult is a struct to hold the search ahead execution blocks results +type SearchAheadExecBlocksResult struct { + Slot string `json:"slot,omitempty"` + Root rpctypes.BytesHexStr `json:"root,omitempty"` + ExecHash rpctypes.BytesHexStr `json:"exec_hash,omitempty"` + ExecNumber uint64 `json:"exec_number,omitempty"` + Orphaned bool `json:"orphaned,omitempty"` +} + // SearchAheadGraffitiResult is a struct to hold the search ahead blocks results with a given graffiti type SearchAheadGraffitiResult struct { Graffiti string `json:"graffiti,omitempty"`