From c5c0b14293df8fac8114dfad64e6266b8feeb9d7 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Thu, 16 May 2024 18:15:05 +0200 Subject: [PATCH 1/3] Fixed CPU usage issues caused by inefficiencies in HeadTracker --- .changeset/early-shoes-sit.md | 5 ++ .../evm/headtracker/head_tracker_test.go | 39 +++++++++++++- core/chains/evm/headtracker/heads.go | 54 ++++++++----------- 3 files changed, 65 insertions(+), 33 deletions(-) create mode 100644 .changeset/early-shoes-sit.md diff --git a/.changeset/early-shoes-sit.md b/.changeset/early-shoes-sit.md new file mode 100644 index 00000000000..18ca4fabf33 --- /dev/null +++ b/.changeset/early-shoes-sit.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +Fixed CPU usage issues caused by inefficiencies in HeadTracker #bugfix diff --git a/core/chains/evm/headtracker/head_tracker_test.go b/core/chains/evm/headtracker/head_tracker_test.go index b8bdb1f5703..d2ec84bf48a 100644 --- a/core/chains/evm/headtracker/head_tracker_test.go +++ b/core/chains/evm/headtracker/head_tracker_test.go @@ -31,6 +31,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox/mailboxtest" htmocks "github.com/smartcontractkit/chainlink/v2/common/headtracker/mocks" + evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" evmclimocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client/mocks" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types" @@ -983,7 +984,43 @@ func TestHeadTracker_Backfill(t *testing.T) { }) } -func createHeadTracker(t *testing.T, ethClient *evmclimocks.Client, config headtracker.Config, htConfig headtracker.HeadTrackerConfig, orm headtracker.ORM) *headTrackerUniverse { +func BenchmarkHeadTracker_Backfill(b *testing.B) { + cfg := configtest.NewGeneralConfig(b, nil) + + evmcfg := evmtest.NewChainScopedConfig(b, cfg) + db := pgtest.NewSqlxDB(b) + chainID := big.NewInt(evmclient.NullClientChainID) + orm := headtracker.NewORM(*chainID, db) + ethClient := evmclimocks.NewClient(b) + ethClient.On("ConfiguredChainID").Return(chainID) + ht := createHeadTracker(b, ethClient, evmcfg.EVM(), evmcfg.EVM().HeadTracker(), orm) + ctx := tests.Context(b) + makeHash := func(n int64) gethCommon.Hash { + return gethCommon.BigToHash(big.NewInt(n)) + } + const finalityDepth = 12000 // observed value on Arbitrum + makeBlock := func(n int64) *evmtypes.Head { + return &evmtypes.Head{Number: n, Hash: makeHash(n), ParentHash: makeHash(n - 1)} + } + latest := makeBlock(finalityDepth) + finalized := makeBlock(1) + ethClient.On("HeadByHash", mock.Anything, mock.Anything).Return(func(_ context.Context, hash gethCommon.Hash) (*evmtypes.Head, error) { + number := hash.Big().Int64() + return makeBlock(number), nil + }) + // run initial backfill to populate the database + err := ht.headTracker.Backfill(ctx, latest, finalized) + require.NoError(b, err) + b.ResetTimer() + for i := 0; i < b.N; i++ { + latest = makeBlock(finalityDepth) + finalized = makeBlock(1) + err := ht.headTracker.Backfill(ctx, latest, finalized) + require.NoError(b, err) + } +} + +func createHeadTracker(t testing.TB, ethClient *evmclimocks.Client, config headtracker.Config, htConfig headtracker.HeadTrackerConfig, orm headtracker.ORM) *headTrackerUniverse { lggr, ob := logger.TestObserved(t, zap.DebugLevel) hb := headtracker.NewHeadBroadcaster(lggr) hs := headtracker.NewHeadSaver(lggr, orm, config, htConfig) diff --git a/core/chains/evm/headtracker/heads.go b/core/chains/evm/headtracker/heads.go index 1edfb3e3788..a61e55dcd28 100644 --- a/core/chains/evm/headtracker/heads.go +++ b/core/chains/evm/headtracker/heads.go @@ -26,8 +26,9 @@ type Heads interface { } type heads struct { - heads []*evmtypes.Head - mu sync.RWMutex + heads []*evmtypes.Head + headsMap map[common.Hash]*evmtypes.Head + mu sync.RWMutex } func NewHeads() Heads { @@ -48,12 +49,11 @@ func (h *heads) HeadByHash(hash common.Hash) *evmtypes.Head { h.mu.RLock() defer h.mu.RUnlock() - for _, head := range h.heads { - if head.Hash == hash { - return head - } + if h.headsMap == nil { + return nil } - return nil + + return h.headsMap[hash] } func (h *heads) Count() int { @@ -74,26 +74,23 @@ func (h *heads) MarkFinalized(finalized common.Hash, minBlockToKeep int64) bool } // deep copy to avoid race on head.Parent - h.heads = deepCopy(h.heads, minBlockToKeep) + h.heads, h.headsMap = deepCopy(h.heads, minBlockToKeep) - head := h.heads[0] - foundFinalized := false - for head != nil { - if head.Hash == finalized { - foundFinalized = true - } - - // we might see finalized to move back in chain due to request to lagging RPC, - // we should not override the flag in such cases - head.IsFinalized = head.IsFinalized || foundFinalized - head = head.Parent + finalizedHead, ok := h.headsMap[finalized] + if !ok { + return false + } + for finalizedHead != nil { + finalizedHead.IsFinalized = true + finalizedHead = finalizedHead.Parent } - return foundFinalized + return true } -func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head { +func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) ([]*evmtypes.Head, map[common.Hash]*evmtypes.Head) { headsMap := make(map[common.Hash]*evmtypes.Head, len(oldHeads)) + heads := make([]*evmtypes.Head, 0, len(headsMap)) for _, head := range oldHeads { if head.Hash == head.ParentHash { // shouldn't happen but it is untrusted input @@ -111,18 +108,11 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head // prefer head that was already in heads as it might have been marked as finalized on previous run if _, ok := headsMap[head.Hash]; !ok { headsMap[head.Hash] = &headCopy + heads = append(heads, &headCopy) } } - heads := make([]*evmtypes.Head, 0, len(headsMap)) - // unsorted unique heads - { - for _, head := range headsMap { - heads = append(heads, head) - } - } - - // sort the heads + // sort the heads as original slice might be out of order sort.SliceStable(heads, func(i, j int) bool { // sorting from the highest number to lowest return heads[i].Number > heads[j].Number @@ -137,7 +127,7 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head } } - return heads + return heads, headsMap } func (h *heads) AddHeads(newHeads ...*evmtypes.Head) { @@ -145,5 +135,5 @@ func (h *heads) AddHeads(newHeads ...*evmtypes.Head) { defer h.mu.Unlock() // deep copy to avoid race on head.Parent - h.heads = deepCopy(append(h.heads, newHeads...), 0) + h.heads, h.headsMap = deepCopy(append(h.heads, newHeads...), 0) } From f17bd16caaaf43da330cb2c1bf54b11c80368d80 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Fri, 17 May 2024 18:04:53 +0200 Subject: [PATCH 2/3] added comments --- .changeset/early-shoes-sit.md | 7 ++- .../evm/headtracker/head_tracker_test.go | 7 ++- core/chains/evm/headtracker/heads.go | 54 +++++++++++-------- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/.changeset/early-shoes-sit.md b/.changeset/early-shoes-sit.md index 18ca4fabf33..189fe37e1fc 100644 --- a/.changeset/early-shoes-sit.md +++ b/.changeset/early-shoes-sit.md @@ -2,4 +2,9 @@ "chainlink": patch --- -Fixed CPU usage issues caused by inefficiencies in HeadTracker #bugfix +Fixed CPU usage issues caused by inefficiencies in HeadTracker. + +HeadTracker's support of finality tags caused a drastic increase in the number of tracked blocks on the Arbitrum chain (from 50 to 12,000), which has led to a 30% increase in CPU usage. + +The fix improves the data structure for tracking blocks and makes lookup more efficient. BenchmarkHeadTracker_Backfill shows 40x time reduction. +#bugfix diff --git a/core/chains/evm/headtracker/head_tracker_test.go b/core/chains/evm/headtracker/head_tracker_test.go index d2ec84bf48a..bf2b984b548 100644 --- a/core/chains/evm/headtracker/head_tracker_test.go +++ b/core/chains/evm/headtracker/head_tracker_test.go @@ -984,6 +984,8 @@ func TestHeadTracker_Backfill(t *testing.T) { }) } +// BenchmarkHeadTracker_Backfill - benchmarks HeadTracker's Backfill with focus on efficiency after initial +// backfill on start up func BenchmarkHeadTracker_Backfill(b *testing.B) { cfg := configtest.NewGeneralConfig(b, nil) @@ -1012,9 +1014,10 @@ func BenchmarkHeadTracker_Backfill(b *testing.B) { err := ht.headTracker.Backfill(ctx, latest, finalized) require.NoError(b, err) b.ResetTimer() + // focus benchmark on processing of a new latest block for i := 0; i < b.N; i++ { - latest = makeBlock(finalityDepth) - finalized = makeBlock(1) + latest = makeBlock(int64(finalityDepth + i)) + finalized = makeBlock(int64(i + 1)) err := ht.headTracker.Backfill(ctx, latest, finalized) require.NoError(b, err) } diff --git a/core/chains/evm/headtracker/heads.go b/core/chains/evm/headtracker/heads.go index a61e55dcd28..1edfb3e3788 100644 --- a/core/chains/evm/headtracker/heads.go +++ b/core/chains/evm/headtracker/heads.go @@ -26,9 +26,8 @@ type Heads interface { } type heads struct { - heads []*evmtypes.Head - headsMap map[common.Hash]*evmtypes.Head - mu sync.RWMutex + heads []*evmtypes.Head + mu sync.RWMutex } func NewHeads() Heads { @@ -49,11 +48,12 @@ func (h *heads) HeadByHash(hash common.Hash) *evmtypes.Head { h.mu.RLock() defer h.mu.RUnlock() - if h.headsMap == nil { - return nil + for _, head := range h.heads { + if head.Hash == hash { + return head + } } - - return h.headsMap[hash] + return nil } func (h *heads) Count() int { @@ -74,23 +74,26 @@ func (h *heads) MarkFinalized(finalized common.Hash, minBlockToKeep int64) bool } // deep copy to avoid race on head.Parent - h.heads, h.headsMap = deepCopy(h.heads, minBlockToKeep) + h.heads = deepCopy(h.heads, minBlockToKeep) - finalizedHead, ok := h.headsMap[finalized] - if !ok { - return false - } - for finalizedHead != nil { - finalizedHead.IsFinalized = true - finalizedHead = finalizedHead.Parent + head := h.heads[0] + foundFinalized := false + for head != nil { + if head.Hash == finalized { + foundFinalized = true + } + + // we might see finalized to move back in chain due to request to lagging RPC, + // we should not override the flag in such cases + head.IsFinalized = head.IsFinalized || foundFinalized + head = head.Parent } - return true + return foundFinalized } -func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) ([]*evmtypes.Head, map[common.Hash]*evmtypes.Head) { +func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head { headsMap := make(map[common.Hash]*evmtypes.Head, len(oldHeads)) - heads := make([]*evmtypes.Head, 0, len(headsMap)) for _, head := range oldHeads { if head.Hash == head.ParentHash { // shouldn't happen but it is untrusted input @@ -108,11 +111,18 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) ([]*evmtypes.Head // prefer head that was already in heads as it might have been marked as finalized on previous run if _, ok := headsMap[head.Hash]; !ok { headsMap[head.Hash] = &headCopy - heads = append(heads, &headCopy) } } - // sort the heads as original slice might be out of order + heads := make([]*evmtypes.Head, 0, len(headsMap)) + // unsorted unique heads + { + for _, head := range headsMap { + heads = append(heads, head) + } + } + + // sort the heads sort.SliceStable(heads, func(i, j int) bool { // sorting from the highest number to lowest return heads[i].Number > heads[j].Number @@ -127,7 +137,7 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) ([]*evmtypes.Head } } - return heads, headsMap + return heads } func (h *heads) AddHeads(newHeads ...*evmtypes.Head) { @@ -135,5 +145,5 @@ func (h *heads) AddHeads(newHeads ...*evmtypes.Head) { defer h.mu.Unlock() // deep copy to avoid race on head.Parent - h.heads, h.headsMap = deepCopy(append(h.heads, newHeads...), 0) + h.heads = deepCopy(append(h.heads, newHeads...), 0) } From df3701f8789d3a03db1849a9e0a9c374d0d9dabc Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Fri, 17 May 2024 19:07:30 +0200 Subject: [PATCH 3/3] revert heads back to the fix --- core/chains/evm/headtracker/heads.go | 54 ++++++++++++---------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/core/chains/evm/headtracker/heads.go b/core/chains/evm/headtracker/heads.go index 1edfb3e3788..a61e55dcd28 100644 --- a/core/chains/evm/headtracker/heads.go +++ b/core/chains/evm/headtracker/heads.go @@ -26,8 +26,9 @@ type Heads interface { } type heads struct { - heads []*evmtypes.Head - mu sync.RWMutex + heads []*evmtypes.Head + headsMap map[common.Hash]*evmtypes.Head + mu sync.RWMutex } func NewHeads() Heads { @@ -48,12 +49,11 @@ func (h *heads) HeadByHash(hash common.Hash) *evmtypes.Head { h.mu.RLock() defer h.mu.RUnlock() - for _, head := range h.heads { - if head.Hash == hash { - return head - } + if h.headsMap == nil { + return nil } - return nil + + return h.headsMap[hash] } func (h *heads) Count() int { @@ -74,26 +74,23 @@ func (h *heads) MarkFinalized(finalized common.Hash, minBlockToKeep int64) bool } // deep copy to avoid race on head.Parent - h.heads = deepCopy(h.heads, minBlockToKeep) + h.heads, h.headsMap = deepCopy(h.heads, minBlockToKeep) - head := h.heads[0] - foundFinalized := false - for head != nil { - if head.Hash == finalized { - foundFinalized = true - } - - // we might see finalized to move back in chain due to request to lagging RPC, - // we should not override the flag in such cases - head.IsFinalized = head.IsFinalized || foundFinalized - head = head.Parent + finalizedHead, ok := h.headsMap[finalized] + if !ok { + return false + } + for finalizedHead != nil { + finalizedHead.IsFinalized = true + finalizedHead = finalizedHead.Parent } - return foundFinalized + return true } -func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head { +func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) ([]*evmtypes.Head, map[common.Hash]*evmtypes.Head) { headsMap := make(map[common.Hash]*evmtypes.Head, len(oldHeads)) + heads := make([]*evmtypes.Head, 0, len(headsMap)) for _, head := range oldHeads { if head.Hash == head.ParentHash { // shouldn't happen but it is untrusted input @@ -111,18 +108,11 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head // prefer head that was already in heads as it might have been marked as finalized on previous run if _, ok := headsMap[head.Hash]; !ok { headsMap[head.Hash] = &headCopy + heads = append(heads, &headCopy) } } - heads := make([]*evmtypes.Head, 0, len(headsMap)) - // unsorted unique heads - { - for _, head := range headsMap { - heads = append(heads, head) - } - } - - // sort the heads + // sort the heads as original slice might be out of order sort.SliceStable(heads, func(i, j int) bool { // sorting from the highest number to lowest return heads[i].Number > heads[j].Number @@ -137,7 +127,7 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head } } - return heads + return heads, headsMap } func (h *heads) AddHeads(newHeads ...*evmtypes.Head) { @@ -145,5 +135,5 @@ func (h *heads) AddHeads(newHeads ...*evmtypes.Head) { defer h.mu.Unlock() // deep copy to avoid race on head.Parent - h.heads = deepCopy(append(h.heads, newHeads...), 0) + h.heads, h.headsMap = deepCopy(append(h.heads, newHeads...), 0) }