Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BCI-3668 Optimise HeadTracker's memory usage #14130

Merged
merged 20 commits into from
Sep 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
48a3d95
base benchmarks
dhaidashenko Aug 5, 2024
51dbe01
Reduce allocs noise from mocks in Backfill Benchmark
dhaidashenko Aug 14, 2024
c256670
Optimize HeadTracker's memory usage
dhaidashenko Aug 15, 2024
c48117f
Merge branch 'develop' into feature/BCI-3668-optimise-ht-memory
dhaidashenko Aug 15, 2024
115c9b5
Merge branch 'develop' into feature/BCI-3668-optimise-ht-memory
dhaidashenko Aug 15, 2024
2913d7f
avoid .Parent race in confirmer
dhaidashenko Aug 16, 2024
569627a
optimise block history estimator heads usage
dhaidashenko Aug 16, 2024
67bb44d
Merge branch 'develop' into feature/BCI-3668-optimise-ht-memory
dhaidashenko Aug 16, 2024
0aa326b
update telemetry test to use new Parent and IsFinalized
dhaidashenko Aug 16, 2024
9c19b5d
Merge branch 'feature/BCI-3668-optimise-ht-memory' of github.com:smar…
dhaidashenko Aug 16, 2024
17b795c
Merge branch 'develop' into feature/BCI-3668-optimise-ht-memory
dhaidashenko Aug 28, 2024
76343e5
avoid redundant allocations
dhaidashenko Aug 29, 2024
546ef5d
Revert "avoid redundant allocations"
dhaidashenko Sep 5, 2024
39b773d
Merge branch 'develop' into feature/BCI-3668-optimise-ht-memory
dhaidashenko Sep 5, 2024
7fd8479
lint fixes
dhaidashenko Sep 5, 2024
70b98a4
remove redundant files
dhaidashenko Sep 5, 2024
12746c4
drop cycle detection logic from head methods. Heads cycle prevention …
dhaidashenko Sep 6, 2024
4af52e2
drop unused test
dhaidashenko Sep 6, 2024
ce20941
nits
dhaidashenko Sep 10, 2024
73c7c9a
Merge branch 'develop' into feature/BCI-3668-optimise-ht-memory
dhaidashenko Sep 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/short-shoes-crash.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"chainlink": patch
---

Optimize HeadTracker's memory usage #internal
5 changes: 3 additions & 2 deletions common/txmgr/confirmer.go
Original file line number Diff line number Diff line change
Expand Up @@ -1148,10 +1148,11 @@ func hasReceiptInLongestChain[
}
}
}
if head.GetParent() == nil {

head = head.GetParent()
if head == nil {
return false
}
head = head.GetParent()
}
}

Expand Down
10 changes: 9 additions & 1 deletion core/chains/evm/client/simulated_backend_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,15 @@ func (c *SimulatedBackendClient) SubscribeNewHead(
case h := <-ch:
var head *evmtypes.Head
if h != nil {
head = &evmtypes.Head{Difficulty: h.Difficulty, Timestamp: time.Unix(int64(h.Time), 0), Number: h.Number.Int64(), Hash: h.Hash(), ParentHash: h.ParentHash, Parent: lastHead, EVMChainID: ubig.New(c.chainId)}
head = &evmtypes.Head{
Difficulty: h.Difficulty,
Timestamp: time.Unix(int64(h.Time), 0), //nolint:gosec
Number: h.Number.Int64(),
Hash: h.Hash(),
ParentHash: h.ParentHash,
EVMChainID: ubig.New(c.chainId),
}
head.Parent.Store(lastHead)
lastHead = head
}
select {
Expand Down
3 changes: 2 additions & 1 deletion core/chains/evm/gas/block_history_estimator.go
Original file line number Diff line number Diff line change
Expand Up @@ -655,12 +655,13 @@ func (b *BlockHistoryEstimator) FetchBlocks(ctx context.Context, head *evmtypes.
}

blocks := make(map[int64]evmtypes.Block)
earliestInChain := head.EarliestInChain()
for _, block := range b.getBlocks() {
// Make a best-effort to be re-org resistant using the head
// chain, refetch blocks that got re-org'd out.
// NOTE: Any blocks in the history that are older than the oldest block
// in the provided chain will be assumed final.
if block.Number < head.EarliestInChain().BlockNumber() {
if block.Number < earliestInChain.BlockNumber() {
blocks[block.Number] = block
} else if head.IsInChain(block.Hash) {
blocks[block.Number] = block
Expand Down
6 changes: 3 additions & 3 deletions core/chains/evm/gas/block_history_estimator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -515,7 +515,7 @@ func TestBlockHistoryEstimator_FetchBlocks(t *testing.T) {

head2 := evmtypes.NewHead(big.NewInt(2), b2.Hash, b1.Hash, uint64(time.Now().Unix()), ubig.New(testutils.FixtureChainID))
head3 := evmtypes.NewHead(big.NewInt(3), b3.Hash, b2.Hash, uint64(time.Now().Unix()), ubig.New(testutils.FixtureChainID))
head3.Parent = &head2
head3.Parent.Store(&head2)
err := bhe.FetchBlocks(tests.Context(t), &head3)
require.NoError(t, err)

Expand Down Expand Up @@ -570,7 +570,7 @@ func TestBlockHistoryEstimator_FetchBlocks(t *testing.T) {
// RE-ORG, head2 and head3 have different hash than saved b2 and b3
head2 := evmtypes.NewHead(big.NewInt(2), utils.NewHash(), b1.Hash, uint64(time.Now().Unix()), ubig.New(testutils.FixtureChainID))
head3 := evmtypes.NewHead(big.NewInt(3), utils.NewHash(), head2.Hash, uint64(time.Now().Unix()), ubig.New(testutils.FixtureChainID))
head3.Parent = &head2
head3.Parent.Store(&head2)

ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool {
return len(b) == 2 &&
Expand Down Expand Up @@ -643,7 +643,7 @@ func TestBlockHistoryEstimator_FetchBlocks(t *testing.T) {
// head2 and head3 have identical hash to saved blocks
head2 := evmtypes.NewHead(big.NewInt(2), b2.Hash, b1.Hash, uint64(time.Now().Unix()), ubig.New(testutils.FixtureChainID))
head3 := evmtypes.NewHead(big.NewInt(3), b3.Hash, head2.Hash, uint64(time.Now().Unix()), ubig.New(testutils.FixtureChainID))
head3.Parent = &head2
head3.Parent.Store(&head2)

err := bhe.FetchBlocks(tests.Context(t), &head3)
require.NoError(t, err)
Expand Down
12 changes: 7 additions & 5 deletions core/chains/evm/headtracker/head_saver.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,12 @@ func NewHeadSaver(lggr logger.Logger, orm ORM, config commontypes.Config, htConf
}

func (hs *headSaver) Save(ctx context.Context, head *evmtypes.Head) error {
if err := hs.orm.IdempotentInsertHead(ctx, head); err != nil {
// adding new head might form a cycle, so it's better to validate cached chain before persisting it
if err := hs.heads.AddHeads(head); err != nil {
return err
}

hs.heads.AddHeads(head)

return nil
return hs.orm.IdempotentInsertHead(ctx, head)
}

func (hs *headSaver) Load(ctx context.Context, latestFinalized int64) (chain *evmtypes.Head, err error) {
Expand All @@ -51,7 +50,10 @@ func (hs *headSaver) Load(ctx context.Context, latestFinalized int64) (chain *ev
return nil, err
}

hs.heads.AddHeads(heads...)
err = hs.heads.AddHeads(heads...)
if err != nil {
return nil, fmt.Errorf("failed to populate cache with loaded heads: %w", err)
}
return hs.heads.LatestHead(), nil
}

Expand Down
3 changes: 1 addition & 2 deletions core/chains/evm/headtracker/head_tracker.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,13 @@ import (

"github.com/smartcontractkit/chainlink/v2/common/headtracker"
commontypes "github.com/smartcontractkit/chainlink/v2/common/headtracker/types"
evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client"
httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types"
evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types"
)

func NewHeadTracker(
lggr logger.Logger,
ethClient evmclient.Client,
ethClient httypes.Client,
config commontypes.Config,
htConfig commontypes.HeadTrackerConfig,
headBroadcaster httypes.HeadBroadcaster,
Expand Down
Loading
Loading