From 953d56e216a50b04e937a847aedc52da536b20b5 Mon Sep 17 00:00:00 2001 From: Fridrik Asmundsson Date: Wed, 19 Apr 2023 18:46:51 +0000 Subject: [PATCH 1/3] perf: Address performance of EthGetTransactionCount We have observed that EthGetTransactionCount is one of the hotspots on Glif production notes, and we are seeing regular 10-20 second latencies when calling this rpc method. I tracked the high latency spikes and they were correlated when we were running ExecuteTipSet while following the chain. To address this, we should not rely on tipset computation to get nounce and instead look at the parent tipset and then count the messages sent from the 'addr'. --- chain/messagepool/messagepool.go | 84 ++++++++++++++++++--------- chain/messagepool/messagepool_test.go | 16 +++++ chain/messagepool/provider.go | 38 ++++++++---- 3 files changed, 100 insertions(+), 38 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index b0e7b7e2b7..03025d7d7e 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -169,13 +169,13 @@ type MessagePool struct { sigValCache *lru.TwoQueueCache[string, struct{}] - nonceCache *lru.Cache[nonceCacheKey, uint64] + stateNonceCache *lru.Cache[stateNonceCacheKey, uint64] evtTypes [3]journal.EventType journal journal.Journal } -type nonceCacheKey struct { +type stateNonceCacheKey struct { tsk types.TipSetKey addr address.Address } @@ -371,7 +371,7 @@ func (ms *msgSet) toSlice() []*types.SignedMessage { func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.UpgradeSchedule, netName dtypes.NetworkName, j journal.Journal) (*MessagePool, error) { cache, _ := lru.New2Q[cid.Cid, crypto.Signature](build.BlsSignatureCacheSize) verifcache, _ := lru.New2Q[string, struct{}](build.VerifSigCacheSize) - noncecache, _ := lru.New[nonceCacheKey, uint64](256) + stateNonceCache, _ := lru.New[stateNonceCacheKey, uint64](256) keycache, _ := lru.New[address.Address, address.Address](1_000_000) cfg, err := loadConfig(ctx, ds) @@ -384,26 +384,26 @@ func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.Upgra } mp := &MessagePool{ - ds: ds, - addSema: make(chan struct{}, 1), - closer: make(chan struct{}), - repubTk: build.Clock.Ticker(RepublishInterval), - repubTrigger: make(chan struct{}, 1), - localAddrs: make(map[address.Address]struct{}), - pending: make(map[address.Address]*msgSet), - keyCache: keycache, - minGasPrice: types.NewInt(0), - getNtwkVersion: us.GetNtwkVersion, - pruneTrigger: make(chan struct{}, 1), - pruneCooldown: make(chan struct{}, 1), - blsSigCache: cache, - sigValCache: verifcache, - nonceCache: noncecache, - changes: lps.New(50), - localMsgs: namespace.Wrap(ds, datastore.NewKey(localMsgsDs)), - api: api, - netName: netName, - cfg: cfg, + ds: ds, + addSema: make(chan struct{}, 1), + closer: make(chan struct{}), + repubTk: build.Clock.Ticker(RepublishInterval), + repubTrigger: make(chan struct{}, 1), + localAddrs: make(map[address.Address]struct{}), + pending: make(map[address.Address]*msgSet), + keyCache: keycache, + minGasPrice: types.NewInt(0), + getNtwkVersion: us.GetNtwkVersion, + pruneTrigger: make(chan struct{}, 1), + pruneCooldown: make(chan struct{}, 1), + blsSigCache: cache, + sigValCache: verifcache, + stateNonceCache: stateNonceCache, + changes: lps.New(50), + localMsgs: namespace.Wrap(ds, datastore.NewKey(localMsgsDs)), + api: api, + netName: netName, + cfg: cfg, evtTypes: [...]journal.EventType{ evtTypeMpoolAdd: j.RegisterEventType("mpool", "add"), evtTypeMpoolRemove: j.RegisterEventType("mpool", "remove"), @@ -1068,24 +1068,52 @@ func (mp *MessagePool) getStateNonce(ctx context.Context, addr address.Address, done := metrics.Timer(ctx, metrics.MpoolGetNonceDuration) defer done() - nk := nonceCacheKey{ + nk := stateNonceCacheKey{ tsk: ts.Key(), addr: addr, } - n, ok := mp.nonceCache.Get(nk) + n, ok := mp.stateNonceCache.Get(nk) if ok { return n, nil } - act, err := mp.api.GetActorAfter(addr, ts) + raddr, err := mp.resolveToKey(ctx, addr) + if err != nil { + return 0, err + } + + // get the nonce from the actor before ts + actor, err := mp.api.GetActorBefore(addr, ts) + if err != nil { + return 0, err + } + nextNonce := actor.Nonce + + // loop over all messages sent by 'addr' and find the highest nonce + messages, err := mp.api.MessagesForTipset(ctx, ts) if err != nil { return 0, err } + for _, message := range messages { + msg := message.VMMessage() + + maddr, err := mp.resolveToKey(ctx, msg.From) + if err != nil { + log.Warnf("failed to resolve message from address: %s", err) + continue + } + + if maddr == raddr { + if n := msg.Nonce + 1; n > nextNonce { + nextNonce = n + } + } + } - mp.nonceCache.Add(nk, act.Nonce) + mp.stateNonceCache.Add(nk, nextNonce) - return act.Nonce, nil + return nextNonce, nil } func (mp *MessagePool) getStateBalance(ctx context.Context, addr address.Address, ts *types.TipSet) (types.BigInt, error) { diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 20da2317e9..a781b50748 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -120,6 +120,22 @@ func (tma *testMpoolAPI) PubSubPublish(string, []byte) error { return nil } +func (tma *testMpoolAPI) GetActorBefore(addr address.Address, ts *types.TipSet) (*types.Actor, error) { + balance, ok := tma.balance[addr] + if !ok { + balance = types.NewInt(1000e6) + tma.balance[addr] = balance + } + + nonce := tma.statenonce[addr] + + return &types.Actor{ + Code: builtin2.AccountActorCodeID, + Nonce: nonce, + Balance: balance, + }, nil +} + func (tma *testMpoolAPI) GetActorAfter(addr address.Address, ts *types.TipSet) (*types.Actor, error) { // regression check for load bug if ts == nil { diff --git a/chain/messagepool/provider.go b/chain/messagepool/provider.go index 123a2607ea..764e6c13a9 100644 --- a/chain/messagepool/provider.go +++ b/chain/messagepool/provider.go @@ -2,6 +2,7 @@ package messagepool import ( "context" + "errors" "time" "github.com/ipfs/go-cid" @@ -27,6 +28,7 @@ type Provider interface { SubscribeHeadChanges(func(rev, app []*types.TipSet) error) *types.TipSet PutMessage(ctx context.Context, m types.ChainMsg) (cid.Cid, error) PubSubPublish(string, []byte) error + GetActorBefore(address.Address, *types.TipSet) (*types.Actor, error) GetActorAfter(address.Address, *types.TipSet) (*types.Actor, error) StateDeterministicAddressAtFinality(context.Context, address.Address, *types.TipSet) (address.Address, error) StateNetworkVersion(context.Context, abi.ChainEpoch) network.Version @@ -58,6 +60,23 @@ func (mpp *mpoolProvider) IsLite() bool { return mpp.lite != nil } +func (mpp *mpoolProvider) getActorLite(addr address.Address, ts *types.TipSet) (*types.Actor, error) { + if !mpp.IsLite() { + return nil, errors.New("should not use getActorLite on non lite Provider") + } + + n, err := mpp.lite.GetNonce(context.TODO(), addr, ts.Key()) + if err != nil { + return nil, xerrors.Errorf("getting nonce over lite: %w", err) + } + a, err := mpp.lite.GetActor(context.TODO(), addr, ts.Key()) + if err != nil { + return nil, xerrors.Errorf("getting actor over lite: %w", err) + } + a.Nonce = n + return a, nil +} + func (mpp *mpoolProvider) SubscribeHeadChanges(cb func(rev, app []*types.TipSet) error) *types.TipSet { mpp.sm.ChainStore().SubscribeHeadChanges( store.WrapHeadChangeCoalescer( @@ -77,18 +96,17 @@ func (mpp *mpoolProvider) PubSubPublish(k string, v []byte) error { return mpp.ps.Publish(k, v) // nolint } +func (mpp *mpoolProvider) GetActorBefore(addr address.Address, ts *types.TipSet) (*types.Actor, error) { + if mpp.IsLite() { + return mpp.getActorLite(addr, ts) + } + + return mpp.sm.LoadActor(context.TODO(), addr, ts) +} + func (mpp *mpoolProvider) GetActorAfter(addr address.Address, ts *types.TipSet) (*types.Actor, error) { if mpp.IsLite() { - n, err := mpp.lite.GetNonce(context.TODO(), addr, ts.Key()) - if err != nil { - return nil, xerrors.Errorf("getting nonce over lite: %w", err) - } - a, err := mpp.lite.GetActor(context.TODO(), addr, ts.Key()) - if err != nil { - return nil, xerrors.Errorf("getting actor over lite: %w", err) - } - a.Nonce = n - return a, nil + return mpp.getActorLite(addr, ts) } stcid, _, err := mpp.sm.TipSetState(context.TODO(), ts) From 553da395e4abd7409884c7f460f679a895709325 Mon Sep 17 00:00:00 2001 From: Fridrik Asmundsson Date: Fri, 21 Apr 2023 11:56:05 +0000 Subject: [PATCH 2/3] perf: Increase noncecache in MessagePool Bumped from 256 to 32k entries which should be about 6MB of cached entries given average nonceCacheKey of 200 bytes --- chain/messagepool/messagepool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 03025d7d7e..54be744636 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -371,7 +371,7 @@ func (ms *msgSet) toSlice() []*types.SignedMessage { func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.UpgradeSchedule, netName dtypes.NetworkName, j journal.Journal) (*MessagePool, error) { cache, _ := lru.New2Q[cid.Cid, crypto.Signature](build.BlsSignatureCacheSize) verifcache, _ := lru.New2Q[string, struct{}](build.VerifSigCacheSize) - stateNonceCache, _ := lru.New[stateNonceCacheKey, uint64](256) + stateNonceCache, _ := lru.New[stateNonceCacheKey, uint64](32768) // 32k * ~200 bytes = 6MB keycache, _ := lru.New[address.Address, address.Address](1_000_000) cfg, err := loadConfig(ctx, ds) From 4028c05feabee46968377810e603927b76064d88 Mon Sep 17 00:00:00 2001 From: Fridrik Asmundsson Date: Fri, 21 Apr 2023 20:03:13 +0000 Subject: [PATCH 3/3] address review comment --- chain/messagepool/messagepool.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 54be744636..6c3e776c09 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -1078,17 +1078,17 @@ func (mp *MessagePool) getStateNonce(ctx context.Context, addr address.Address, return n, nil } - raddr, err := mp.resolveToKey(ctx, addr) + // get the nonce from the actor before ts + actor, err := mp.api.GetActorBefore(addr, ts) if err != nil { return 0, err } + nextNonce := actor.Nonce - // get the nonce from the actor before ts - actor, err := mp.api.GetActorBefore(addr, ts) + raddr, err := mp.resolveToKey(ctx, addr) if err != nil { return 0, err } - nextNonce := actor.Nonce // loop over all messages sent by 'addr' and find the highest nonce messages, err := mp.api.MessagesForTipset(ctx, ts)