From d964f78af065f3f295753b6f7924f7df2f3b54a6 Mon Sep 17 00:00:00 2001
From: Ryan He <163962984+ryanmorphl2@users.noreply.github.com>
Date: Tue, 9 Jul 2024 12:11:39 +0800
Subject: [PATCH] upgrade rlp, transactions process optimization (#111)
* upgrade rlp to go-ethereum v.14.6
* core/types: transaction and receipt encoding/decoding optimizations #27976
* core/types: use new atomic types in caches #29411, fix eth/handler bug
* eth/fetcher: throttle peers which deliver many invalid transactions #25573
* core: preallocate map in tx_pool #25737
* core/txpool: protect cache with mutex #27898
* upgrade memsize to v0.0.2
---
cmd/devp2p/internal/ethtest/helpers.go | 6 +-
cmd/devp2p/internal/ethtest/suite.go | 4 +
cmd/devp2p/internal/ethtest/transaction.go | 10 ++-
core/tx_list.go | 34 ++++++--
core/tx_pool.go | 6 +-
core/types/hashing.go | 18 ++++
core/types/receipt.go | 21 +++--
core/types/receipt_test.go | 2 +-
core/types/transaction.go | 33 +++++---
core/types/transaction_signing.go | 5 +-
eth/fetcher/tx_fetcher.go | 94 ++++++++++++---------
eth/handler.go | 2 +-
go.mod | 2 +-
go.sum | 4 +-
rlp/decode.go | 34 ++++++--
rlp/decode_test.go | 39 ++++++++-
rlp/doc.go | 11 +--
rlp/encbuffer.go | 98 ++++++++++++++++------
rlp/encode.go | 21 +++--
rlp/encode_test.go | 32 +++++--
rlp/internal/rlpstruct/rlpstruct.go | 4 +-
rlp/iterator.go | 4 +-
rlp/iterator_test.go | 2 +-
rlp/raw.go | 41 ++++++---
rlp/raw_test.go | 67 +++++++++++++--
rlp/rlpgen/gen.go | 18 +++-
rlp/rlpgen/gen_test.go | 25 ++++--
rlp/rlpgen/main.go | 12 +--
rlp/rlpgen/types.go | 16 ++++
rlp/typecache.go | 6 +-
rlp/unsafe.go | 7 +-
31 files changed, 488 insertions(+), 190 deletions(-)
diff --git a/cmd/devp2p/internal/ethtest/helpers.go b/cmd/devp2p/internal/ethtest/helpers.go
index 9320fe94e..dca9177c6 100644
--- a/cmd/devp2p/internal/ethtest/helpers.go
+++ b/cmd/devp2p/internal/ethtest/helpers.go
@@ -457,9 +457,13 @@ func (s *Suite) waitAnnounce(conn *Conn, blockAnnouncement *NewBlock) error {
return fmt.Errorf("wrong block hash in announcement: expected %v, got %v", blockAnnouncement.Block.Hash(), hashes[0].Hash)
}
return nil
+
+ // ignore tx announcements from previous tests
case *NewPooledTransactionHashes:
- // ignore tx announcements from previous tests
continue
+ case *Transactions:
+ continue
+
default:
return fmt.Errorf("unexpected: %s", pretty.Sdump(msg))
}
diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go
index 8049a3339..f0b714fb6 100644
--- a/cmd/devp2p/internal/ethtest/suite.go
+++ b/cmd/devp2p/internal/ethtest/suite.go
@@ -778,9 +778,13 @@ func (s *Suite) TestNewPooledTxs66(t *utesting.T) {
t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg))
}
return
+
// ignore propagated txs from previous tests
case *NewPooledTransactionHashes:
continue
+ case *Transactions:
+ continue
+
// ignore block announcements from previous tests
case *NewBlockHashes:
continue
diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go
index 128b272f0..9884fbf4a 100644
--- a/cmd/devp2p/internal/ethtest/transaction.go
+++ b/cmd/devp2p/internal/ethtest/transaction.go
@@ -29,7 +29,7 @@ import (
"github.com/scroll-tech/go-ethereum/params"
)
-//var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7")
+// var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7")
var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
func (s *Suite) sendSuccessfulTxs(t *utesting.T, isEth66 bool) error {
@@ -200,10 +200,12 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction
}
// update nonce
nonce = txs[len(txs)-1].Nonce()
- // Wait for the transaction announcement(s) and make sure all sent txs are being propagated
+
+ // Wait for the transaction announcement(s) and make sure all sent txs are being propagated.
+ // all txs should be announced within a couple announcements.
recvHashes := make([]common.Hash, 0)
- // all txs should be announced within 3 announcements
- for i := 0; i < 3; i++ {
+
+ for i := 0; i < 20; i++ {
switch msg := recvConn.readAndServe(s.chain, timeout).(type) {
case *Transactions:
for _, tx := range *msg {
diff --git a/core/tx_list.go b/core/tx_list.go
index f21926426..6388a5d3e 100644
--- a/core/tx_list.go
+++ b/core/tx_list.go
@@ -20,6 +20,7 @@ import (
"container/heap"
"math"
"math/big"
+ "slices"
"sort"
"sync"
"sync/atomic"
@@ -49,6 +50,7 @@ func (h *nonceHeap) Pop() interface{} {
old := *h
n := len(old)
x := old[n-1]
+ old[n-1] = 0
*h = old[0 : n-1]
return x
}
@@ -56,9 +58,10 @@ func (h *nonceHeap) Pop() interface{} {
// txSortedMap is a nonce->transaction hash map with a heap based index to allow
// iterating over the contents in a nonce-incrementing way.
type txSortedMap struct {
- items map[uint64]*types.Transaction // Hash map storing the transaction data
- index *nonceHeap // Heap of nonces of all the stored transactions (non-strict mode)
- cache types.Transactions // Cache of the transactions already sorted
+ items map[uint64]*types.Transaction // Hash map storing the transaction data
+ index *nonceHeap // Heap of nonces of all the stored transactions (non-strict mode)
+ cache types.Transactions // Cache of the transactions already sorted
+ cacheMu sync.Mutex // Mutex covering the cache
}
// newTxSortedMap creates a new nonce-sorted transaction map.
@@ -81,7 +84,9 @@ func (m *txSortedMap) Put(tx *types.Transaction) {
if m.items[nonce] == nil {
heap.Push(m.index, nonce)
}
+ m.cacheMu.Lock()
m.items[nonce], m.cache = tx, nil
+ m.cacheMu.Unlock()
}
// Forward removes all transactions from the map with a nonce lower than the
@@ -97,9 +102,11 @@ func (m *txSortedMap) Forward(threshold uint64) types.Transactions {
delete(m.items, nonce)
}
// If we had a cached order, shift the front
+ m.cacheMu.Lock()
if m.cache != nil {
m.cache = m.cache[len(removed):]
}
+ m.cacheMu.Unlock()
return removed
}
@@ -123,7 +130,9 @@ func (m *txSortedMap) reheap() {
*m.index = append(*m.index, nonce)
}
heap.Init(m.index)
+ m.cacheMu.Lock()
m.cache = nil
+ m.cacheMu.Unlock()
}
// filter is identical to Filter, but **does not** regenerate the heap. This method
@@ -139,7 +148,9 @@ func (m *txSortedMap) filter(filter func(*types.Transaction) bool) types.Transac
}
}
if len(removed) > 0 {
+ m.cacheMu.Lock()
m.cache = nil
+ m.cacheMu.Unlock()
}
return removed
}
@@ -153,19 +164,21 @@ func (m *txSortedMap) Cap(threshold int) types.Transactions {
}
// Otherwise gather and drop the highest nonce'd transactions
var drops types.Transactions
-
- sort.Sort(*m.index)
+ slices.Sort(*m.index)
for size := len(m.items); size > threshold; size-- {
drops = append(drops, m.items[(*m.index)[size-1]])
delete(m.items, (*m.index)[size-1])
}
*m.index = (*m.index)[:threshold]
- heap.Init(m.index)
+ // The sorted m.index slice is still a valid heap, so there is no need to
+ // reheap after deleting tail items.
// If we had a cache, shift the back
+ m.cacheMu.Lock()
if m.cache != nil {
m.cache = m.cache[:len(m.cache)-len(drops)]
}
+ m.cacheMu.Unlock()
return drops
}
@@ -185,7 +198,9 @@ func (m *txSortedMap) Remove(nonce uint64) bool {
}
}
delete(m.items, nonce)
+ m.cacheMu.Lock()
m.cache = nil
+ m.cacheMu.Unlock()
return true
}
@@ -195,7 +210,7 @@ func (m *txSortedMap) Remove(nonce uint64) bool {
// removed from the list.
//
// Note, all transactions with nonces lower than start will also be returned to
-// prevent getting into and invalid state. This is not something that should ever
+// prevent getting into an invalid state. This is not something that should ever
// happen but better to be self correcting than failing!
func (m *txSortedMap) Ready(start uint64) types.Transactions {
// Short circuit if no transactions are available
@@ -209,7 +224,9 @@ func (m *txSortedMap) Ready(start uint64) types.Transactions {
delete(m.items, next)
heap.Pop(m.index)
}
+ m.cacheMu.Lock()
m.cache = nil
+ m.cacheMu.Unlock()
return ready
}
@@ -220,6 +237,8 @@ func (m *txSortedMap) Len() int {
}
func (m *txSortedMap) flatten() types.Transactions {
+ m.cacheMu.Lock()
+ defer m.cacheMu.Unlock()
// If the sorting was not cached yet, create and cache it
if m.cache == nil {
m.cache = make(types.Transactions, 0, len(m.items))
@@ -604,6 +623,7 @@ func (l *txPricedList) underpricedFor(h *priceHeap, tx *types.Transaction) bool
// Discard finds a number of most underpriced transactions, removes them from the
// priced list and returns them for further removal from the entire pool.
+// If noPending is set to true, we will only consider the floating list
//
// Note local transaction won't be considered for eviction.
func (l *txPricedList) Discard(slots int, force bool) (types.Transactions, bool) {
diff --git a/core/tx_pool.go b/core/tx_pool.go
index 95907dd97..b6454c1c3 100644
--- a/core/tx_pool.go
+++ b/core/tx_pool.go
@@ -510,11 +510,11 @@ func (pool *TxPool) Content() (map[common.Address]types.Transactions, map[common
pool.mu.Lock()
defer pool.mu.Unlock()
- pending := make(map[common.Address]types.Transactions)
+ pending := make(map[common.Address]types.Transactions, len(pool.pending))
for addr, list := range pool.pending {
pending[addr] = list.Flatten()
}
- queued := make(map[common.Address]types.Transactions)
+ queued := make(map[common.Address]types.Transactions, len(pool.queue))
for addr, list := range pool.queue {
queued[addr] = list.Flatten()
}
@@ -1677,7 +1677,7 @@ type accountSet struct {
// derivations.
func newAccountSet(signer types.Signer, addrs ...common.Address) *accountSet {
as := &accountSet{
- accounts: make(map[common.Address]struct{}),
+ accounts: make(map[common.Address]struct{}, len(addrs)),
signer: signer,
}
for _, addr := range addrs {
diff --git a/core/types/hashing.go b/core/types/hashing.go
index 3c2ce3b3f..6ae2ee66f 100644
--- a/core/types/hashing.go
+++ b/core/types/hashing.go
@@ -18,6 +18,8 @@ package types
import (
"bytes"
+ "fmt"
+ "math"
"sync"
"golang.org/x/crypto/sha3"
@@ -37,6 +39,22 @@ var encodeBufferPool = sync.Pool{
New: func() interface{} { return new(bytes.Buffer) },
}
+// getPooledBuffer retrieves a buffer from the pool and creates a byte slice of the
+// requested size from it.
+//
+// The caller should return the *bytes.Buffer object back into encodeBufferPool after use!
+// The returned byte slice must not be used after returning the buffer.
+func getPooledBuffer(size uint64) ([]byte, *bytes.Buffer, error) {
+ if size > math.MaxInt {
+ return nil, nil, fmt.Errorf("can't get buffer of size %d", size)
+ }
+ buf := encodeBufferPool.Get().(*bytes.Buffer)
+ buf.Reset()
+ buf.Grow(int(size))
+ b := buf.Bytes()[:int(size)]
+ return b, buf, nil
+}
+
// rlpHash encodes x and hashes the encoded bytes.
func rlpHash(x interface{}) (h common.Hash) {
sha := hasherPool.Get().(crypto.KeccakState)
diff --git a/core/types/receipt.go b/core/types/receipt.go
index d2fd1e423..27163212c 100644
--- a/core/types/receipt.go
+++ b/core/types/receipt.go
@@ -38,8 +38,7 @@ var (
receiptStatusSuccessfulRLP = []byte{0x01}
)
-// This error is returned when a typed receipt is decoded, but the string is empty.
-var errEmptyTypedReceipt = errors.New("empty typed receipt bytes")
+var errShortTypedReceipt = errors.New("typed receipt too short")
const (
// ReceiptStatusFailed is the status code of a transaction if execution failed.
@@ -191,7 +190,7 @@ func (r *Receipt) MarshalBinary() ([]byte, error) {
// DecodeRLP implements rlp.Decoder, and loads the consensus fields of a receipt
// from an RLP stream.
func (r *Receipt) DecodeRLP(s *rlp.Stream) error {
- kind, _, err := s.Kind()
+ kind, size, err := s.Kind()
switch {
case err != nil:
return err
@@ -203,15 +202,19 @@ func (r *Receipt) DecodeRLP(s *rlp.Stream) error {
}
r.Type = LegacyTxType
return r.setFromRLP(dec)
- case kind == rlp.String:
+ case kind == rlp.Byte:
+ return errShortTypedReceipt
+ default:
// It's an EIP-2718 typed tx receipt.
- b, err := s.Bytes()
+ b, buf, err := getPooledBuffer(size)
if err != nil {
return err
}
+ defer encodeBufferPool.Put(buf)
+ if err := s.ReadBytes(b); err != nil {
+ return err
+ }
return r.decodeTyped(b)
- default:
- return rlp.ErrExpectedList
}
}
@@ -234,8 +237,8 @@ func (r *Receipt) UnmarshalBinary(b []byte) error {
// decodeTyped decodes a typed receipt from the canonical format.
func (r *Receipt) decodeTyped(b []byte) error {
- if len(b) == 0 {
- return errEmptyTypedReceipt
+ if len(b) <= 1 {
+ return errShortTypedReceipt
}
switch b[0] {
case DynamicFeeTxType, AccessListTxType, BlobTxType, L1MessageTxType:
diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go
index 3d9759ee3..01872bb78 100644
--- a/core/types/receipt_test.go
+++ b/core/types/receipt_test.go
@@ -86,7 +86,7 @@ func TestDecodeEmptyTypedReceipt(t *testing.T) {
input := []byte{0x80}
var r Receipt
err := rlp.DecodeBytes(input, &r)
- if err != errEmptyTypedReceipt {
+ if err != errShortTypedReceipt {
t.Fatal("wrong error:", err)
}
}
diff --git a/core/types/transaction.go b/core/types/transaction.go
index 38e19e1e7..916035367 100644
--- a/core/types/transaction.go
+++ b/core/types/transaction.go
@@ -20,6 +20,7 @@ import (
"bytes"
"container/heap"
"errors"
+ "fmt"
"io"
"math/big"
"sync/atomic"
@@ -60,9 +61,9 @@ type Transaction struct {
time time.Time // Time first seen locally (spam avoidance)
// caches
- hash atomic.Value
- size atomic.Value
- from atomic.Value
+ hash atomic.Pointer[common.Hash]
+ size atomic.Pointer[common.StorageSize]
+ from atomic.Pointer[sigCache]
}
// NewTx creates a new transaction.
@@ -148,13 +149,19 @@ func (tx *Transaction) DecodeRLP(s *rlp.Stream) error {
return errShortTypedTx
default:
// It's an EIP-2718 typed TX envelope.
- var b []byte
- if b, err = s.Bytes(); err != nil {
+ // First read the tx payload bytes into a temporary buffer.
+ b, buf, err := getPooledBuffer(size)
+ if err != nil {
+ return err
+ }
+ defer encodeBufferPool.Put(buf)
+ if err := s.ReadBytes(b); err != nil {
return err
}
+ // Now decode the inner transaction.
inner, err := tx.decodeTyped(b)
if err == nil {
- tx.setDecoded(inner, len(b))
+ tx.setDecoded(inner, int(size))
}
return err
}
@@ -209,7 +216,8 @@ func (tx *Transaction) setDecoded(inner TxData, size int) {
tx.inner = inner
tx.time = time.Now()
if size > 0 {
- tx.size.Store(common.StorageSize(size))
+ newsize := common.StorageSize(size)
+ tx.size.Store(&newsize)
}
}
@@ -461,7 +469,7 @@ func (tx *Transaction) WithoutBlobTxSidecar() *Transaction {
// Hash returns the transaction hash.
func (tx *Transaction) Hash() common.Hash {
if hash := tx.hash.Load(); hash != nil {
- return hash.(common.Hash)
+ return *hash
}
var h common.Hash
@@ -470,7 +478,7 @@ func (tx *Transaction) Hash() common.Hash {
} else {
h = prefixedRlpHash(tx.Type(), tx.inner)
}
- tx.hash.Store(h)
+ tx.hash.Store(&h)
return h
}
@@ -478,7 +486,7 @@ func (tx *Transaction) Hash() common.Hash {
// and returning it, or returning a previously cached value.
func (tx *Transaction) Size() common.StorageSize {
if size := tx.size.Load(); size != nil {
- return size.(common.StorageSize)
+ return *size
}
// Cache miss, encode and cache.
@@ -498,7 +506,7 @@ func (tx *Transaction) Size() common.StorageSize {
size += 1
}
- tx.size.Store(size)
+ tx.size.Store(&size)
return size
}
@@ -509,6 +517,9 @@ func (tx *Transaction) WithSignature(signer Signer, sig []byte) (*Transaction, e
if err != nil {
return nil, err
}
+ if r == nil || s == nil || v == nil {
+ return nil, fmt.Errorf("%w: r: %s, s: %s, v: %s", ErrInvalidSig, r, s, v)
+ }
cpy := tx.inner.copy()
cpy.setSignatureValues(signer.ChainID(), v, r, s)
return &Transaction{inner: cpy, time: tx.time}, nil
diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go
index 20dc3795e..5590976db 100644
--- a/core/types/transaction_signing.go
+++ b/core/types/transaction_signing.go
@@ -131,8 +131,7 @@ func MustSignNewTx(prv *ecdsa.PrivateKey, s Signer, txdata TxData) *Transaction
// signing method. The cache is invalidated if the cached signer does
// not match the signer used in the current call.
func Sender(signer Signer, tx *Transaction) (common.Address, error) {
- if sc := tx.from.Load(); sc != nil {
- sigCache := sc.(sigCache)
+ if sigCache := tx.from.Load(); sigCache != nil {
// If the signer used to derive from in a previous
// call is not the same as used current, invalidate
// the cache.
@@ -145,7 +144,7 @@ func Sender(signer Signer, tx *Transaction) (common.Address, error) {
if err != nil {
return common.Address{}, err
}
- tx.from.Store(sigCache{signer: signer, from: addr})
+ tx.from.Store(&sigCache{signer: signer, from: addr})
return addr, nil
}
diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go
index c4845a34d..ed06d34b7 100644
--- a/eth/fetcher/tx_fetcher.go
+++ b/eth/fetcher/tx_fetcher.go
@@ -263,54 +263,72 @@ func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error {
// direct request replies. The differentiation is important so the fetcher can
// re-shedule missing transactions as soon as possible.
func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error {
- // Keep track of all the propagated transactions
- if direct {
- txReplyInMeter.Mark(int64(len(txs)))
- } else {
- txBroadcastInMeter.Mark(int64(len(txs)))
+ var (
+ inMeter = txReplyInMeter
+ knownMeter = txReplyKnownMeter
+ underpricedMeter = txReplyUnderpricedMeter
+ otherRejectMeter = txReplyOtherRejectMeter
+ )
+ if !direct {
+ inMeter = txBroadcastInMeter
+ knownMeter = txBroadcastKnownMeter
+ underpricedMeter = txBroadcastUnderpricedMeter
+ otherRejectMeter = txBroadcastOtherRejectMeter
}
+ // Keep track of all the propagated transactions
+ inMeter.Mark(int64(len(txs)))
+
// Push all the transactions into the pool, tracking underpriced ones to avoid
// re-requesting them and dropping the peer in case of malicious transfers.
var (
- added = make([]common.Hash, 0, len(txs))
- duplicate int64
- underpriced int64
- otherreject int64
+ added = make([]common.Hash, 0, len(txs))
)
- errs := f.addTxs(txs)
- for i, err := range errs {
- // Track the transaction hash if the price is too low for us.
- // Avoid re-request this transaction when we receive another
- // announcement.
- if errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced) {
- for f.underpriced.Cardinality() >= maxTxUnderpricedSetSize {
- f.underpriced.Pop()
- }
- f.underpriced.Add(txs[i].Hash())
+ // proceed in batches
+ for i := 0; i < len(txs); i += 128 {
+ end := i + 128
+ if end > len(txs) {
+ end = len(txs)
}
- // Track a few interesting failure types
- switch {
- case err == nil: // Noop, but need to handle to not count these
+ var (
+ duplicate int64
+ underpriced int64
+ otherreject int64
+ )
+ batch := txs[i:end]
+ for j, err := range f.addTxs(batch) {
+ // Track the transaction hash if the price is too low for us.
+ // Avoid re-request this transaction when we receive another
+ // announcement.
+ if errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced) {
+ for f.underpriced.Cardinality() >= maxTxUnderpricedSetSize {
+ f.underpriced.Pop()
+ }
+ f.underpriced.Add(batch[j].Hash())
+ }
+ // Track a few interesting failure types
+ switch {
+ case err == nil: // Noop, but need to handle to not count these
- case errors.Is(err, core.ErrAlreadyKnown):
- duplicate++
+ case errors.Is(err, core.ErrAlreadyKnown):
+ duplicate++
- case errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced):
- underpriced++
+ case errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced):
+ underpriced++
- default:
- otherreject++
+ default:
+ otherreject++
+ }
+ added = append(added, batch[j].Hash())
+ }
+ knownMeter.Mark(duplicate)
+ underpricedMeter.Mark(underpriced)
+ otherRejectMeter.Mark(otherreject)
+
+ // If 'other reject' is >25% of the deliveries in any batch, sleep a bit.
+ if otherreject > 128/4 {
+ time.Sleep(200 * time.Millisecond)
+ log.Warn("Peer delivering stale transactions", "peer", peer, "rejected", otherreject)
}
- added = append(added, txs[i].Hash())
- }
- if direct {
- txReplyKnownMeter.Mark(duplicate)
- txReplyUnderpricedMeter.Mark(underpriced)
- txReplyOtherRejectMeter.Mark(otherreject)
- } else {
- txBroadcastKnownMeter.Mark(duplicate)
- txBroadcastUnderpricedMeter.Mark(underpriced)
- txBroadcastOtherRejectMeter.Mark(otherreject)
}
select {
case f.cleanup <- &txDelivery{origin: peer, hashes: added, direct: direct}:
diff --git a/eth/handler.go b/eth/handler.go
index 8e0a4a8a0..c831a5b4f 100644
--- a/eth/handler.go
+++ b/eth/handler.go
@@ -417,7 +417,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error {
return
}
peer.Log().Debug("Whitelist block verified", "number", number, "hash", hash)
-
+ res.Done <- nil
case <-timeout.C:
peer.Log().Warn("Whitelist challenge timed out, dropping", "addr", peer.RemoteAddr(), "type", peer.Name())
h.removePeer(peer.ID())
diff --git a/go.mod b/go.mod
index 39cdcacd3..c82cf5a33 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4
github.com/fatih/color v1.13.0
github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e
- github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5
+ github.com/fjl/memsize v0.0.2
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff
github.com/go-stack/stack v1.8.1
github.com/golang-jwt/jwt/v4 v4.5.0
diff --git a/go.sum b/go.sum
index 6068c10d4..d587d519f 100644
--- a/go.sum
+++ b/go.sum
@@ -152,8 +152,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY=
github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY=
-github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
-github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
+github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA=
+github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
diff --git a/rlp/decode.go b/rlp/decode.go
index 930110380..dfb6edf90 100644
--- a/rlp/decode.go
+++ b/rlp/decode.go
@@ -90,7 +90,7 @@ func Decode(r io.Reader, val interface{}) error {
// DecodeBytes parses RLP data from b into val. Please see package-level documentation for
// the decoding rules. The input must contain exactly one value and no trailing data.
func DecodeBytes(b []byte, val interface{}) error {
- r := bytes.NewReader(b)
+ r := (*sliceReader)(&b)
stream := streamPool.Get().(*Stream)
defer streamPool.Put(stream)
@@ -99,7 +99,7 @@ func DecodeBytes(b []byte, val interface{}) error {
if err := stream.Decode(val); err != nil {
return err
}
- if r.Len() > 0 {
+ if len(b) > 0 {
return ErrMoreThanOneValue
}
return nil
@@ -158,17 +158,17 @@ func makeDecoder(typ reflect.Type, tags rlpstruct.Tags) (dec decoder, err error)
switch {
case typ == rawValueType:
return decodeRawValue, nil
- case typ.AssignableTo(reflect.PtrTo(bigInt)):
+ case typ.AssignableTo(reflect.PointerTo(bigInt)):
return decodeBigInt, nil
case typ.AssignableTo(bigInt):
return decodeBigIntNoPtr, nil
- case typ == reflect.PtrTo(u256Int):
+ case typ == reflect.PointerTo(u256Int):
return decodeU256, nil
case typ == u256Int:
return decodeU256NoPtr, nil
case kind == reflect.Ptr:
return makePtrDecoder(typ, tags)
- case reflect.PtrTo(typ).Implements(decoderInterface):
+ case reflect.PointerTo(typ).Implements(decoderInterface):
return decodeDecoder, nil
case isUint(kind):
return decodeUint, nil
@@ -262,7 +262,7 @@ func decodeU256(s *Stream, val reflect.Value) error {
func makeListDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) {
etype := typ.Elem()
- if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) {
+ if etype.Kind() == reflect.Uint8 && !reflect.PointerTo(etype).Implements(decoderInterface) {
if typ.Kind() == reflect.Array {
return decodeByteArray, nil
}
@@ -474,7 +474,7 @@ func makeSimplePtrDecoder(etype reflect.Type, etypeinfo *typeinfo) decoder {
//
// This decoder is used for pointer-typed struct fields with struct tag "nil".
func makeNilPtrDecoder(etype reflect.Type, etypeinfo *typeinfo, ts rlpstruct.Tags) decoder {
- typ := reflect.PtrTo(etype)
+ typ := reflect.PointerTo(etype)
nilPtr := reflect.Zero(typ)
// Determine the value kind that results in nil pointer.
@@ -1182,3 +1182,23 @@ func (s *Stream) listLimit() (inList bool, limit uint64) {
}
return true, s.stack[len(s.stack)-1]
}
+
+type sliceReader []byte
+
+func (sr *sliceReader) Read(b []byte) (int, error) {
+ if len(*sr) == 0 {
+ return 0, io.EOF
+ }
+ n := copy(b, *sr)
+ *sr = (*sr)[n:]
+ return n, nil
+}
+
+func (sr *sliceReader) ReadByte() (byte, error) {
+ if len(*sr) == 0 {
+ return 0, io.EOF
+ }
+ b := (*sr)[0]
+ *sr = (*sr)[1:]
+ return b, nil
+}
diff --git a/rlp/decode_test.go b/rlp/decode_test.go
index 662ae0e2d..a824ebe10 100644
--- a/rlp/decode_test.go
+++ b/rlp/decode_test.go
@@ -440,6 +440,16 @@ type optionalPtrField struct {
B *[3]byte `rlp:"optional"`
}
+type nonOptionalPtrField struct {
+ A uint
+ B *[3]byte
+}
+
+type multipleOptionalFields struct {
+ A *[3]byte `rlp:"optional"`
+ B *[3]byte `rlp:"optional"`
+}
+
type optionalPtrFieldNil struct {
A uint
B *[3]byte `rlp:"optional,nil"`
@@ -453,7 +463,7 @@ type ignoredField struct {
var (
veryBigInt = new(big.Int).Add(
- big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16),
+ new(big.Int).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16),
big.NewInt(0xFFFF),
)
veryVeryBigInt = new(big.Int).Exp(veryBigInt, big.NewInt(8), nil)
@@ -765,6 +775,30 @@ var decodeTests = []decodeTest{
ptr: new(optionalPtrField),
value: optionalPtrField{A: 1, B: &[3]byte{1, 2, 3}},
},
+ {
+ // all optional fields nil
+ input: "C0",
+ ptr: new(multipleOptionalFields),
+ value: multipleOptionalFields{A: nil, B: nil},
+ },
+ {
+ // all optional fields set
+ input: "C88301020383010203",
+ ptr: new(multipleOptionalFields),
+ value: multipleOptionalFields{A: &[3]byte{1, 2, 3}, B: &[3]byte{1, 2, 3}},
+ },
+ {
+ // nil optional field appears before a non-nil one
+ input: "C58083010203",
+ ptr: new(multipleOptionalFields),
+ error: "rlp: input string too short for [3]uint8, decoding into (rlp.multipleOptionalFields).A",
+ },
+ {
+ // decode a nil ptr into a ptr that is not nil or not optional
+ input: "C20180",
+ ptr: new(nonOptionalPtrField),
+ error: "rlp: input string too short for [3]uint8, decoding into (rlp.nonOptionalPtrField).B",
+ },
{
input: "C101",
ptr: new(optionalPtrFieldNil),
@@ -1064,7 +1098,6 @@ func TestInvalidOptionalField(t *testing.T) {
t.Errorf("wrong error for %T: %v", test.v, err.Error())
}
}
-
}
func ExampleDecode() {
@@ -1245,7 +1278,7 @@ func encodeTestSlice(n uint) []byte {
}
func unhex(str string) []byte {
- b, err := hex.DecodeString(strings.Replace(str, " ", "", -1))
+ b, err := hex.DecodeString(strings.ReplaceAll(str, " ", ""))
if err != nil {
panic(fmt.Sprintf("invalid hex string: %q", str))
}
diff --git a/rlp/doc.go b/rlp/doc.go
index 113828e39..60dff3fa1 100644
--- a/rlp/doc.go
+++ b/rlp/doc.go
@@ -27,8 +27,7 @@ value zero equivalent to the empty string).
RLP values are distinguished by a type tag. The type tag precedes the value in the input
stream and defines the size and kind of the bytes that follow.
-
-Encoding Rules
+# Encoding Rules
Package rlp uses reflection and encodes RLP based on the Go type of the value.
@@ -37,7 +36,7 @@ call EncodeRLP on nil pointer values.
To encode a pointer, the value being pointed to is encoded. A nil pointer to a struct
type, slice or array always encodes as an empty RLP list unless the slice or array has
-elememt type byte. A nil pointer to any other value encodes as the empty string.
+element type byte. A nil pointer to any other value encodes as the empty string.
Struct values are encoded as an RLP list of all their encoded public fields. Recursive
struct types are supported.
@@ -58,8 +57,7 @@ An interface value encodes as the value contained in the interface.
Floating point numbers, maps, channels and functions are not supported.
-
-Decoding Rules
+# Decoding Rules
Decoding uses the following type-dependent rules:
@@ -99,8 +97,7 @@ To decode into an interface value, one of these types is stored in the value:
Non-empty interface types are not supported when decoding.
Signed integers, floating point numbers, maps, channels and functions cannot be decoded into.
-
-Struct Tags
+# Struct Tags
As with other encoding packages, the "-" tag ignores fields.
diff --git a/rlp/encbuffer.go b/rlp/encbuffer.go
index 64dd4fd88..8d3a3b229 100644
--- a/rlp/encbuffer.go
+++ b/rlp/encbuffer.go
@@ -1,3 +1,19 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
package rlp
import (
@@ -39,27 +55,31 @@ func (buf *encBuffer) size() int {
return len(buf.str) + buf.lhsize
}
-// toBytes creates the encoder output.
-func (w *encBuffer) toBytes() []byte {
- out := make([]byte, w.size())
+// makeBytes creates the encoder output.
+func (buf *encBuffer) makeBytes() []byte {
+ out := make([]byte, buf.size())
+ buf.copyTo(out)
+ return out
+}
+
+func (buf *encBuffer) copyTo(dst []byte) {
strpos := 0
pos := 0
- for _, head := range w.lheads {
+ for _, head := range buf.lheads {
// write string data before header
- n := copy(out[pos:], w.str[strpos:head.offset])
+ n := copy(dst[pos:], buf.str[strpos:head.offset])
pos += n
strpos += n
// write the header
- enc := head.encode(out[pos:])
+ enc := head.encode(dst[pos:])
pos += len(enc)
}
// copy string data after the last list header
- copy(out[pos:], w.str[strpos:])
- return out
+ copy(dst[pos:], buf.str[strpos:])
}
-// toWriter writes the encoder output to w.
-func (buf *encBuffer) toWriter(w io.Writer) (err error) {
+// writeTo writes the encoder output to w.
+func (buf *encBuffer) writeTo(w io.Writer) (err error) {
strpos := 0
for _, head := range buf.lheads {
// write string data before header
@@ -121,38 +141,42 @@ func (buf *encBuffer) writeBytes(b []byte) {
}
}
+func (buf *encBuffer) writeString(s string) {
+ buf.writeBytes([]byte(s))
+}
+
// wordBytes is the number of bytes in a big.Word
const wordBytes = (32 << (uint64(^big.Word(0)) >> 63)) / 8
// writeBigInt writes i as an integer.
-func (w *encBuffer) writeBigInt(i *big.Int) {
+func (buf *encBuffer) writeBigInt(i *big.Int) {
bitlen := i.BitLen()
if bitlen <= 64 {
- w.writeUint64(i.Uint64())
+ buf.writeUint64(i.Uint64())
return
}
// Integer is larger than 64 bits, encode from i.Bits().
// The minimal byte length is bitlen rounded up to the next
// multiple of 8, divided by 8.
length := ((bitlen + 7) & -8) >> 3
- w.encodeStringHeader(length)
- w.str = append(w.str, make([]byte, length)...)
+ buf.encodeStringHeader(length)
+ buf.str = append(buf.str, make([]byte, length)...)
index := length
- buf := w.str[len(w.str)-length:]
+ bytesBuf := buf.str[len(buf.str)-length:]
for _, d := range i.Bits() {
for j := 0; j < wordBytes && index > 0; j++ {
index--
- buf[index] = byte(d)
+ bytesBuf[index] = byte(d)
d >>= 8
}
}
}
// writeUint256 writes z as an integer.
-func (w *encBuffer) writeUint256(z *uint256.Int) {
+func (buf *encBuffer) writeUint256(z *uint256.Int) {
bitlen := z.BitLen()
if bitlen <= 64 {
- w.writeUint64(z.Uint64())
+ buf.writeUint64(z.Uint64())
return
}
nBytes := byte((bitlen + 7) / 8)
@@ -162,7 +186,7 @@ func (w *encBuffer) writeUint256(z *uint256.Int) {
binary.BigEndian.PutUint64(b[17:25], z[1])
binary.BigEndian.PutUint64(b[25:33], z[0])
b[32-nBytes] = 0x80 + nBytes
- w.str = append(w.str, b[32-nBytes:]...)
+ buf.str = append(buf.str, b[32-nBytes:]...)
}
// list adds a new list header to the header stack. It returns the index of the header.
@@ -268,6 +292,19 @@ func (r *encReader) next() []byte {
}
}
+func encBufferFromWriter(w io.Writer) *encBuffer {
+ switch w := w.(type) {
+ case EncoderBuffer:
+ return w.buf
+ case *EncoderBuffer:
+ return w.buf
+ case *encBuffer:
+ return w
+ default:
+ return nil
+ }
+}
+
// EncoderBuffer is a buffer for incremental encoding.
//
// The zero value is NOT ready for use. To get a usable buffer,
@@ -295,14 +332,10 @@ func (w *EncoderBuffer) Reset(dst io.Writer) {
// If the destination writer has an *encBuffer, use it.
// Note that w.ownBuffer is left false here.
if dst != nil {
- if outer, ok := dst.(*encBuffer); ok {
+ if outer := encBufferFromWriter(dst); outer != nil {
*w = EncoderBuffer{outer, nil, false}
return
}
- if outer, ok := dst.(EncoderBuffer); ok {
- *w = EncoderBuffer{outer.buf, nil, false}
- return
- }
}
// Get a fresh buffer.
@@ -319,7 +352,7 @@ func (w *EncoderBuffer) Reset(dst io.Writer) {
func (w *EncoderBuffer) Flush() error {
var err error
if w.dst != nil {
- err = w.buf.toWriter(w.dst)
+ err = w.buf.writeTo(w.dst)
}
// Release the internal buffer.
if w.ownBuffer {
@@ -331,7 +364,15 @@ func (w *EncoderBuffer) Flush() error {
// ToBytes returns the encoded bytes.
func (w *EncoderBuffer) ToBytes() []byte {
- return w.buf.toBytes()
+ return w.buf.makeBytes()
+}
+
+// AppendToBytes appends the encoded bytes to dst.
+func (w *EncoderBuffer) AppendToBytes(dst []byte) []byte {
+ size := w.buf.size()
+ out := append(dst, make([]byte, size)...)
+ w.buf.copyTo(out[len(dst):])
+ return out
}
// Write appends b directly to the encoder output.
@@ -365,6 +406,11 @@ func (w EncoderBuffer) WriteBytes(b []byte) {
w.buf.writeBytes(b)
}
+// WriteString encodes s as an RLP string.
+func (w EncoderBuffer) WriteString(s string) {
+ w.buf.writeString(s)
+}
+
// List starts a list. It returns an internal index. Call EndList with
// this index after encoding the content to finish the list.
func (w EncoderBuffer) List() int {
diff --git a/rlp/encode.go b/rlp/encode.go
index ec0b9ef09..2c3ce4bcb 100644
--- a/rlp/encode.go
+++ b/rlp/encode.go
@@ -30,7 +30,10 @@ import (
var (
// Common encoded values.
// These are useful when implementing EncodeRLP.
+
+ // EmptyString is the encoding of an empty string.
EmptyString = []byte{0x80}
+ // EmptyList is the encoding of an empty list.
EmptyList = []byte{0xC0}
)
@@ -57,20 +60,16 @@ type Encoder interface {
// Please see package-level documentation of encoding rules.
func Encode(w io.Writer, val interface{}) error {
// Optimization: reuse *encBuffer when called by EncodeRLP.
- if buf, ok := w.(*encBuffer); ok {
+ if buf := encBufferFromWriter(w); buf != nil {
return buf.encode(val)
}
- if ebuf, ok := w.(EncoderBuffer); ok {
- return ebuf.buf.encode(val)
- }
buf := getEncBuffer()
defer encBufferPool.Put(buf)
-
if err := buf.encode(val); err != nil {
return err
}
- return buf.toWriter(w)
+ return buf.writeTo(w)
}
// EncodeToBytes returns the RLP encoding of val.
@@ -82,7 +81,7 @@ func EncodeToBytes(val interface{}) ([]byte, error) {
if err := buf.encode(val); err != nil {
return nil, err
}
- return buf.toBytes(), nil
+ return buf.makeBytes(), nil
}
// EncodeToReader returns a reader from which the RLP encoding of val
@@ -142,17 +141,17 @@ func makeWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) {
switch {
case typ == rawValueType:
return writeRawValue, nil
- case typ.AssignableTo(reflect.PtrTo(bigInt)):
+ case typ.AssignableTo(reflect.PointerTo(bigInt)):
return writeBigIntPtr, nil
case typ.AssignableTo(bigInt):
return writeBigIntNoPtr, nil
- case typ == reflect.PtrTo(u256Int):
+ case typ == reflect.PointerTo(u256Int):
return writeU256IntPtr, nil
case typ == u256Int:
return writeU256IntNoPtr, nil
case kind == reflect.Ptr:
return makePtrWriter(typ, ts)
- case reflect.PtrTo(typ).Implements(encoderInterface):
+ case reflect.PointerTo(typ).Implements(encoderInterface):
return makeEncoderWriter(typ), nil
case isUint(kind):
return writeUint, nil
@@ -420,7 +419,7 @@ func makeEncoderWriter(typ reflect.Type) writer {
// package json simply doesn't call MarshalJSON for this case, but encodes the
// value as if it didn't implement the interface. We don't want to handle it that
// way.
- return fmt.Errorf("rlp: unadressable value of type %v, EncodeRLP is pointer method", val.Type())
+ return fmt.Errorf("rlp: unaddressable value of type %v, EncodeRLP is pointer method", val.Type())
}
return val.Addr().Interface().(Encoder).EncodeRLP(w)
}
diff --git a/rlp/encode_test.go b/rlp/encode_test.go
index 014de38eb..442e9ad66 100644
--- a/rlp/encode_test.go
+++ b/rlp/encode_test.go
@@ -21,7 +21,6 @@ import (
"errors"
"fmt"
"io"
- "io/ioutil"
"math/big"
"runtime"
"sync"
@@ -121,15 +120,15 @@ var encTests = []encTest{
{val: big.NewInt(0xFFFFFFFFFFFF), output: "86FFFFFFFFFFFF"},
{val: big.NewInt(0xFFFFFFFFFFFFFF), output: "87FFFFFFFFFFFFFF"},
{
- val: big.NewInt(0).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")),
+ val: new(big.Int).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")),
output: "8F102030405060708090A0B0C0D0E0F2",
},
{
- val: big.NewInt(0).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")),
+ val: new(big.Int).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")),
output: "9C0100020003000400050006000700080009000A000B000C000D000E01",
},
{
- val: big.NewInt(0).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")),
+ val: new(big.Int).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")),
output: "A1010000000000000000000000000000000000000000000000000000000000000000",
},
{
@@ -322,6 +321,10 @@ var encTests = []encTest{
{val: &optionalBigIntField{A: 1}, output: "C101"},
{val: &optionalPtrField{A: 1}, output: "C101"},
{val: &optionalPtrFieldNil{A: 1}, output: "C101"},
+ {val: &multipleOptionalFields{A: nil, B: nil}, output: "C0"},
+ {val: &multipleOptionalFields{A: &[3]byte{1, 2, 3}, B: &[3]byte{1, 2, 3}}, output: "C88301020383010203"},
+ {val: &multipleOptionalFields{A: nil, B: &[3]byte{1, 2, 3}}, output: "C58083010203"}, // encodes without error but decode will fail
+ {val: &nonOptionalPtrField{A: 1}, output: "C20180"}, // encodes without error but decode will fail
// nil
{val: (*uint)(nil), output: "80"},
@@ -393,7 +396,7 @@ var encTests = []encTest{
{val: &struct{ TE testEncoder }{testEncoder{errors.New("test error")}}, error: "test error"},
// Verify the error for non-addressable non-pointer Encoder.
- {val: testEncoder{}, error: "rlp: unadressable value of type rlp.testEncoder, EncodeRLP is pointer method"},
+ {val: testEncoder{}, error: "rlp: unaddressable value of type rlp.testEncoder, EncodeRLP is pointer method"},
// Verify Encoder takes precedence over []byte.
{val: []byteEncoder{0, 1, 2, 3, 4}, output: "C5C0C0C0C0C0"},
@@ -431,13 +434,28 @@ func TestEncodeToBytes(t *testing.T) {
runEncTests(t, EncodeToBytes)
}
+func TestEncodeAppendToBytes(t *testing.T) {
+ buffer := make([]byte, 20)
+ runEncTests(t, func(val interface{}) ([]byte, error) {
+ w := NewEncoderBuffer(nil)
+ defer w.Flush()
+
+ err := Encode(w, val)
+ if err != nil {
+ return nil, err
+ }
+ output := w.AppendToBytes(buffer[:0])
+ return output, nil
+ })
+}
+
func TestEncodeToReader(t *testing.T) {
runEncTests(t, func(val interface{}) ([]byte, error) {
_, r, err := EncodeToReader(val)
if err != nil {
return nil, err
}
- return ioutil.ReadAll(r)
+ return io.ReadAll(r)
})
}
@@ -478,7 +496,7 @@ func TestEncodeToReaderReturnToPool(t *testing.T) {
go func() {
for i := 0; i < 1000; i++ {
_, r, _ := EncodeToReader("foo")
- ioutil.ReadAll(r)
+ io.ReadAll(r)
r.Read(buf)
r.Read(buf)
r.Read(buf)
diff --git a/rlp/internal/rlpstruct/rlpstruct.go b/rlp/internal/rlpstruct/rlpstruct.go
index 1ebaa960e..2e3eeb688 100644
--- a/rlp/internal/rlpstruct/rlpstruct.go
+++ b/rlp/internal/rlpstruct/rlpstruct.go
@@ -1,4 +1,4 @@
-// Copyright 2021 The go-ethereum Authors
+// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@@ -44,7 +44,7 @@ type Type struct {
Elem *Type // non-nil for Kind values of Ptr, Slice, Array
}
-// defaultNilValue determines whether a nil pointer to t encodes/decodes
+// DefaultNilValue determines whether a nil pointer to t encodes/decodes
// as an empty string or empty list.
func (t Type) DefaultNilValue() NilKind {
k := t.Kind
diff --git a/rlp/iterator.go b/rlp/iterator.go
index 559e03a86..95bd3f258 100644
--- a/rlp/iterator.go
+++ b/rlp/iterator.go
@@ -1,4 +1,4 @@
-// Copyright 2019 The go-ethereum Authors
+// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@@ -23,7 +23,6 @@ type listIterator struct {
}
// NewListIterator creates an iterator for the (list) represented by data
-// TODO: Consider removing this implementation, as it is no longer used.
func NewListIterator(data RawValue) (*listIterator, error) {
k, t, c, err := readKind(data)
if err != nil {
@@ -36,7 +35,6 @@ func NewListIterator(data RawValue) (*listIterator, error) {
data: data[t : t+c],
}
return it, nil
-
}
// Next forwards the iterator one step, returns true if it was not at end yet
diff --git a/rlp/iterator_test.go b/rlp/iterator_test.go
index 5fa944482..b1d7e9a7e 100644
--- a/rlp/iterator_test.go
+++ b/rlp/iterator_test.go
@@ -1,4 +1,4 @@
-// Copyright 2019 The go-ethereum Authors
+// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/rlp/raw.go b/rlp/raw.go
index 4d0905a33..879e3bfe5 100644
--- a/rlp/raw.go
+++ b/rlp/raw.go
@@ -28,19 +28,35 @@ type RawValue []byte
var rawValueType = reflect.TypeOf(RawValue{})
+// StringSize returns the encoded size of a string.
+func StringSize(s string) uint64 {
+ switch n := len(s); n {
+ case 0:
+ return 1
+ case 1:
+ if s[0] <= 0x7f {
+ return 1
+ } else {
+ return 2
+ }
+ default:
+ return uint64(headsize(uint64(n)) + n)
+ }
+}
+
// BytesSize returns the encoded size of a byte slice.
func BytesSize(b []byte) uint64 {
- switch {
- case len(b) == 0:
+ switch n := len(b); n {
+ case 0:
return 1
- case len(b) == 1:
+ case 1:
if b[0] <= 0x7f {
return 1
} else {
return 2
}
default:
- return uint64(headsize(uint64(len(b))) + len(b))
+ return uint64(headsize(uint64(n)) + n)
}
}
@@ -50,7 +66,8 @@ func ListSize(contentSize uint64) uint64 {
return uint64(headsize(contentSize)) + contentSize
}
-// IntSize returns the encoded size of the integer x.
+// IntSize returns the encoded size of the integer x. Note: The return type of this
+// function is 'int' for backwards-compatibility reasons. The result is always positive.
func IntSize(x uint64) int {
if x < 0x80 {
return 1
@@ -88,18 +105,20 @@ func SplitUint64(b []byte) (x uint64, rest []byte, err error) {
if err != nil {
return 0, b, err
}
- switch {
- case len(content) == 0:
+ switch n := len(content); n {
+ case 0:
return 0, rest, nil
- case len(content) == 1:
+ case 1:
if content[0] == 0 {
return 0, b, ErrCanonInt
}
return uint64(content[0]), rest, nil
- case len(content) > 8:
- return 0, b, errUintOverflow
default:
- x, err = readSize(content, byte(len(content)))
+ if n > 8 {
+ return 0, b, errUintOverflow
+ }
+
+ x, err = readSize(content, byte(n))
if err != nil {
return 0, b, ErrCanonInt
}
diff --git a/rlp/raw_test.go b/rlp/raw_test.go
index 46adff22c..7952b55db 100644
--- a/rlp/raw_test.go
+++ b/rlp/raw_test.go
@@ -60,15 +60,35 @@ func TestCountValues(t *testing.T) {
}
}
-func TestSplitTypes(t *testing.T) {
- if _, _, err := SplitString(unhex("C100")); err != ErrExpectedString {
- t.Errorf("SplitString returned %q, want %q", err, ErrExpectedString)
+func TestSplitString(t *testing.T) {
+ for i, test := range []string{
+ "C0",
+ "C100",
+ "C3010203",
+ "C88363617483646F67",
+ "F8384C6F72656D20697073756D20646F6C6F722073697420616D65742C20636F6E7365637465747572206164697069736963696E6720656C6974",
+ } {
+ if _, _, err := SplitString(unhex(test)); !errors.Is(err, ErrExpectedString) {
+ t.Errorf("test %d: error mismatch: have %q, want %q", i, err, ErrExpectedString)
}
- if _, _, err := SplitList(unhex("01")); err != ErrExpectedList {
- t.Errorf("SplitString returned %q, want %q", err, ErrExpectedList)
}
- if _, _, err := SplitList(unhex("81FF")); err != ErrExpectedList {
- t.Errorf("SplitString returned %q, want %q", err, ErrExpectedList)
+}
+
+func TestSplitList(t *testing.T) {
+ for i, test := range []string{
+ "80",
+ "00",
+ "01",
+ "8180",
+ "81FF",
+ "820400",
+ "83636174",
+ "83646F67",
+ "B8384C6F72656D20697073756D20646F6C6F722073697420616D65742C20636F6E7365637465747572206164697069736963696E6720656C6974",
+ } {
+ if _, _, err := SplitList(unhex(test)); !errors.Is(err, ErrExpectedList) {
+ t.Errorf("test %d: error mismatch: have %q, want %q", i, err, ErrExpectedList)
+ }
}
}
@@ -283,3 +303,36 @@ func TestAppendUint64Random(t *testing.T) {
t.Fatal(err)
}
}
+
+func TestBytesSize(t *testing.T) {
+ tests := []struct {
+ v []byte
+ size uint64
+ }{
+ {v: []byte{}, size: 1},
+ {v: []byte{0x1}, size: 1},
+ {v: []byte{0x7E}, size: 1},
+ {v: []byte{0x7F}, size: 1},
+ {v: []byte{0x80}, size: 2},
+ {v: []byte{0xFF}, size: 2},
+ {v: []byte{0xFF, 0xF0}, size: 3},
+ {v: make([]byte, 55), size: 56},
+ {v: make([]byte, 56), size: 58},
+ }
+
+ for _, test := range tests {
+ s := BytesSize(test.v)
+ if s != test.size {
+ t.Errorf("BytesSize(%#x) -> %d, want %d", test.v, s, test.size)
+ }
+ s = StringSize(string(test.v))
+ if s != test.size {
+ t.Errorf("StringSize(%#x) -> %d, want %d", test.v, s, test.size)
+ }
+ // Sanity check:
+ enc, _ := EncodeToBytes(test.v)
+ if uint64(len(enc)) != test.size {
+ t.Errorf("len(EncodeToBytes(%#x)) -> %d, test says %d", test.v, len(enc), test.size)
+ }
+ }
+}
diff --git a/rlp/rlpgen/gen.go b/rlp/rlpgen/gen.go
index eb2a96b0e..792de27ee 100644
--- a/rlp/rlpgen/gen.go
+++ b/rlp/rlpgen/gen.go
@@ -1,3 +1,19 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
package main
import (
@@ -142,7 +158,7 @@ type op interface {
// basicOp handles basic types bool, uint*, string.
type basicOp struct {
typ types.Type
- writeMethod string // calle write the value
+ writeMethod string // EncoderBuffer writer method name
writeArgType types.Type // parameter type of writeMethod
decMethod string
decResultType types.Type // return type of decMethod
diff --git a/rlp/rlpgen/gen_test.go b/rlp/rlpgen/gen_test.go
index 4c0028471..3b4f5df28 100644
--- a/rlp/rlpgen/gen_test.go
+++ b/rlp/rlpgen/gen_test.go
@@ -1,3 +1,19 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
package main
import (
@@ -8,7 +24,6 @@ import (
"go/parser"
"go/token"
"go/types"
- "io/ioutil"
"os"
"path/filepath"
"testing"
@@ -51,16 +66,16 @@ func TestOutput(t *testing.T) {
// Set this environment variable to regenerate the test outputs.
if os.Getenv("WRITE_TEST_FILES") != "" {
- ioutil.WriteFile(outputFile, output, 0644)
+ os.WriteFile(outputFile, output, 0644)
}
// Check if output matches.
- wantOutput, err := ioutil.ReadFile(outputFile)
+ wantOutput, err := os.ReadFile(outputFile)
if err != nil {
t.Fatal("error loading expected test output:", err)
}
if !bytes.Equal(output, wantOutput) {
- t.Fatal("output mismatch:\n", string(output))
+ t.Fatalf("output mismatch, want: %v got %v", string(wantOutput), string(output))
}
})
}
@@ -68,7 +83,7 @@ func TestOutput(t *testing.T) {
func loadTestSource(file string, typeName string) (*buildContext, *types.Named, error) {
// Load the test input.
- content, err := ioutil.ReadFile(file)
+ content, err := os.ReadFile(file)
if err != nil {
return nil, nil, err
}
diff --git a/rlp/rlpgen/main.go b/rlp/rlpgen/main.go
index bbf44603c..2fec5ff1d 100644
--- a/rlp/rlpgen/main.go
+++ b/rlp/rlpgen/main.go
@@ -1,4 +1,4 @@
-// Copyright 2021 The go-ethereum Authors
+// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@@ -22,7 +22,6 @@ import (
"flag"
"fmt"
"go/types"
- "io/ioutil"
"os"
"golang.org/x/tools/go/packages"
@@ -52,7 +51,7 @@ func main() {
}
if *output == "-" {
os.Stdout.Write(code)
- } else if err := ioutil.WriteFile(*output, code, 0644); err != nil {
+ } else if err := os.WriteFile(*output, code, 0600); err != nil {
fatal(err)
}
}
@@ -74,9 +73,8 @@ type Config struct {
func (cfg *Config) process() (code []byte, err error) {
// Load packages.
pcfg := &packages.Config{
- Mode: packages.NeedName | packages.NeedTypes | packages.NeedImports | packages.NeedDeps,
+ Mode: packages.NeedName | packages.NeedTypes,
Dir: cfg.Dir,
- BuildFlags: []string{"-tags", "norlpgen"},
}
ps, err := packages.Load(pcfg, pathOfPackageRLP, ".")
if err != nil {
@@ -107,7 +105,7 @@ func (cfg *Config) process() (code []byte, err error) {
// Find the type and generate.
typ, err := lookupStructType(pkg.Scope(), cfg.Type)
if err != nil {
- return nil, fmt.Errorf("can't find %s in %s: %v", typ, pkg, err)
+ return nil, fmt.Errorf("can't find %s in %s: %v", cfg.Type, pkg, err)
}
code, err = bctx.generate(typ, cfg.GenerateEncoder, cfg.GenerateDecoder)
if err != nil {
@@ -118,8 +116,6 @@ func (cfg *Config) process() (code []byte, err error) {
// This is done here to avoid processing these lines with gofmt.
var header bytes.Buffer
fmt.Fprint(&header, "// Code generated by rlpgen. DO NOT EDIT.\n\n")
- fmt.Fprint(&header, "//go:build !norlpgen\n")
- fmt.Fprint(&header, "// +build !norlpgen\n\n")
return append(header.Bytes(), code...), nil
}
diff --git a/rlp/rlpgen/types.go b/rlp/rlpgen/types.go
index ba7bd7ee1..ea7dc96d8 100644
--- a/rlp/rlpgen/types.go
+++ b/rlp/rlpgen/types.go
@@ -1,3 +1,19 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
package main
import (
diff --git a/rlp/typecache.go b/rlp/typecache.go
index 6225be76a..0fcb5848e 100644
--- a/rlp/typecache.go
+++ b/rlp/typecache.go
@@ -18,6 +18,7 @@ package rlp
import (
"fmt"
+ "maps"
"reflect"
"sync"
"sync/atomic"
@@ -90,10 +91,7 @@ func (c *typeCache) generate(typ reflect.Type, tags rlpstruct.Tags) *typeinfo {
}
// Copy cur to next.
- c.next = make(map[typekey]*typeinfo, len(cur)+1)
- for k, v := range cur {
- c.next[k] = v
- }
+ c.next = maps.Clone(cur)
// Generate.
info := c.infoWhileGenerating(typ, tags)
diff --git a/rlp/unsafe.go b/rlp/unsafe.go
index 2152ba35f..10868caaf 100644
--- a/rlp/unsafe.go
+++ b/rlp/unsafe.go
@@ -26,10 +26,5 @@ import (
// byteArrayBytes returns a slice of the byte array v.
func byteArrayBytes(v reflect.Value, length int) []byte {
- var s []byte
- hdr := (*reflect.SliceHeader)(unsafe.Pointer(&s))
- hdr.Data = v.UnsafeAddr()
- hdr.Cap = length
- hdr.Len = length
- return s
+ return unsafe.Slice((*byte)(unsafe.Pointer(v.UnsafeAddr())), length)
}