From 46d8bc473e9c07cb827129c4776bade967b92b28 Mon Sep 17 00:00:00 2001 From: CMajeri Date: Wed, 28 Jul 2021 15:08:25 +0200 Subject: [PATCH 01/25] Enables fast tracing for CALL family --- core/vm/logger.go | 70 ++++++++++++++++++++++++++++++++++++++++-- internal/ethapi/api.go | 6 +++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/core/vm/logger.go b/core/vm/logger.go index 900a5e58543a..f9579d93b429 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -110,12 +110,22 @@ type Tracer interface { CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) } +type wrappedLog struct { + parent *wrappedLog + error error + log StructLog + children []*wrappedLog +} + // StructLogger is an EVM state logger and implements Tracer. // // StructLogger can capture state based on the given Log configuration and also keeps // a track record of modified storage which is used in reporting snapshots of the // contract their storage. type StructLogger struct { + current *wrappedLog + depth int + cfg LogConfig storage map[common.Address]Storage @@ -145,12 +155,45 @@ func (l *StructLogger) Reset() { // CaptureStart implements the Tracer interface to initialize the tracing operation. func (l *StructLogger) CaptureStart(env *EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { + l.depth = 0 + l.current = &wrappedLog{} +} + +func errToStr(err error) string { + if err == nil { + return "" + } + return err.Error() } // CaptureState logs a new structured log message and pushes it out to the environment // // CaptureState also tracks SLOAD/SSTORE ops to track storage change. func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, err error) { + for ; l.depth > depth-1; l.depth-- { + l.current = l.current.parent + } + if err != nil { + if l.current.error != nil { + l.current.error=fmt.Errorf("%s:%s", l.current.error.Error(), err.Error()) + } + l.current.error = err + } + switch op { + case CALL,DELEGATECALL,STATICCALL,CALLCODE: + l.depth = l.depth+1 + wl := &wrappedLog{ + parent: l.current, + error: l.current.error, + } + l.current.children = append(l.current.children, wl) + l.current = wl + case REVERT: + l.current.error = ErrExecutionReverted + return + default: + return + } memory := scope.Memory stack := scope.Stack contract := scope.Contract @@ -205,7 +248,7 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui } // create a new snapshot of the EVM. log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, rdata, storage, depth, env.StateDB.GetRefund(), err} - l.logs = append(l.logs, log) + l.current.log = log } // CaptureFault implements the Tracer interface to trace an execution fault @@ -215,6 +258,16 @@ func (l *StructLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost ui // CaptureEnd is called after the call finishes to finalize the tracing. func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) { + for ; l.depth > 1; l.depth-- { + l.current = l.current.parent + } + l.current.log = StructLog{ + Op: CALL, + GasCost: gasUsed, + ReturnData: output, + Depth: 0, + Err: err, + } l.output = output l.err = err if l.cfg.Debug { @@ -225,8 +278,21 @@ func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration } } +// Depth first append for all children (stack max depth is 1024) +func (l *wrappedLog) getLogs() []StructLog { + var logs []StructLog + l.log.Err = l.error + logs = append(logs, l.log) + for _, child := range l.children { + logs = append(logs, child.getLogs()...) + } + return logs +} + // StructLogs returns the captured log entries. -func (l *StructLogger) StructLogs() []StructLog { return l.logs } +func (l *StructLogger) StructLogs() []StructLog { + return l.current.getLogs() +} // Error returns the VM error captured by the trace. func (l *StructLogger) Error() error { return l.err } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 1af98e1071da..d1068afc4a4b 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1142,13 +1142,17 @@ type StructLogRes struct { func FormatLogs(logs []vm.StructLog) []StructLogRes { formatted := make([]StructLogRes, len(logs)) for index, trace := range logs { + var errString string + if trace.Err != nil { + errString = trace.Err.Error() + } formatted[index] = StructLogRes{ Pc: trace.Pc, Op: trace.Op.String(), Gas: trace.Gas, GasCost: trace.GasCost, Depth: trace.Depth, - Error: trace.ErrorString(), + Error: errString, } if trace.Stack != nil { stack := make([]string, len(trace.Stack)) From 0b450bcb6d150c2508898c301f81cc9820897ee9 Mon Sep 17 00:00:00 2001 From: CMajeri Date: Wed, 28 Jul 2021 15:21:38 +0200 Subject: [PATCH 02/25] Removes useless code --- core/vm/logger.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/core/vm/logger.go b/core/vm/logger.go index f9579d93b429..0ca35a6fa477 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -159,13 +159,6 @@ func (l *StructLogger) CaptureStart(env *EVM, from common.Address, to common.Add l.current = &wrappedLog{} } -func errToStr(err error) string { - if err == nil { - return "" - } - return err.Error() -} - // CaptureState logs a new structured log message and pushes it out to the environment // // CaptureState also tracks SLOAD/SSTORE ops to track storage change. @@ -174,9 +167,6 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui l.current = l.current.parent } if err != nil { - if l.current.error != nil { - l.current.error=fmt.Errorf("%s:%s", l.current.error.Error(), err.Error()) - } l.current.error = err } switch op { From ff52ea4c2c606b492602dd06d79314f885e99d46 Mon Sep 17 00:00:00 2001 From: Bastien Giegel Date: Mon, 7 Feb 2022 11:13:14 +0100 Subject: [PATCH 03/25] =?UTF-8?q?apply=20log=20wrapper=20chervin=E2=80=99s?= =?UTF-8?q?=20patch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eth/tracers/logger/logger.go | 60 ++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index 8461935822d8..c54d294877c3 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -99,12 +99,22 @@ func (s *StructLog) ErrorString() string { return "" } +type wrappedLog struct { + parent *wrappedLog + error error + log StructLog + children []*wrappedLog +} + // StructLogger is an EVM state logger and implements EVMLogger. // // StructLogger can capture state based on the given Log configuration and also keeps // a track record of modified storage which is used in reporting snapshots of the // contract their storage. type StructLogger struct { + current *wrappedLog + depth int + cfg Config env *vm.EVM @@ -136,12 +146,36 @@ func (l *StructLogger) Reset() { // CaptureStart implements the EVMLogger interface to initialize the tracing operation. func (l *StructLogger) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { l.env = env + l.depth = 0 + l.current = &wrappedLog{} } // CaptureState logs a new structured log message and pushes it out to the environment // // CaptureState also tracks SLOAD/SSTORE ops to track storage change. func (l *StructLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { + for ; l.depth > depth-1; l.depth-- { + l.current = l.current.parent + } + if err != nil { + l.current.error = err + } + switch op { + case vm.CALL, vm.DELEGATECALL, vm.STATICCALL, vm.CALLCODE: + l.depth = l.depth + 1 + wl := &wrappedLog{ + parent: l.current, + error: l.current.error, + } + l.current.children = append(l.current.children, wl) + l.current = wl + case vm.REVERT: + l.current.error = vm.ErrExecutionReverted + return + default: + return + } + memory := scope.Memory stack := scope.Stack contract := scope.Contract @@ -198,7 +232,7 @@ func (l *StructLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, s } // create a new snapshot of the EVM. log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, rdata, storage, depth, l.env.StateDB.GetRefund(), err} - l.logs = append(l.logs, log) + l.current.log = log } // CaptureFault implements the EVMLogger interface to trace an execution fault @@ -208,6 +242,17 @@ func (l *StructLogger) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, s // CaptureEnd is called after the call finishes to finalize the tracing. func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) { + for ; l.depth > 1; l.depth-- { + l.current = l.current.parent + } + l.current.log = StructLog{ + Op: vm.CALL, + GasCost: gasUsed, + ReturnData: output, + Depth: 0, + Err: err, + } + l.output = output l.err = err if l.cfg.Debug { @@ -223,8 +268,19 @@ func (l *StructLogger) CaptureEnter(typ vm.OpCode, from common.Address, to commo func (l *StructLogger) CaptureExit(output []byte, gasUsed uint64, err error) {} +// Depth first append for all children (stack max depth is 1024) +func (l *wrappedLog) getLogs() []StructLog { + var logs []StructLog + l.log.Err = l.error + logs = append(logs, l.log) + for _, child := range l.children { + logs = append(logs, child.getLogs()...) + } + return logs +} + // StructLogs returns the captured log entries. -func (l *StructLogger) StructLogs() []StructLog { return l.logs } +func (l *StructLogger) StructLogs() []StructLog { return l.current.getLogs() } // Error returns the VM error captured by the trace. func (l *StructLogger) Error() error { return l.err } From 4c6d40391b86574a6dc38034deeb7860cfdfae6e Mon Sep 17 00:00:00 2001 From: CMajeri Date: Fri, 27 May 2022 17:22:29 +0200 Subject: [PATCH 04/25] Detect silent failures (no REVERT or error message) --- eth/tracers/logger/logger.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index c54d294877c3..a4c6c21ee59b 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -154,7 +154,14 @@ func (l *StructLogger) CaptureStart(env *vm.EVM, from common.Address, to common. // // CaptureState also tracks SLOAD/SSTORE ops to track storage change. func (l *StructLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { - for ; l.depth > depth-1; l.depth-- { + stack := scope.Stack + for i := l.depth - (depth - 1); l.depth > depth-1; l.depth, i = l.depth-1, i-1 { + if l.current.error == nil { + switch stack.Data()[len(stack.Data())-i].Bytes32()[31] { + case 0x00: + l.current.error = fmt.Errorf("call failed") + } + } l.current = l.current.parent } if err != nil { @@ -177,7 +184,6 @@ func (l *StructLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, s } memory := scope.Memory - stack := scope.Stack contract := scope.Contract // check if already accumulated the specified number of logs if l.cfg.Limit != 0 && l.cfg.Limit <= len(l.logs) { From a29939938bbc506900665c40903baf8c34524cfd Mon Sep 17 00:00:00 2001 From: CMajeri Date: Tue, 21 Jun 2022 09:36:16 +0200 Subject: [PATCH 05/25] Simplify loop --- eth/tracers/logger/logger.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index 0f1f89320cb9..78b279efd725 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -174,7 +174,8 @@ func (l *StructLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, s memory := scope.Memory stack := scope.Stack contract := scope.Contract - for i := l.depth - (depth - 1); l.depth > depth-1; l.depth, i = l.depth-1, i-1 { + for ; l.depth > depth-1; l.depth = l.depth - 1 { + i := l.depth - (depth - 1) if l.current.error == nil { switch stack.Data()[len(stack.Data())-i].Bytes32()[31] { case 0x00: From d0dc349fd36bd79f94516c866251783641ed12f1 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Wed, 31 Aug 2022 16:14:53 +0200 Subject: [PATCH 06/25] graphql: return correct logs for tx (#25612) * graphql: fix tx logs * minor * Use optimized search for selecting tx logs --- graphql/graphql.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 97b460c205ce..66c25841db98 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "math/big" + "sort" "strconv" "github.com/ethereum/go-ethereum" @@ -478,13 +479,16 @@ func (t *Transaction) getLogs(ctx context.Context) (*[]*Log, error) { if err != nil { return nil, err } - ret := make([]*Log, 0, len(logs)) - for _, log := range logs { + var ret []*Log + // Select tx logs from all block logs + ix := sort.Search(len(logs), func(i int) bool { return uint64(logs[i].TxIndex) == t.index }) + for ix < len(logs) && uint64(logs[ix].TxIndex) == t.index { ret = append(ret, &Log{ r: t.r, transaction: t, - log: log, + log: logs[ix], }) + ix++ } return &ret, nil } From 3b41be695e6e12829abf6e5edec0606ee37f14d9 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 13 Sep 2022 21:49:52 +0200 Subject: [PATCH 07/25] graphql: fixes missing tx logs (#25745) * graphql: fix tx logs * graphql: refactor test service setup * graphql: add test for tx logs --- graphql/graphql.go | 2 +- graphql/graphql_test.go | 227 +++++++++++++++++++++------------------- graphql/service.go | 9 +- 3 files changed, 124 insertions(+), 114 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 66c25841db98..356ff669fb16 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -481,7 +481,7 @@ func (t *Transaction) getLogs(ctx context.Context) (*[]*Log, error) { } var ret []*Log // Select tx logs from all block logs - ix := sort.Search(len(logs), func(i int) bool { return uint64(logs[i].TxIndex) == t.index }) + ix := sort.Search(len(logs), func(i int) bool { return uint64(logs[i].TxIndex) >= t.index }) for ix < len(logs) && uint64(logs[ix].TxIndex) == t.index { ret = append(ret, &Log{ r: t.r, diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index d55f4e063486..491c73152113 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -17,6 +17,8 @@ package graphql import ( + "context" + "encoding/json" "fmt" "io" "math/big" @@ -51,15 +53,21 @@ func TestBuildSchema(t *testing.T) { } defer stack.Close() // Make sure the schema can be parsed and matched up to the object model. - if err := newHandler(stack, nil, nil, []string{}, []string{}); err != nil { + if _, err := newHandler(stack, nil, nil, []string{}, []string{}); err != nil { t.Errorf("Could not construct GraphQL handler: %v", err) } } // Tests that a graphQL request is successfully handled when graphql is enabled on the specified endpoint func TestGraphQLBlockSerialization(t *testing.T) { - stack := createNode(t, true, false) + stack := createNode(t) defer stack.Close() + genesis := &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + } + newGQLService(t, stack, genesis, 10, func(i int, gen *core.BlockGen) {}) // start node if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -161,8 +169,55 @@ func TestGraphQLBlockSerialization(t *testing.T) { } func TestGraphQLBlockSerializationEIP2718(t *testing.T) { - stack := createNode(t, true, true) + // Account for signing txes + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + dad = common.HexToAddress("0x0000000000000000000000000000000000000dad") + ) + stack := createNode(t) defer stack.Close() + genesis := &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + Alloc: core.GenesisAlloc{ + address: {Balance: funds}, + // The address 0xdad sloads 0x00 and 0x01 + dad: { + Code: []byte{byte(vm.PC), byte(vm.PC), byte(vm.SLOAD), byte(vm.SLOAD)}, + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + BaseFee: big.NewInt(params.InitialBaseFee), + } + signer := types.LatestSigner(genesis.Config) + newGQLService(t, stack, genesis, 1, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(common.Address{1}) + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: uint64(0), + To: &dad, + Value: big.NewInt(100), + Gas: 50000, + GasPrice: big.NewInt(params.InitialBaseFee), + }) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.AccessListTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(1), + To: &dad, + Gas: 30000, + GasPrice: big.NewInt(params.InitialBaseFee), + Value: big.NewInt(50), + AccessList: types.AccessList{{ + Address: dad, + StorageKeys: []common.Hash{{0}}, + }}, + }) + gen.AddTx(tx) + }) // start node if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -198,7 +253,7 @@ func TestGraphQLBlockSerializationEIP2718(t *testing.T) { // Tests that a graphQL request is not handled successfully when graphql is not enabled on the specified endpoint func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { - stack := createNode(t, false, false) + stack := createNode(t) defer stack.Close() if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -212,7 +267,59 @@ func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { assert.Equal(t, http.StatusNotFound, resp.StatusCode) } -func createNode(t *testing.T, gqlEnabled bool, txEnabled bool) *node.Node { +func TestGraphQLTransactionLogs(t *testing.T) { + var ( + key, _ = crypto.GenerateKey() + addr = crypto.PubkeyToAddress(key.PublicKey) + dadStr = "0x0000000000000000000000000000000000000dad" + dad = common.HexToAddress(dadStr) + genesis = &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + Alloc: core.GenesisAlloc{ + addr: {Balance: big.NewInt(params.Ether)}, + dad: { + // LOG0(0, 0), LOG0(0, 0), RETURN(0, 0) + Code: common.Hex2Bytes("60006000a060006000a060006000f3"), + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + } + signer = types.LatestSigner(genesis.Config) + stack = createNode(t) + ) + defer stack.Close() + + handler := newGQLService(t, stack, genesis, 1, func(i int, gen *core.BlockGen) { + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Nonce: 1, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Nonce: 2, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + }) + // start node + if err := stack.Start(); err != nil { + t.Fatalf("could not start node: %v", err) + } + query := `{block { transactions { logs { account { address } } } } }` + res := handler.Schema.Exec(context.Background(), query, "", map[string]interface{}{}) + if res.Errors != nil { + t.Fatalf("graphql query failed: %v", res.Errors) + } + have, err := json.Marshal(res.Data) + if err != nil { + t.Fatalf("failed to encode graphql response: %s", err) + } + want := fmt.Sprintf(`{"block":{"transactions":[{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]},{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]},{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]}]}}`, dadStr, dadStr, dadStr, dadStr, dadStr, dadStr) + if string(have) != want { + t.Errorf("response unmatch. expected %s, got %s", want, have) + } +} + +func createNode(t *testing.T) *node.Node { stack, err := node.New(&node.Config{ HTTPHost: "127.0.0.1", HTTPPort: 0, @@ -222,83 +329,12 @@ func createNode(t *testing.T, gqlEnabled bool, txEnabled bool) *node.Node { if err != nil { t.Fatalf("could not create node: %v", err) } - if !gqlEnabled { - return stack - } - if !txEnabled { - createGQLService(t, stack) - } else { - createGQLServiceWithTransactions(t, stack) - } return stack } -func createGQLService(t *testing.T, stack *node.Node) { - // create backend - ethConf := ðconfig.Config{ - Genesis: &core.Genesis{ - Config: params.AllEthashProtocolChanges, - GasLimit: 11500000, - Difficulty: big.NewInt(1048576), - }, - Ethash: ethash.Config{ - PowMode: ethash.ModeFake, - }, - NetworkId: 1337, - TrieCleanCache: 5, - TrieCleanCacheJournal: "triecache", - TrieCleanCacheRejournal: 60 * time.Minute, - TrieDirtyCache: 5, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 5, - } - ethBackend, err := eth.New(stack, ethConf) - if err != nil { - t.Fatalf("could not create eth backend: %v", err) - } - // Create some blocks and import them - chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), - ethash.NewFaker(), ethBackend.ChainDb(), 10, func(i int, gen *core.BlockGen) {}) - _, err = ethBackend.BlockChain().InsertChain(chain) - if err != nil { - t.Fatalf("could not create import blocks: %v", err) - } - // create gql service - filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) - err = New(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) - if err != nil { - t.Fatalf("could not create graphql service: %v", err) - } -} - -func createGQLServiceWithTransactions(t *testing.T, stack *node.Node) { - // create backend - key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address := crypto.PubkeyToAddress(key.PublicKey) - funds := big.NewInt(1000000000000000) - dad := common.HexToAddress("0x0000000000000000000000000000000000000dad") - +func newGQLService(t *testing.T, stack *node.Node, gspec *core.Genesis, genBlocks int, genfunc func(i int, gen *core.BlockGen)) *handler { ethConf := ðconfig.Config{ - Genesis: &core.Genesis{ - Config: params.AllEthashProtocolChanges, - GasLimit: 11500000, - Difficulty: big.NewInt(1048576), - Alloc: core.GenesisAlloc{ - address: {Balance: funds}, - // The address 0xdad sloads 0x00 and 0x01 - dad: { - Code: []byte{ - byte(vm.PC), - byte(vm.PC), - byte(vm.SLOAD), - byte(vm.SLOAD), - }, - Nonce: 0, - Balance: big.NewInt(0), - }, - }, - BaseFee: big.NewInt(params.InitialBaseFee), - }, + Genesis: gspec, Ethash: ethash.Config{ PowMode: ethash.ModeFake, }, @@ -310,49 +346,22 @@ func createGQLServiceWithTransactions(t *testing.T, stack *node.Node) { TrieTimeout: 60 * time.Minute, SnapshotCache: 5, } - ethBackend, err := eth.New(stack, ethConf) if err != nil { t.Fatalf("could not create eth backend: %v", err) } - signer := types.LatestSigner(ethConf.Genesis.Config) - - legacyTx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ - Nonce: uint64(0), - To: &dad, - Value: big.NewInt(100), - Gas: 50000, - GasPrice: big.NewInt(params.InitialBaseFee), - }) - envelopTx, _ := types.SignNewTx(key, signer, &types.AccessListTx{ - ChainID: ethConf.Genesis.Config.ChainID, - Nonce: uint64(1), - To: &dad, - Gas: 30000, - GasPrice: big.NewInt(params.InitialBaseFee), - Value: big.NewInt(50), - AccessList: types.AccessList{{ - Address: dad, - StorageKeys: []common.Hash{{0}}, - }}, - }) - // Create some blocks and import them chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), - ethash.NewFaker(), ethBackend.ChainDb(), 1, func(i int, b *core.BlockGen) { - b.SetCoinbase(common.Address{1}) - b.AddTx(legacyTx) - b.AddTx(envelopTx) - }) - + ethash.NewFaker(), ethBackend.ChainDb(), genBlocks, genfunc) _, err = ethBackend.BlockChain().InsertChain(chain) if err != nil { t.Fatalf("could not create import blocks: %v", err) } - // create gql service + // Set up handler filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) - err = New(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) + handler, err := newHandler(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) if err != nil { t.Fatalf("could not create graphql service: %v", err) } + return handler } diff --git a/graphql/service.go b/graphql/service.go index 019026bc7ea7..6f6e58335991 100644 --- a/graphql/service.go +++ b/graphql/service.go @@ -57,17 +57,18 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // New constructs a new GraphQL service instance. func New(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { - return newHandler(stack, backend, filterSystem, cors, vhosts) + _, err := newHandler(stack, backend, filterSystem, cors, vhosts) + return err } // newHandler returns a new `http.Handler` that will answer GraphQL queries. // It additionally exports an interactive query browser on the / endpoint. -func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { +func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) (*handler, error) { q := Resolver{backend, filterSystem} s, err := graphql.ParseSchema(schema, &q) if err != nil { - return err + return nil, err } h := handler{Schema: s} handler := node.NewHTTPHandlerStack(h, cors, vhosts, nil) @@ -76,5 +77,5 @@ func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters. stack.RegisterHandler("GraphQL", "/graphql", handler) stack.RegisterHandler("GraphQL", "/graphql/", handler) - return nil + return &h, nil } From 972007a517c49ee9e2a359950d81c74467492ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 14 Sep 2022 11:07:10 +0200 Subject: [PATCH 08/25] Release Geth v1.10.24 --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 5f24b41f2852..349bdf1c18d1 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 23 // Patch version component of the current release + VersionPatch = 24 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) From 8f61fc8b73565ff15d27fd9731b945d9d1878ec3 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 15 Sep 2022 17:50:54 +0200 Subject: [PATCH 09/25] params: set TerminalTotalDifficultyPassed to true (#25769) * params: set TerminalTotalDifficultyPassed to true * Update params/config.go Co-authored-by: Martin Holst Swende --- params/config.go | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/params/config.go b/params/config.go index 4d6eec8939bc..80e671f9bf41 100644 --- a/params/config.go +++ b/params/config.go @@ -59,25 +59,26 @@ var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(1_150_000), - DAOForkBlock: big.NewInt(1_920_000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2_463_000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2_675_000), - EIP158Block: big.NewInt(2_675_000), - ByzantiumBlock: big.NewInt(4_370_000), - ConstantinopleBlock: big.NewInt(7_280_000), - PetersburgBlock: big.NewInt(7_280_000), - IstanbulBlock: big.NewInt(9_069_000), - MuirGlacierBlock: big.NewInt(9_200_000), - BerlinBlock: big.NewInt(12_244_000), - LondonBlock: big.NewInt(12_965_000), - ArrowGlacierBlock: big.NewInt(13_773_000), - GrayGlacierBlock: big.NewInt(15_050_000), - TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 - Ethash: new(EthashConfig), + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(1_150_000), + DAOForkBlock: big.NewInt(1_920_000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2_463_000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2_675_000), + EIP158Block: big.NewInt(2_675_000), + ByzantiumBlock: big.NewInt(4_370_000), + ConstantinopleBlock: big.NewInt(7_280_000), + PetersburgBlock: big.NewInt(7_280_000), + IstanbulBlock: big.NewInt(9_069_000), + MuirGlacierBlock: big.NewInt(9_200_000), + BerlinBlock: big.NewInt(12_244_000), + LondonBlock: big.NewInt(12_965_000), + ArrowGlacierBlock: big.NewInt(13_773_000), + GrayGlacierBlock: big.NewInt(15_050_000), + TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 + TerminalTotalDifficultyPassed: true, + Ethash: new(EthashConfig), } // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. From 69568c554880b3567bace64f8848ff1be27d084d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 15 Sep 2022 17:55:58 +0200 Subject: [PATCH 10/25] params: release Geth v1.10.25 --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 349bdf1c18d1..3b5978e849cd 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 24 // Patch version component of the current release + VersionPatch = 25 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) From 85e469f787be53192fcbcee86ce67635020a4ad5 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 31 Aug 2022 17:58:18 +0200 Subject: [PATCH 11/25] eth/protocols/snap: fix problems due to idle-but-busy peers (#25651) --- eth/protocols/snap/sync.go | 90 +++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 35 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index deaa4456e0fd..1455bacbcb88 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -2248,14 +2248,18 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.accountIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.accountIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.accountReqs[id] if !ok { @@ -2360,14 +2364,18 @@ func (s *Syncer) onByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) error // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.bytecodeIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.bytecodeReqs[id] if !ok { @@ -2469,14 +2477,18 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.storageIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.storageIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.storageReqs[id] if !ok { @@ -2596,14 +2608,18 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.trienodeHealIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.trienodeHealIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.trienodeHealReqs[id] if !ok { @@ -2691,14 +2707,18 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeHealIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.bytecodeHealIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.bytecodeHealReqs[id] if !ok { From 937ea491f95e5676d2ce3e0364ba9f05452bc351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 9 Sep 2022 11:42:57 +0300 Subject: [PATCH 12/25] eth/protocols/snap: throttle trie heal requests when peers DoS us (#25666) * eth/protocols/snap: throttle trie heal requests when peers DoS us * eth/protocols/snap: lower heal throttle log to debug Co-authored-by: Martin Holst Swende * eth/protocols/snap: fix comment Co-authored-by: Martin Holst Swende --- eth/protocols/snap/sync.go | 107 ++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 7 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 1455bacbcb88..eb8260bf7c97 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -21,10 +21,12 @@ import ( "encoding/json" "errors" "fmt" + gomath "math" "math/big" "math/rand" "sort" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -78,6 +80,29 @@ const ( // and waste round trip times. If it's too high, we're capping responses and // waste bandwidth. maxTrieRequestCount = maxRequestSize / 512 + + // trienodeHealRateMeasurementImpact is the impact a single measurement has on + // the local node's trienode processing capacity. A value closer to 0 reacts + // slower to sudden changes, but it is also more stable against temporary hiccups. + trienodeHealRateMeasurementImpact = 0.005 + + // minTrienodeHealThrottle is the minimum divisor for throttling trie node + // heal requests to avoid overloading the local node and exessively expanding + // the state trie bedth wise. + minTrienodeHealThrottle = 1 + + // maxTrienodeHealThrottle is the maximum divisor for throttling trie node + // heal requests to avoid overloading the local node and exessively expanding + // the state trie bedth wise. + maxTrienodeHealThrottle = maxTrieRequestCount + + // trienodeHealThrottleIncrease is the multiplier for the throttle when the + // rate of arriving data is higher than the rate of processing it. + trienodeHealThrottleIncrease = 1.33 + + // trienodeHealThrottleDecrease is the divisor for the throttle when the + // rate of arriving data is lower than the rate of processing it. + trienodeHealThrottleDecrease = 1.25 ) var ( @@ -431,6 +456,11 @@ type Syncer struct { trienodeHealReqs map[uint64]*trienodeHealRequest // Trie node requests currently running bytecodeHealReqs map[uint64]*bytecodeHealRequest // Bytecode requests currently running + trienodeHealRate float64 // Average heal rate for processing trie node data + trienodeHealPend uint64 // Number of trie nodes currently pending for processing + trienodeHealThrottle float64 // Divisor for throttling the amount of trienode heal data requested + trienodeHealThrottled time.Time // Timestamp the last time the throttle was updated + trienodeHealSynced uint64 // Number of state trie nodes downloaded trienodeHealBytes common.StorageSize // Number of state trie bytes persisted to disk trienodeHealDups uint64 // Number of state trie nodes already processed @@ -476,9 +506,10 @@ func NewSyncer(db ethdb.KeyValueStore) *Syncer { trienodeHealIdlers: make(map[string]struct{}), bytecodeHealIdlers: make(map[string]struct{}), - trienodeHealReqs: make(map[uint64]*trienodeHealRequest), - bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), - stateWriter: db.NewBatch(), + trienodeHealReqs: make(map[uint64]*trienodeHealRequest), + bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), + trienodeHealThrottle: maxTrienodeHealThrottle, // Tune downward instead of insta-filling with junk + stateWriter: db.NewBatch(), extProgress: new(SyncProgress), } @@ -1321,6 +1352,10 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai if cap > maxTrieRequestCount { cap = maxTrieRequestCount } + cap = int(float64(cap) / s.trienodeHealThrottle) + if cap <= 0 { + cap = 1 + } var ( hashes = make([]common.Hash, 0, cap) paths = make([]string, 0, cap) @@ -2090,6 +2125,10 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { // processTrienodeHealResponse integrates an already validated trienode response // into the healer tasks. func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { + var ( + start = time.Now() + fills int + ) for i, hash := range res.hashes { node := res.nodes[i] @@ -2098,6 +2137,8 @@ func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { res.task.trieTasks[res.paths[i]] = res.hashes[i] continue } + fills++ + // Push the trie node into the state syncer s.trienodeHealSynced++ s.trienodeHealBytes += common.StorageSize(len(node)) @@ -2121,6 +2162,50 @@ func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { log.Crit("Failed to persist healing data", "err", err) } log.Debug("Persisted set of healing data", "type", "trienodes", "bytes", common.StorageSize(batch.ValueSize())) + + // Calculate the processing rate of one filled trie node + rate := float64(fills) / (float64(time.Since(start)) / float64(time.Second)) + + // Update the currently measured trienode queueing and processing throughput. + // + // The processing rate needs to be updated uniformly independent if we've + // processed 1x100 trie nodes or 100x1 to keep the rate consistent even in + // the face of varying network packets. As such, we cannot just measure the + // time it took to process N trie nodes and update once, we need one update + // per trie node. + // + // Naively, that would be: + // + // for i:=0; i time.Second { + // Periodically adjust the trie node throttler + if float64(pending) > 2*s.trienodeHealRate { + s.trienodeHealThrottle *= trienodeHealThrottleIncrease + } else { + s.trienodeHealThrottle /= trienodeHealThrottleDecrease + } + if s.trienodeHealThrottle > maxTrienodeHealThrottle { + s.trienodeHealThrottle = maxTrienodeHealThrottle + } else if s.trienodeHealThrottle < minTrienodeHealThrottle { + s.trienodeHealThrottle = minTrienodeHealThrottle + } + s.trienodeHealThrottled = time.Now() + + log.Debug("Updated trie node heal throttler", "rate", s.trienodeHealRate, "pending", pending, "throttle", s.trienodeHealThrottle) + } } // processBytecodeHealResponse integrates an already validated bytecode response @@ -2655,10 +2740,12 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error // Cross reference the requested trienodes with the response to find gaps // that the serving node is missing - hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) - hash := make([]byte, 32) - - nodes := make([][]byte, len(req.hashes)) + var ( + hasher = sha3.NewLegacyKeccak256().(crypto.KeccakState) + hash = make([]byte, 32) + nodes = make([][]byte, len(req.hashes)) + fills uint64 + ) for i, j := 0, 0; i < len(trienodes); i++ { // Find the next hash that we've been served, leaving misses with nils hasher.Reset() @@ -2670,16 +2757,22 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error } if j < len(req.hashes) { nodes[j] = trienodes[i] + fills++ j++ continue } // We've either ran out of hashes, or got unrequested data logger.Warn("Unexpected healing trienodes", "count", len(trienodes)-i) + // Signal this request as failed, and ready for rescheduling s.scheduleRevertTrienodeHealRequest(req) return errors.New("unexpected healing trienode") } // Response validated, send it to the scheduler for filling + atomic.AddUint64(&s.trienodeHealPend, fills) + defer func() { + atomic.AddUint64(&s.trienodeHealPend, ^(fills - 1)) + }() response := &trienodeHealResponse{ paths: req.paths, task: req.task, From a32e69a28c2abef6a6b96e8d2466f1f27fd693ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 6 Sep 2022 12:57:03 +0300 Subject: [PATCH 13/25] trie: check childrens' existence concurrently for snap heal (#25694) --- trie/sync.go | 57 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/trie/sync.go b/trie/sync.go index 303fcbfa22e2..862ce7e16e6c 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -19,6 +19,7 @@ package trie import ( "errors" "fmt" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/prque" @@ -381,11 +382,11 @@ func (s *Sync) scheduleCodeRequest(req *codeRequest) { // retrieval scheduling. func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { // Gather all the children of the node, irrelevant whether known or not - type child struct { + type childNode struct { path []byte node node } - var children []child + var children []childNode switch node := (object).(type) { case *shortNode: @@ -393,14 +394,14 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { if hasTerm(key) { key = key[:len(key)-1] } - children = []child{{ + children = []childNode{{ node: node.Val, path: append(append([]byte(nil), req.path...), key...), }} case *fullNode: for i := 0; i < 17; i++ { if node.Children[i] != nil { - children = append(children, child{ + children = append(children, childNode{ node: node.Children[i], path: append(append([]byte(nil), req.path...), byte(i)), }) @@ -410,7 +411,10 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { panic(fmt.Sprintf("unknown node: %+v", node)) } // Iterate over the children, and request all unknown ones - requests := make([]*nodeRequest, 0, len(children)) + var ( + missing = make(chan *nodeRequest, len(children)) + pending sync.WaitGroup + ) for _, child := range children { // Notify any external watcher of a new key/value node if req.callback != nil { @@ -433,19 +437,36 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { if s.membatch.hasNode(child.path) { continue } - // If database says duplicate, then at least the trie node is present - // and we hold the assumption that it's NOT legacy contract code. - chash := common.BytesToHash(node) - if rawdb.HasTrieNode(s.database, chash) { - continue - } - // Locally unknown node, schedule for retrieval - requests = append(requests, &nodeRequest{ - path: child.path, - hash: chash, - parent: req, - callback: req.callback, - }) + // Check the presence of children concurrently + pending.Add(1) + go func(child childNode) { + defer pending.Done() + + // If database says duplicate, then at least the trie node is present + // and we hold the assumption that it's NOT legacy contract code. + chash := common.BytesToHash(node) + if rawdb.HasTrieNode(s.database, chash) { + return + } + // Locally unknown node, schedule for retrieval + missing <- &nodeRequest{ + path: child.path, + hash: chash, + parent: req, + callback: req.callback, + } + }(child) + } + } + pending.Wait() + + requests := make([]*nodeRequest, 0, len(children)) + for done := false; !done; { + select { + case miss := <-missing: + requests = append(requests, miss) + default: + done = true } } return requests, nil From 99bbb337011ef614fc5d25c130aa1cef92915366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 20 Sep 2022 14:14:24 +0300 Subject: [PATCH 14/25] eth: fix a rare datarace on CHT challenge reply / shutdown (#25831) --- eth/handler.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index 4224a9f33a84..143147b0c815 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -391,11 +391,16 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { if h.checkpointHash != (common.Hash{}) { // Request the peer's checkpoint header for chain height/weight validation resCh := make(chan *eth.Response) - if _, err := peer.RequestHeadersByNumber(h.checkpointNumber, 1, 0, false, resCh); err != nil { + + req, err := peer.RequestHeadersByNumber(h.checkpointNumber, 1, 0, false, resCh) + if err != nil { return err } // Start a timer to disconnect if the peer doesn't reply in time go func() { + // Ensure the request gets cancelled in case of error/drop + defer req.Close() + timeout := time.NewTimer(syncChallengeTimeout) defer timeout.Stop() @@ -437,10 +442,15 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { // If we have any explicit peer required block hashes, request them for number, hash := range h.requiredBlocks { resCh := make(chan *eth.Response) - if _, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh); err != nil { + + req, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh) + if err != nil { return err } - go func(number uint64, hash common.Hash) { + go func(number uint64, hash common.Hash, req *eth.Request) { + // Ensure the request gets cancelled in case of error/drop + defer req.Close() + timeout := time.NewTimer(syncChallengeTimeout) defer timeout.Stop() @@ -469,7 +479,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Warn("Required block challenge timed out, dropping", "addr", peer.RemoteAddr(), "type", peer.Name()) h.removePeer(peer.ID()) } - }(number, hash) + }(number, hash, req) } // Handle incoming messages until the connection is torn down return handler(peer) From 27600a5b8452c39ce9ef5c502c98a0c1ee35859b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 27 Oct 2022 15:25:01 +0200 Subject: [PATCH 15/25] eth/filters: change filter block to be by-ref (#26054) This PR changes the block field in the filter to be a pointer, to disambiguate between empty hash and no hash --- eth/filters/filter.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 0a70c9ece1db..8d80647fc241 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -34,8 +34,8 @@ type Filter struct { addresses []common.Address topics [][]common.Hash - block common.Hash // Block hash if filtering a single block - begin, end int64 // Range interval if filtering multiple blocks + block *common.Hash // Block hash if filtering a single block + begin, end int64 // Range interval if filtering multiple blocks matcher *bloombits.Matcher } @@ -78,7 +78,7 @@ func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Add func (sys *FilterSystem) NewBlockFilter(block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { // Create a generic filter and convert it into a block filter filter := newFilter(sys, addresses, topics) - filter.block = block + filter.block = &block return filter } @@ -96,8 +96,8 @@ func newFilter(sys *FilterSystem, addresses []common.Address, topics [][]common. // first block that contains matches, updating the start of the filter accordingly. func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { // If we're doing singleton block filtering, execute and return - if f.block != (common.Hash{}) { - header, err := f.sys.backend.HeaderByHash(ctx, f.block) + if f.block != nil { + header, err := f.sys.backend.HeaderByHash(ctx, *f.block) if err != nil { return nil, err } From 211dbb719711420358a3cdf718ce476c212adcf5 Mon Sep 17 00:00:00 2001 From: Jordan Krage Date: Wed, 2 Nov 2022 09:29:33 -0500 Subject: [PATCH 16/25] rpc: handle wrong HTTP batch response length (#26064) --- rpc/client.go | 1 + rpc/client_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++ rpc/http.go | 3 +++ 3 files changed, 52 insertions(+) diff --git a/rpc/client.go b/rpc/client.go index d3ce0297754c..fcd31319a3c5 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -31,6 +31,7 @@ import ( ) var ( + ErrBadResult = errors.New("bad result in JSON-RPC response") ErrClientQuit = errors.New("client is closed") ErrNoResult = errors.New("no result in JSON-RPC response") ErrSubscriptionQueueOverflow = errors.New("subscription queue overflow") diff --git a/rpc/client_test.go b/rpc/client_test.go index 04c847d0d626..5ae549d86533 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -19,6 +19,7 @@ package rpc import ( "context" "encoding/json" + "errors" "fmt" "math/rand" "net" @@ -144,6 +145,53 @@ func TestClientBatchRequest(t *testing.T) { } } +func TestClientBatchRequest_len(t *testing.T) { + b, err := json.Marshal([]jsonrpcMessage{ + {Version: "2.0", ID: json.RawMessage("1"), Method: "foo", Result: json.RawMessage(`"0x1"`)}, + {Version: "2.0", ID: json.RawMessage("2"), Method: "bar", Result: json.RawMessage(`"0x2"`)}, + }) + if err != nil { + t.Fatal("failed to encode jsonrpc message:", err) + } + s := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + _, err := rw.Write(b) + if err != nil { + t.Error("failed to write response:", err) + } + })) + t.Cleanup(s.Close) + + client, err := Dial(s.URL) + if err != nil { + t.Fatal("failed to dial test server:", err) + } + defer client.Close() + + t.Run("too-few", func(t *testing.T) { + batch := []BatchElem{ + {Method: "foo"}, + {Method: "bar"}, + {Method: "baz"}, + } + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second) + defer cancelFn() + if err := client.BatchCallContext(ctx, batch); !errors.Is(err, ErrBadResult) { + t.Errorf("expected %q but got: %v", ErrBadResult, err) + } + }) + + t.Run("too-many", func(t *testing.T) { + batch := []BatchElem{ + {Method: "foo"}, + } + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second) + defer cancelFn() + if err := client.BatchCallContext(ctx, batch); !errors.Is(err, ErrBadResult) { + t.Errorf("expected %q but got: %v", ErrBadResult, err) + } + }) +} + func TestClientNotify(t *testing.T) { server := newTestServer() defer server.Stop() diff --git a/rpc/http.go b/rpc/http.go index 858d80858652..b82ea2707a54 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -173,6 +173,9 @@ func (c *Client) sendBatchHTTP(ctx context.Context, op *requestOp, msgs []*jsonr if err := json.NewDecoder(respBody).Decode(&respmsgs); err != nil { return err } + if len(respmsgs) != len(msgs) { + return fmt.Errorf("batch has %d requests but response has %d: %w", len(msgs), len(respmsgs), ErrBadResult) + } for i := 0; i < len(respmsgs); i++ { op.resp <- &respmsgs[i] } From e5eb32acee19cc9fca6a03b10283b7484246b15a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 3 Nov 2022 11:59:33 +0100 Subject: [PATCH 17/25] params: release geth v1.10.26 stable --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 3b5978e849cd..d9a3958512c3 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 25 // Patch version component of the current release + VersionPatch = 26 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) From a0491a04b5beb295bc90d4408881ac4aaa00712d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Aug 2023 06:50:09 -0400 Subject: [PATCH 18/25] deps: update supranational/blst to 0.3.11 (#27890) build(deps): bump github.com/supranational/blst Bumps [github.com/supranational/blst](https://github.com/supranational/blst) from 0.3.11-0.20230406105308-e9dfc5ee724b to 0.3.11. - [Release notes](https://github.com/supranational/blst/releases) - [Commits](https://github.com/supranational/blst/commits/v0.3.11) --- updated-dependencies: - dependency-name: github.com/supranational/blst dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9c7121c0355c..7694f6eeaf47 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.2.0 github.com/stretchr/testify v1.8.1 - github.com/supranational/blst v0.3.11-0.20230406105308-e9dfc5ee724b + github.com/supranational/blst v0.3.11 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v2 v2.24.1 diff --git a/go.sum b/go.sum index 591764e65830..a54b22e3dc0d 100644 --- a/go.sum +++ b/go.sum @@ -426,8 +426,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/supranational/blst v0.3.11-0.20230406105308-e9dfc5ee724b h1:u49mjRnygnB34h8OKbnNJFVUtWSKIKb1KukdV8bILUM= -github.com/supranational/blst v0.3.11-0.20230406105308-e9dfc5ee724b/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= From 55863ce0d811d490888b297d36dae2673ce9409b Mon Sep 17 00:00:00 2001 From: imulmat4 <117636097+imulmat4@users.noreply.github.com> Date: Fri, 11 Aug 2023 02:12:26 -0500 Subject: [PATCH 19/25] core/txpool/blobpool: fix metrics name for prometheus export (#27901) --- core/txpool/blobpool/metrics.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/txpool/blobpool/metrics.go b/core/txpool/blobpool/metrics.go index 280913b3a916..070cc5ca4712 100644 --- a/core/txpool/blobpool/metrics.go +++ b/core/txpool/blobpool/metrics.go @@ -35,15 +35,15 @@ var ( // The below metrics track the per-shelf metrics for the primary blob store // and the temporary limbo store. - shelfDatausedGaugeName = "blobpool/shelf-%d/dataused" - shelfDatagapsGaugeName = "blobpool/shelf-%d/datagaps" - shelfSlotusedGaugeName = "blobpool/shelf-%d/slotused" - shelfSlotgapsGaugeName = "blobpool/shelf-%d/slotgaps" + shelfDatausedGaugeName = "blobpool/shelf_%d/dataused" + shelfDatagapsGaugeName = "blobpool/shelf_%d/datagaps" + shelfSlotusedGaugeName = "blobpool/shelf_%d/slotused" + shelfSlotgapsGaugeName = "blobpool/shelf_%d/slotgaps" - limboShelfDatausedGaugeName = "blobpool/limbo/shelf-%d/dataused" - limboShelfDatagapsGaugeName = "blobpool/limbo/shelf-%d/datagaps" - limboShelfSlotusedGaugeName = "blobpool/limbo/shelf-%d/slotused" - limboShelfSlotgapsGaugeName = "blobpool/limbo/shelf-%d/slotgaps" + limboShelfDatausedGaugeName = "blobpool/limbo/shelf_%d/dataused" + limboShelfDatagapsGaugeName = "blobpool/limbo/shelf_%d/datagaps" + limboShelfSlotusedGaugeName = "blobpool/limbo/shelf_%d/slotused" + limboShelfSlotgapsGaugeName = "blobpool/limbo/shelf_%d/slotgaps" // The oversized metrics aggregate the shelf stats above the max blob count // limits to track transactions that are just huge, but don't contain blobs. From 770db145ab9b37187b03456246496766ea769f54 Mon Sep 17 00:00:00 2001 From: Delweng Date: Fri, 11 Aug 2023 21:15:49 +0800 Subject: [PATCH 20/25] ethdb/leveldb: support more than 7 levels in metrics (#27904) --- ethdb/leveldb/leveldb.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 190327accb75..c0e0eb250a75 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -75,7 +75,7 @@ type Database struct { seekCompGauge metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt manualMemAllocGauge metrics.Gauge // Gauge to track the amount of memory that has been manually allocated (not a part of runtime/GC) - levelsGauge [7]metrics.Gauge // Gauge for tracking the number of tables in levels + levelsGauge []metrics.Gauge // Gauge for tracking the number of tables in levels quitLock sync.Mutex // Mutex protecting the quit channel access quitChan chan chan error // Quit channel to stop the metrics collection before closing the database @@ -146,13 +146,8 @@ func NewCustom(file string, namespace string, customize func(options *opt.Option ldb.seekCompGauge = metrics.NewRegisteredGauge(namespace+"compact/seek", nil) ldb.manualMemAllocGauge = metrics.NewRegisteredGauge(namespace+"memory/manualalloc", nil) - // leveldb has only up to 7 levels - for i := range ldb.levelsGauge { - ldb.levelsGauge[i] = metrics.NewRegisteredGauge(namespace+fmt.Sprintf("tables/level%v", i), nil) - } - // Start up the metrics gathering and return - go ldb.meter(metricsGatheringInterval) + go ldb.meter(metricsGatheringInterval, namespace) return ldb, nil } @@ -271,7 +266,7 @@ func (db *Database) Path() string { // meter periodically retrieves internal leveldb counters and reports them to // the metrics subsystem. -func (db *Database) meter(refresh time.Duration) { +func (db *Database) meter(refresh time.Duration, namespace string) { // Create the counters to store current and previous compaction values compactions := make([][]int64, 2) for i := 0; i < 2; i++ { @@ -360,8 +355,11 @@ func (db *Database) meter(refresh time.Duration) { db.nonlevel0CompGauge.Update(int64(stats.NonLevel0Comp)) db.seekCompGauge.Update(int64(stats.SeekComp)) - // update tables amount for i, tables := range stats.LevelTablesCounts { + // Append metrics for additional layers + if i >= len(db.levelsGauge) { + db.levelsGauge = append(db.levelsGauge, metrics.NewRegisteredGauge(namespace+fmt.Sprintf("tables/level%v", i), nil)) + } db.levelsGauge[i].Update(int64(tables)) } From 811a674059af075203b4897fe995f1664e954048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sat, 12 Aug 2023 01:04:12 +0300 Subject: [PATCH 21/25] all: update golang/x/ext and fix slice sorting fallout (#27909) The Go authors updated golang/x/ext to change the function signature of the slices sort method. It's an entire shitshow now because x/ext is not tagged, so everyone's codebase just picked a new version that some other dep depends on, causing our code to fail building. This PR updates the dep on our code too and does all the refactorings to follow upstream... --- accounts/keystore/account_cache.go | 4 +-- cmd/devp2p/dns_route53.go | 12 +++++-- cmd/devp2p/nodeset.go | 14 +++++--- common/types.go | 12 +++---- consensus/clique/snapshot.go | 2 +- consensus/clique/snapshot_test.go | 2 +- core/blockchain.go | 4 +-- core/rawdb/accessors_chain.go | 4 +-- core/rawdb/chain_iterator_test.go | 8 ++--- core/state/snapshot/difflayer.go | 4 +-- core/state/snapshot/iterator_fast.go | 18 +++++++---- eth/api_debug_test.go | 2 +- eth/gasprice/feehistory.go | 4 +-- eth/gasprice/gasprice.go | 6 ++-- eth/protocols/snap/sync_test.go | 16 +++++----- eth/tracers/api_test.go | 2 +- ethdb/dbtest/testsuite.go | 2 +- go.mod | 4 +-- go.sum | 8 ++--- internal/ethapi/api_test.go | 2 +- les/servingqueue.go | 10 ++++-- les/utils/limiter.go | 10 ++++-- metrics/writer.go | 8 ++--- metrics/writer_test.go | 2 +- p2p/discover/table_util_test.go | 8 ++--- p2p/discover/v4_lookup_test.go | 4 +-- p2p/discover/v5_udp_test.go | 4 +-- p2p/dnsdisc/tree.go | 4 +-- p2p/peer.go | 2 +- p2p/protocol.go | 15 ++++++--- p2p/server.go | 2 +- tests/fuzzers/rangeproof/rangeproof-fuzzer.go | 4 +-- tests/fuzzers/stacktrie/trie_fuzzer.go | 4 +-- trie/iterator_test.go | 4 +-- trie/proof_test.go | 32 +++++++++---------- trie/triedb/pathdb/history.go | 6 ++-- trie/triedb/pathdb/testutils.go | 2 +- trie/trienode/node.go | 6 ++-- 38 files changed, 144 insertions(+), 113 deletions(-) diff --git a/accounts/keystore/account_cache.go b/accounts/keystore/account_cache.go index a88b0fa5f369..4ed1439514e1 100644 --- a/accounts/keystore/account_cache.go +++ b/accounts/keystore/account_cache.go @@ -40,8 +40,8 @@ import ( const minReloadInterval = 2 * time.Second // byURL defines the sorting order for accounts. -func byURL(a, b accounts.Account) bool { - return a.URL.Cmp(b.URL) < 0 +func byURL(a, b accounts.Account) int { + return a.URL.Cmp(b.URL) } // AmbiguousAddrError is returned when attempting to unlock diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index eae6f70f5827..21a32f9414e4 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -288,11 +288,17 @@ func makeDeletionChanges(records map[string]recordSet, keep map[string]string) [ // sortChanges ensures DNS changes are in leaf-added -> root-changed -> leaf-deleted order. func sortChanges(changes []types.Change) { score := map[string]int{"CREATE": 1, "UPSERT": 2, "DELETE": 3} - slices.SortFunc(changes, func(a, b types.Change) bool { + slices.SortFunc(changes, func(a, b types.Change) int { if a.Action == b.Action { - return *a.ResourceRecordSet.Name < *b.ResourceRecordSet.Name + return strings.Compare(*a.ResourceRecordSet.Name, *b.ResourceRecordSet.Name) } - return score[string(a.Action)] < score[string(b.Action)] + if score[string(a.Action)] < score[string(b.Action)] { + return -1 + } + if score[string(a.Action)] > score[string(b.Action)] { + return 1 + } + return 0 }) } diff --git a/cmd/devp2p/nodeset.go b/cmd/devp2p/nodeset.go index c040c1a908ba..7360dc5bcfd9 100644 --- a/cmd/devp2p/nodeset.go +++ b/cmd/devp2p/nodeset.go @@ -77,8 +77,8 @@ func (ns nodeSet) nodes() []*enode.Node { result = append(result, n.N) } // Sort by ID. - slices.SortFunc(result, func(a, b *enode.Node) bool { - return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) < 0 + slices.SortFunc(result, func(a, b *enode.Node) int { + return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) }) return result } @@ -103,8 +103,14 @@ func (ns nodeSet) topN(n int) nodeSet { for _, v := range ns { byscore = append(byscore, v) } - slices.SortFunc(byscore, func(a, b nodeJSON) bool { - return a.Score >= b.Score + slices.SortFunc(byscore, func(a, b nodeJSON) int { + if a.Score > b.Score { + return -1 + } + if a.Score < b.Score { + return 1 + } + return 0 }) result := make(nodeSet, n) for _, v := range byscore[:n] { diff --git a/common/types.go b/common/types.go index 93fb0904545e..bf74e43716a1 100644 --- a/common/types.go +++ b/common/types.go @@ -65,9 +65,9 @@ func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) } // If b is larger than len(h), b will be cropped from the left. func HexToHash(s string) Hash { return BytesToHash(FromHex(s)) } -// Less compares two hashes. -func (h Hash) Less(other Hash) bool { - return bytes.Compare(h[:], other[:]) < 0 +// Cmp compares two hashes. +func (h Hash) Cmp(other Hash) int { + return bytes.Compare(h[:], other[:]) } // Bytes gets the byte representation of the underlying hash. @@ -231,9 +231,9 @@ func IsHexAddress(s string) bool { return len(s) == 2*AddressLength && isHex(s) } -// Less compares two addresses. -func (a Address) Less(other Address) bool { - return bytes.Compare(a[:], other[:]) < 0 +// Cmp compares two addresses. +func (a Address) Cmp(other Address) int { + return bytes.Compare(a[:], other[:]) } // Bytes gets the string representation of the underlying address. diff --git a/consensus/clique/snapshot.go b/consensus/clique/snapshot.go index 0f1b9f4b3a16..a97115121b82 100644 --- a/consensus/clique/snapshot.go +++ b/consensus/clique/snapshot.go @@ -308,7 +308,7 @@ func (s *Snapshot) signers() []common.Address { for sig := range s.Signers { sigs = append(sigs, sig) } - slices.SortFunc(sigs, common.Address.Less) + slices.SortFunc(sigs, common.Address.Cmp) return sigs } diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index f30900f274a6..26cebe008a88 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -53,7 +53,7 @@ func (ap *testerAccountPool) checkpoint(header *types.Header, signers []string) for i, signer := range signers { auths[i] = ap.address(signer) } - slices.SortFunc(auths, common.Address.Less) + slices.SortFunc(auths, common.Address.Cmp) for i, auth := range auths { copy(header.Extra[extraVanity+i*common.AddressLength:], auth.Bytes()) } diff --git a/core/blockchain.go b/core/blockchain.go index 3952c31b688f..ba6eb574667c 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -996,8 +996,8 @@ func (bc *BlockChain) procFutureBlocks() { } } if len(blocks) > 0 { - slices.SortFunc(blocks, func(a, b *types.Block) bool { - return a.NumberU64() < b.NumberU64() + slices.SortFunc(blocks, func(a, b *types.Block) int { + return a.Number().Cmp(b.Number()) }) // Insert one by one as chain insertion needs contiguous ancestry between blocks for i := range blocks { diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 9eeaaee941ff..c7abd44f365c 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -902,9 +902,9 @@ func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) { Header: block.Header(), Body: block.Body(), }) - slices.SortFunc(badBlocks, func(a, b *badBlock) bool { + slices.SortFunc(badBlocks, func(a, b *badBlock) int { // Note: sorting in descending number order. - return a.Header.Number.Uint64() >= b.Header.Number.Uint64() + return -a.Header.Number.Cmp(b.Header.Number) }) if len(badBlocks) > badBlockToKeep { badBlocks = badBlocks[:badBlockToKeep] diff --git a/core/rawdb/chain_iterator_test.go b/core/rawdb/chain_iterator_test.go index 9e18c8605cf1..9580cd92a873 100644 --- a/core/rawdb/chain_iterator_test.go +++ b/core/rawdb/chain_iterator_test.go @@ -19,12 +19,12 @@ package rawdb import ( "math/big" "reflect" + "sort" "sync" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "golang.org/x/exp/slices" ) func TestChainIterator(t *testing.T) { @@ -92,11 +92,9 @@ func TestChainIterator(t *testing.T) { } } if !c.reverse { - slices.Sort(numbers) + sort.Ints(numbers) } else { - slices.SortFunc(numbers, func(a, b int) bool { - return a > b // Sort descending - }) + sort.Sort(sort.Reverse(sort.IntSlice(numbers))) } if !reflect.DeepEqual(numbers, c.expect) { t.Fatalf("Case %d failed, visit element mismatch, want %v, got %v", i, c.expect, numbers) diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index b10b43b1ae58..b6aca599c509 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -525,7 +525,7 @@ func (dl *diffLayer) AccountList() []common.Hash { dl.accountList = append(dl.accountList, hash) } } - slices.SortFunc(dl.accountList, common.Hash.Less) + slices.SortFunc(dl.accountList, common.Hash.Cmp) dl.memory += uint64(len(dl.accountList) * common.HashLength) return dl.accountList } @@ -563,7 +563,7 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) for k := range storageMap { storageList = append(storageList, k) } - slices.SortFunc(storageList, common.Hash.Less) + slices.SortFunc(storageList, common.Hash.Cmp) dl.storageList[accountHash] = storageList dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) return storageList, destructed diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index 339f930ffd81..0502d9cf8596 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -33,19 +33,25 @@ type weightedIterator struct { priority int } -func (it *weightedIterator) Less(other *weightedIterator) bool { +func (it *weightedIterator) Cmp(other *weightedIterator) int { // Order the iterators primarily by the account hashes hashI := it.it.Hash() hashJ := other.it.Hash() switch bytes.Compare(hashI[:], hashJ[:]) { case -1: - return true + return -1 case 1: - return false + return 1 } // Same account/storage-slot in multiple layers, split by priority - return it.priority < other.priority + if it.priority < other.priority { + return -1 + } + if it.priority > other.priority { + return 1 + } + return 0 } // fastIterator is a more optimized multi-layer iterator which maintains a @@ -155,9 +161,7 @@ func (fi *fastIterator) init() { } } // Re-sort the entire list - slices.SortFunc(fi.iterators, func(a, b *weightedIterator) bool { - return a.Less(b) - }) + slices.SortFunc(fi.iterators, func(a, b *weightedIterator) int { return a.Cmp(b) }) fi.initiated = false } diff --git a/eth/api_debug_test.go b/eth/api_debug_test.go index 4b08bd4fc57d..6a1b537c25ce 100644 --- a/eth/api_debug_test.go +++ b/eth/api_debug_test.go @@ -105,7 +105,7 @@ func TestAccountRange(t *testing.T) { } // Test to see if it's possible to recover from the middle of the previous // set and get an even split between the first and second sets. - slices.SortFunc(hList, common.Hash.Less) + slices.SortFunc(hList, common.Hash.Cmp) middleH := hList[AccountRangeMaxResults/2] middleResult := accountRangeTest(t, &trie, sdb, middleH, AccountRangeMaxResults, AccountRangeMaxResults) missing, infirst, insecond := 0, 0, 0 diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index b9cee45ae77c..226991b24b2c 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -111,8 +111,8 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { reward, _ := tx.EffectiveGasTip(bf.block.BaseFee()) sorter[i] = txGasAndReward{gasUsed: bf.receipts[i].GasUsed, reward: reward} } - slices.SortStableFunc(sorter, func(a, b txGasAndReward) bool { - return a.reward.Cmp(b.reward) < 0 + slices.SortStableFunc(sorter, func(a, b txGasAndReward) int { + return a.reward.Cmp(b.reward) }) var txIndex int diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index 9eccd7242862..b71964981145 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -208,7 +208,7 @@ func (oracle *Oracle) SuggestTipCap(ctx context.Context) (*big.Int, error) { } price := lastPrice if len(results) > 0 { - slices.SortFunc(results, func(a, b *big.Int) bool { return a.Cmp(b) < 0 }) + slices.SortFunc(results, func(a, b *big.Int) int { return a.Cmp(b) }) price = results[(len(results)-1)*oracle.percentile/100] } if price.Cmp(oracle.maxPrice) > 0 { @@ -247,12 +247,12 @@ func (oracle *Oracle) getBlockValues(ctx context.Context, blockNum uint64, limit sortedTxs := make([]*types.Transaction, len(txs)) copy(sortedTxs, txs) baseFee := block.BaseFee() - slices.SortFunc(sortedTxs, func(a, b *types.Transaction) bool { + slices.SortFunc(sortedTxs, func(a, b *types.Transaction) int { // It's okay to discard the error because a tx would never be // accepted into a block with an invalid effective tip. tip1, _ := a.EffectiveGasTip(baseFee) tip2, _ := b.EffectiveGasTip(baseFee) - return tip1.Cmp(tip2) < 0 + return tip1.Cmp(tip2) }) var prices []*big.Int diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 79b1cad25b0b..0aa6fd8730dd 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1321,8 +1321,8 @@ type kv struct { k, v []byte } -func (k *kv) less(other *kv) bool { - return bytes.Compare(k.k, other.k) < 0 +func (k *kv) cmp(other *kv) int { + return bytes.Compare(k.k, other.k) } func key32(i uint64) []byte { @@ -1382,7 +1382,7 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) { accTrie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Commit the state changes into db and re-create the trie // for accessing later. @@ -1444,7 +1444,7 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) { accTrie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Commit the state changes into db and re-create the trie // for accessing later. @@ -1491,7 +1491,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Commit account trie root, set, _ := accTrie.Commit(true) @@ -1556,7 +1556,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (strin storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Commit account trie root, set, _ := accTrie.Commit(true) @@ -1600,7 +1600,7 @@ func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Databas trie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) root, nodes, _ := trie.Commit(false) return root, nodes, entries } @@ -1651,7 +1651,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo trie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) root, nodes, _ := trie.Commit(false) return root, nodes, entries } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 11d7ff4b694e..0f78af9a01e5 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -790,7 +790,7 @@ func newAccounts(n int) (accounts []Account) { addr := crypto.PubkeyToAddress(key.PublicKey) accounts = append(accounts, Account{key: key, addr: addr}) } - slices.SortFunc(accounts, func(a, b Account) bool { return a.addr.Less(b.addr) }) + slices.SortFunc(accounts, func(a, b Account) int { return a.addr.Cmp(b.addr) }) return accounts } diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go index ac10ddde93b2..0d3d5f5aa694 100644 --- a/ethdb/dbtest/testsuite.go +++ b/ethdb/dbtest/testsuite.go @@ -527,7 +527,7 @@ func makeDataset(size, ksize, vsize int, order bool) ([][]byte, [][]byte) { vals = append(vals, randBytes(vsize)) } if order { - slices.SortFunc(keys, func(a, b []byte) bool { return bytes.Compare(a, b) < 0 }) + slices.SortFunc(keys, func(a, b []byte) int { return bytes.Compare(a, b) }) } return keys, vals } diff --git a/go.mod b/go.mod index 7694f6eeaf47..e4273ef5b034 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( github.com/urfave/cli/v2 v2.24.1 go.uber.org/automaxprocs v1.5.2 golang.org/x/crypto v0.9.0 - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc + golang.org/x/exp v0.0.0-20230810033253-352e893a4cad golang.org/x/sync v0.3.0 golang.org/x/sys v0.9.0 golang.org/x/text v0.9.0 @@ -125,7 +125,7 @@ require ( github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - golang.org/x/mod v0.10.0 // indirect + golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index a54b22e3dc0d..50271e857fdf 100644 --- a/go.sum +++ b/go.sum @@ -479,8 +479,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU= +golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -490,8 +490,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 671b9b2e4972..b0877dc37849 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -817,7 +817,7 @@ func newAccounts(n int) (accounts []Account) { addr := crypto.PubkeyToAddress(key.PublicKey) accounts = append(accounts, Account{key: key, addr: addr}) } - slices.SortFunc(accounts, func(a, b Account) bool { return a.addr.Less(b.addr) }) + slices.SortFunc(accounts, func(a, b Account) int { return a.addr.Cmp(b.addr) }) return accounts } diff --git a/les/servingqueue.go b/les/servingqueue.go index c25925403ad7..68cad9cb5a17 100644 --- a/les/servingqueue.go +++ b/les/servingqueue.go @@ -215,8 +215,14 @@ func (sq *servingQueue) freezePeers() { tasks.list = append(tasks.list, task) tasks.sumTime += task.expTime } - slices.SortFunc(peerList, func(a, b *peerTasks) bool { - return a.priority < b.priority + slices.SortFunc(peerList, func(a, b *peerTasks) int { + if a.priority < b.priority { + return -1 + } + if a.priority > b.priority { + return 1 + } + return 0 }) drop := true for _, tasks := range peerList { diff --git a/les/utils/limiter.go b/les/utils/limiter.go index 155bd801500a..70b7ff64f721 100644 --- a/les/utils/limiter.go +++ b/les/utils/limiter.go @@ -369,8 +369,14 @@ func (l *Limiter) dropRequests() { priority: w / float64(nq.sumCost), }) } - slices.SortFunc(list, func(a, b dropListItem) bool { - return a.priority < b.priority + slices.SortFunc(list, func(a, b dropListItem) int { + if a.priority < b.priority { + return -1 + } + if a.priority < b.priority { + return 1 + } + return 0 }) for _, item := range list { for _, request := range item.nq.queue { diff --git a/metrics/writer.go b/metrics/writer.go index 3d60485b6b5b..82434e9d1d62 100644 --- a/metrics/writer.go +++ b/metrics/writer.go @@ -3,6 +3,7 @@ package metrics import ( "fmt" "io" + "strings" "time" "golang.org/x/exp/slices" @@ -23,8 +24,7 @@ func WriteOnce(r Registry, w io.Writer) { r.Each(func(name string, i interface{}) { namedMetrics = append(namedMetrics, namedMetric{name, i}) }) - - slices.SortFunc(namedMetrics, namedMetric.less) + slices.SortFunc(namedMetrics, namedMetric.cmp) for _, namedMetric := range namedMetrics { switch metric := namedMetric.m.(type) { case Counter: @@ -92,6 +92,6 @@ type namedMetric struct { m interface{} } -func (m namedMetric) less(other namedMetric) bool { - return m.name < other.name +func (m namedMetric) cmp(other namedMetric) int { + return strings.Compare(m.name, other.name) } diff --git a/metrics/writer_test.go b/metrics/writer_test.go index a4c92addc9b9..8376bf8975c8 100644 --- a/metrics/writer_test.go +++ b/metrics/writer_test.go @@ -14,7 +14,7 @@ func TestMetricsSorting(t *testing.T) { {name: "ggg"}, } - slices.SortFunc(namedMetrics, namedMetric.less) + slices.SortFunc(namedMetrics, namedMetric.cmp) for i, name := range []string{"bbb", "fff", "ggg", "zzz"} { if namedMetrics[i].name != name { t.Fail() diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 52544a2c9348..8f3813bdcf0f 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -217,14 +217,14 @@ func nodeEqual(n1 *enode.Node, n2 *enode.Node) bool { } func sortByID(nodes []*enode.Node) { - slices.SortFunc(nodes, func(a, b *enode.Node) bool { - return string(a.ID().Bytes()) < string(b.ID().Bytes()) + slices.SortFunc(nodes, func(a, b *enode.Node) int { + return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) }) } func sortedByDistanceTo(distbase enode.ID, slice []*node) bool { - return slices.IsSortedFunc(slice, func(a, b *node) bool { - return enode.DistCmp(distbase, a.ID(), b.ID()) < 0 + return slices.IsSortedFunc(slice, func(a, b *node) int { + return enode.DistCmp(distbase, a.ID(), b.ID()) }) } diff --git a/p2p/discover/v4_lookup_test.go b/p2p/discover/v4_lookup_test.go index 83c462628867..1f9ad69d0a23 100644 --- a/p2p/discover/v4_lookup_test.go +++ b/p2p/discover/v4_lookup_test.go @@ -302,8 +302,8 @@ func (tn *preminedTestnet) closest(n int) (nodes []*enode.Node) { nodes = append(nodes, tn.node(d, i)) } } - slices.SortFunc(nodes, func(a, b *enode.Node) bool { - return enode.DistCmp(tn.target.id(), a.ID(), b.ID()) < 0 + slices.SortFunc(nodes, func(a, b *enode.Node) int { + return enode.DistCmp(tn.target.id(), a.ID(), b.ID()) }) return nodes[:n] } diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 887d46c4975a..880b71a991ce 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -61,8 +61,8 @@ func TestUDPv5_lookupE2E(t *testing.T) { for i := range nodes { expectedResult[i] = nodes[i].Self() } - slices.SortFunc(expectedResult, func(a, b *enode.Node) bool { - return enode.DistCmp(target.ID(), a.ID(), b.ID()) < 0 + slices.SortFunc(expectedResult, func(a, b *enode.Node) int { + return enode.DistCmp(target.ID(), a.ID(), b.ID()) }) // Do the lookup. diff --git a/p2p/dnsdisc/tree.go b/p2p/dnsdisc/tree.go index f91aff458fb0..06b7681f18ae 100644 --- a/p2p/dnsdisc/tree.go +++ b/p2p/dnsdisc/tree.go @@ -214,8 +214,8 @@ func (t *Tree) build(entries []entry) entry { } func sortByID(nodes []*enode.Node) []*enode.Node { - slices.SortFunc(nodes, func(a, b *enode.Node) bool { - return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) < 0 + slices.SortFunc(nodes, func(a, b *enode.Node) int { + return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) }) return nodes } diff --git a/p2p/peer.go b/p2p/peer.go index e800be80fc6e..65a7903f5863 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -386,7 +386,7 @@ func countMatchingProtocols(protocols []Protocol, caps []Cap) int { // matchProtocols creates structures for matching named subprotocols. func matchProtocols(protocols []Protocol, caps []Cap, rw MsgReadWriter) map[string]*protoRW { - slices.SortFunc(caps, Cap.Less) + slices.SortFunc(caps, Cap.Cmp) offset := baseProtocolLength result := make(map[string]*protoRW) diff --git a/p2p/protocol.go b/p2p/protocol.go index 4b3cca6d4a09..9bb6785a2250 100644 --- a/p2p/protocol.go +++ b/p2p/protocol.go @@ -18,6 +18,7 @@ package p2p import ( "fmt" + "strings" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" @@ -77,10 +78,16 @@ func (cap Cap) String() string { return fmt.Sprintf("%s/%d", cap.Name, cap.Version) } -// Less defines the canonical sorting order of capabilities. -func (cap Cap) Less(other Cap) bool { +// Cmp defines the canonical sorting order of capabilities. +func (cap Cap) Cmp(other Cap) int { if cap.Name == other.Name { - return cap.Version < other.Version + if cap.Version < other.Version { + return -1 + } + if cap.Version > other.Version { + return 1 + } + return 0 } - return cap.Name < other.Name + return strings.Compare(cap.Name, other.Name) } diff --git a/p2p/server.go b/p2p/server.go index d4e2be67830e..8f42765a8c26 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -510,7 +510,7 @@ func (srv *Server) setupLocalNode() error { for _, p := range srv.Protocols { srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap()) } - slices.SortFunc(srv.ourHandshake.Caps, Cap.Less) + slices.SortFunc(srv.ourHandshake.Caps, Cap.Cmp) // Create the local node. db, err := enode.OpenDB(srv.NodeDatabase) diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index d5aee206da66..aa81e5c9d9b1 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -98,8 +98,8 @@ func (f *fuzzer) fuzz() int { if len(entries) <= 1 { return 0 } - slices.SortFunc(entries, func(a, b *kv) bool { - return bytes.Compare(a.k, b.k) < 0 + slices.SortFunc(entries, func(a, b *kv) int { + return bytes.Compare(a.k, b.k) }) var ok = 0 diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 1eb639f6441a..391bdf300b72 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -182,8 +182,8 @@ func (f *fuzzer) fuzz() int { dbA.Commit(rootA, false) // Stacktrie requires sorted insertion - slices.SortFunc(vals, func(a, b kv) bool { - return bytes.Compare(a.k, b.k) < 0 + slices.SortFunc(vals, func(a, b kv) int { + return bytes.Compare(a.k, b.k) }) for _, kv := range vals { if f.debugging { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index bf20c00c6436..e711ffb81583 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -84,8 +84,8 @@ type kv struct { t bool } -func (k *kv) less(other *kv) bool { - return bytes.Compare(k.k, other.k) < 0 +func (k *kv) cmp(other *kv) int { + return bytes.Compare(k.k, other.k) } func TestIteratorLargeData(t *testing.T) { diff --git a/trie/proof_test.go b/trie/proof_test.go index d8effd0d04b2..e8ea116c8803 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -173,7 +173,7 @@ func TestRangeProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) end := mrand.Intn(len(entries)-start) + start + 1 @@ -206,7 +206,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) end := mrand.Intn(len(entries)-start) + start + 1 @@ -278,7 +278,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Case 1 start, end := 100, 200 @@ -335,7 +335,7 @@ func TestOneElementRangeProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // One element with existent edge proof, both edge proofs // point to the SAME key. @@ -422,7 +422,7 @@ func TestAllElementsProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var k [][]byte var v [][]byte @@ -474,7 +474,7 @@ func TestSingleSideRangeProof(t *testing.T) { trie.MustUpdate(value.k, value.v) entries = append(entries, value) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} for _, pos := range cases { @@ -509,7 +509,7 @@ func TestReverseSingleSideRangeProof(t *testing.T) { trie.MustUpdate(value.k, value.v) entries = append(entries, value) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} for _, pos := range cases { @@ -543,7 +543,7 @@ func TestBadRangeProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) @@ -646,7 +646,7 @@ func TestSameSideProofs(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) pos := 1000 first := decreaseKey(common.CopyBytes(entries[pos].k)) @@ -690,7 +690,7 @@ func TestHasRightElement(t *testing.T) { trie.MustUpdate(value.k, value.v) entries = append(entries, value) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var cases = []struct { start int @@ -762,7 +762,7 @@ func TestEmptyRangeProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var cases = []struct { pos int @@ -797,7 +797,7 @@ func TestBloatedProof(t *testing.T) { for _, kv := range kvs { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var keys [][]byte var vals [][]byte @@ -831,7 +831,7 @@ func TestEmptyValueRangeProof(t *testing.T) { for _, kv := range values { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Create a new entry with a slightly modified key mid := len(entries) / 2 @@ -875,7 +875,7 @@ func TestAllElementsEmptyValueRangeProof(t *testing.T) { for _, kv := range values { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Create a new entry with a slightly modified key mid := len(entries) / 2 @@ -981,7 +981,7 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) start := 2 end := start + size @@ -1018,7 +1018,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var keys [][]byte var values [][]byte diff --git a/trie/triedb/pathdb/history.go b/trie/triedb/pathdb/history.go index c12fc5518472..ce82532507c7 100644 --- a/trie/triedb/pathdb/history.go +++ b/trie/triedb/pathdb/history.go @@ -262,20 +262,20 @@ func newHistory(root common.Hash, parent common.Hash, block uint64, states *trie for addr := range states.Accounts { accountList = append(accountList, addr) } - slices.SortFunc(accountList, func(a, b common.Address) bool { return a.Less(b) }) + slices.SortFunc(accountList, common.Address.Cmp) for addr, slots := range states.Storages { slist := make([]common.Hash, 0, len(slots)) for slotHash := range slots { slist = append(slist, slotHash) } - slices.SortFunc(slist, func(a, b common.Hash) bool { return a.Less(b) }) + slices.SortFunc(slist, common.Hash.Cmp) storageList[addr] = slist } for addr := range states.Incomplete { incomplete = append(incomplete, addr) } - slices.SortFunc(incomplete, func(a, b common.Address) bool { return a.Less(b) }) + slices.SortFunc(incomplete, common.Address.Cmp) return &history{ meta: &meta{ diff --git a/trie/triedb/pathdb/testutils.go b/trie/triedb/pathdb/testutils.go index cb3a240cc4f3..4406dbc52124 100644 --- a/trie/triedb/pathdb/testutils.go +++ b/trie/triedb/pathdb/testutils.go @@ -117,7 +117,7 @@ func hash(states map[common.Hash][]byte) (common.Hash, []byte) { for hash := range states { hs = append(hs, hash) } - slices.SortFunc(hs, func(a, b common.Hash) bool { return a.Less(b) }) + slices.SortFunc(hs, common.Hash.Cmp) var input []byte for _, hash := range hs { diff --git a/trie/trienode/node.go b/trie/trienode/node.go index 8998bcba060a..98d5588b6d3a 100644 --- a/trie/trienode/node.go +++ b/trie/trienode/node.go @@ -18,10 +18,10 @@ package trienode import ( "fmt" + "sort" "strings" "github.com/ethereum/go-ethereum/common" - "golang.org/x/exp/slices" ) // Node is a wrapper which contains the encoded blob of the trie node and its @@ -83,9 +83,7 @@ func (set *NodeSet) ForEachWithOrder(callback func(path string, n *Node)) { paths = append(paths, path) } // Bottom-up, the longest path first - slices.SortFunc(paths, func(a, b string) bool { - return a > b // Sort in reverse order - }) + sort.Sort(sort.Reverse(sort.StringSlice(paths))) for _, path := range paths { callback(path, set.Nodes[path]) } From f298ec4fe14bfaa99df76d64c76f7cd988519393 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 12 Aug 2023 00:21:46 +0200 Subject: [PATCH 22/25] go.mod, build: upgrade c-kzg-4844 (#27907) This upgrades to the latest release of ckzg, and also attempts to fix some blst-related build errors that occur on launchpad.net. --- build/ci.go | 19 ++++++++++++++----- build/deb/ethereum/deb.rules | 2 +- go.mod | 2 +- go.sum | 4 ++-- internal/build/env.go | 14 ++++++++------ internal/build/gotool.go | 4 ++-- 6 files changed, 28 insertions(+), 17 deletions(-) diff --git a/build/ci.go b/build/ci.go index 78b8bc69726b..242f4d841e5f 100644 --- a/build/ci.go +++ b/build/ci.go @@ -200,6 +200,7 @@ func doInstall(cmdline []string) { staticlink = flag.Bool("static", false, "Create statically-linked executable") ) flag.CommandLine.Parse(cmdline) + env := build.Env() // Configure the toolchain. tc := build.GoToolchain{GOARCH: *arch, CC: *cc} @@ -207,12 +208,16 @@ func doInstall(cmdline []string) { csdb := build.MustLoadChecksums("build/checksums.txt") tc.Root = build.DownloadGo(csdb, dlgoVersion) } - // Disable CLI markdown doc generation in release builds and enable linking - // the CKZG library since we can make it portable here. - buildTags := []string{"urfave_cli_no_docs", "ckzg"} + + // Disable CLI markdown doc generation in release builds. + buildTags := []string{"urfave_cli_no_docs"} + + // Enable linking the CKZG library since we can make it work with additional flags. + if env.UbuntuVersion != "trusty" { + buildTags = append(buildTags, "ckzg") + } // Configure the build. - env := build.Env() gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...) // arm64 CI builders are memory-constrained and can't handle concurrent builds, @@ -298,10 +303,14 @@ func doTest(cmdline []string) { csdb := build.MustLoadChecksums("build/checksums.txt") tc.Root = build.DownloadGo(csdb, dlgoVersion) } - gotest := tc.Go("test", "-tags=ckzg") + gotest := tc.Go("test") + // CI needs a bit more time for the statetests (default 10m). gotest.Args = append(gotest.Args, "-timeout=20m") + // Enable CKZG backend in CI. + gotest.Args = append(gotest.Args, "-tags=ckzg") + // Test a single package at a time. CI builders are slow // and some tests run into timeouts under load. gotest.Args = append(gotest.Args, "-p", "1") diff --git a/build/deb/ethereum/deb.rules b/build/deb/ethereum/deb.rules index 475e628b2f5e..daca793e555f 100644 --- a/build/deb/ethereum/deb.rules +++ b/build/deb/ethereum/deb.rules @@ -28,7 +28,7 @@ override_dh_auto_build: mv .mod $(GOPATH)/pkg/mod # A fresh Go was built, all dependency downloads faked, hope build works now - ../.go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} + ../.go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} -ubuntu {{.Distro}} override_dh_auto_test: diff --git a/go.mod b/go.mod index e4273ef5b034..d62f4ebfd5b0 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/deckarep/golang-set/v2 v2.1.0 github.com/docker/docker v1.6.2 github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 - github.com/ethereum/c-kzg-4844 v0.3.0 + github.com/ethereum/c-kzg-4844 v0.3.1 github.com/fatih/color v1.7.0 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 diff --git a/go.sum b/go.sum index 50271e857fdf..8f91afee82bf 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/ethereum/c-kzg-4844 v0.3.0 h1:3Y3hD6l5i0dEYsBL50C+Om644kve3pNqoAcvE26o9zI= -github.com/ethereum/c-kzg-4844 v0.3.0/go.mod h1:WI2Nd82DMZAAZI1wV2neKGost9EKjvbpQR9OqE5Qqa8= +github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= +github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= diff --git a/internal/build/env.go b/internal/build/env.go index d70c0d50a499..9cfcbb69b343 100644 --- a/internal/build/env.go +++ b/internal/build/env.go @@ -28,12 +28,13 @@ import ( var ( // These flags override values in build env. - GitCommitFlag = flag.String("git-commit", "", `Overrides git commit hash embedded into executables`) - GitBranchFlag = flag.String("git-branch", "", `Overrides git branch being built`) - GitTagFlag = flag.String("git-tag", "", `Overrides git tag being built`) - BuildnumFlag = flag.String("buildnum", "", `Overrides CI build number`) - PullRequestFlag = flag.Bool("pull-request", false, `Overrides pull request status of the build`) - CronJobFlag = flag.Bool("cron-job", false, `Overrides cron job status of the build`) + GitCommitFlag = flag.String("git-commit", "", `Overrides git commit hash embedded into executables`) + GitBranchFlag = flag.String("git-branch", "", `Overrides git branch being built`) + GitTagFlag = flag.String("git-tag", "", `Overrides git tag being built`) + BuildnumFlag = flag.String("buildnum", "", `Overrides CI build number`) + PullRequestFlag = flag.Bool("pull-request", false, `Overrides pull request status of the build`) + CronJobFlag = flag.Bool("cron-job", false, `Overrides cron job status of the build`) + UbuntuVersionFlag = flag.String("ubuntu", "", `Sets the ubuntu version being built for`) ) // Environment contains metadata provided by the build environment. @@ -43,6 +44,7 @@ type Environment struct { Repo string // name of GitHub repo Commit, Date, Branch, Tag string // Git info Buildnum string + UbuntuVersion string // Ubuntu version being built for IsPullRequest bool IsCronJob bool } diff --git a/internal/build/gotool.go b/internal/build/gotool.go index b5452f80cc3c..296ba8c36ecb 100644 --- a/internal/build/gotool.go +++ b/internal/build/gotool.go @@ -54,8 +54,8 @@ func (g *GoToolchain) Go(command string, args ...string) *exec.Cmd { tool.Env = append(tool.Env, "CC="+os.Getenv("CC")) } // CKZG by default is not portable, append the necessary build flags to make - // it not rely on modern CPU instructions and enable linking against - tool.Env = append(tool.Env, "CGO_CFLAGS=-D__BLST_PORTABLE__") + // it not rely on modern CPU instructions and enable linking against. + tool.Env = append(tool.Env, "CGO_CFLAGS=-O2 -g -D__BLST_PORTABLE__") return tool } From a867039c53790bca68c44911c8a0d86dfdc503b5 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 12 Aug 2023 10:32:50 +0200 Subject: [PATCH 23/25] internal/build: apply -ubuntu to env (#27910) --- internal/build/env.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/build/env.go b/internal/build/env.go index 9cfcbb69b343..0854fba2498d 100644 --- a/internal/build/env.go +++ b/internal/build/env.go @@ -170,5 +170,8 @@ func applyEnvFlags(env Environment) Environment { if *CronJobFlag { env.IsCronJob = true } + if *UbuntuVersionFlag != "" { + env.UbuntuVersion = *UbuntuVersionFlag + } return env } From bed84606583893fdb698cc1b5058cc47b4dbd837 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 12 Aug 2023 11:05:07 +0200 Subject: [PATCH 24/25] params: release go-ethereum v1.12.2 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 4d70038e4c8f..7417dc0be2f6 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 12 // Minor version component of the current release - VersionPatch = 2 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 12 // Minor version component of the current release + VersionPatch = 2 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 916d6a441a866cb618ae826c220866de118899f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 14 Nov 2023 15:02:24 +0300 Subject: [PATCH 25/25] params: release Geth v1.15.5 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index bcab461a4358..5fb9631f1b3b 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 5 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 13 // Minor version component of the current release + VersionPatch = 5 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string.