From 6e33dbf96a06cb3ad3bdbd5d0539e54de254ce6b Mon Sep 17 00:00:00 2001 From: Marius G <90795310+bearpebble@users.noreply.github.com> Date: Tue, 30 Jul 2024 15:31:27 +0200 Subject: [PATCH 01/39] p2p: fix flaky test TestServerPortMapping (#30241) The test specifies `ListenAddr: ":0"`, which means a random ephemeral port will be chosen for the TCP listener by the OS. Additionally, since no `DiscAddr` was specified, the same port that is chosen automatically by the OS will also be used for the UDP listener in the discovery UDP setup. This sometimes leads to test failures if the TCP listener picks a free TCP port that is already taken for UDP. By specifying `DiscAddr: ":0"`, the UDP port will be chosen independently from the TCP port, fixing the random failure. See issue #29830. Verified using ``` cd p2p go test -c -race stress ./p2p.test -test.run=TestServerPortMapping ... 5m0s: 4556 runs so far, 0 failures ``` The issue described above can technically lead to sporadic failures on systems that specify a listen address via the `--port` flag of 0 while not setting `--discovery.port`. Since the default is using port `30303` and using a random ephemeral port is likely not used much to begin with, not addressing the root cause might be acceptable. --- p2p/server_nat_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/server_nat_test.go b/p2p/server_nat_test.go index cbb1f37e0a..7e19387210 100644 --- a/p2p/server_nat_test.go +++ b/p2p/server_nat_test.go @@ -36,6 +36,7 @@ func TestServerPortMapping(t *testing.T) { PrivateKey: newkey(), NoDial: true, ListenAddr: ":0", + DiscAddr: ":0", NAT: mockNAT, Logger: testlog.Logger(t, log.LvlTrace), clock: clock, From de6d5976794a9ed3b626d4eba57bf7f0806fb970 Mon Sep 17 00:00:00 2001 From: Daniel Knopik <107140945+dknopik@users.noreply.github.com> Date: Wed, 31 Jul 2024 21:38:23 +0200 Subject: [PATCH 02/39] p2p/discover: schedule revalidation also when all nodes are excluded (#30239) ## Issue If `nextTime` has passed, but all nodes are excluded, `get` would return `nil` and `run` would therefore not invoke `schedule`. Then, we schedule a timer for the past, as neither `nextTime` value has been updated. This creates a busy loop, as the timer immediately returns. ## Fix With this PR, revalidation will be also rescheduled when all nodes are excluded. --------- Co-authored-by: lightclient --- p2p/discover/table_reval.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/p2p/discover/table_reval.go b/p2p/discover/table_reval.go index f2ea8b34fa..2465fee906 100644 --- a/p2p/discover/table_reval.go +++ b/p2p/discover/table_reval.go @@ -77,14 +77,18 @@ func (tr *tableRevalidation) nodeEndpointChanged(tab *Table, n *tableNode) { // It returns the next time it should be invoked, which is used in the Table main loop // to schedule a timer. However, run can be called at any time. func (tr *tableRevalidation) run(tab *Table, now mclock.AbsTime) (nextTime mclock.AbsTime) { - if n := tr.fast.get(now, &tab.rand, tr.activeReq); n != nil { - tr.startRequest(tab, n) - tr.fast.schedule(now, &tab.rand) - } - if n := tr.slow.get(now, &tab.rand, tr.activeReq); n != nil { - tr.startRequest(tab, n) - tr.slow.schedule(now, &tab.rand) + reval := func(list *revalidationList) { + if list.nextTime <= now { + if n := list.get(now, &tab.rand, tr.activeReq); n != nil { + tr.startRequest(tab, n) + } + // Update nextTime regardless if any requests were started because + // current value has passed. + list.schedule(now, &tab.rand) + } } + reval(&tr.fast) + reval(&tr.slow) return min(tr.fast.nextTime, tr.slow.nextTime) } @@ -200,7 +204,7 @@ type revalidationList struct { // get returns a random node from the queue. Nodes in the 'exclude' map are not returned. func (list *revalidationList) get(now mclock.AbsTime, rand randomSource, exclude map[enode.ID]struct{}) *tableNode { - if now < list.nextTime || len(list.nodes) == 0 { + if len(list.nodes) == 0 { return nil } for i := 0; i < len(list.nodes)*3; i++ { From ff6e43e8c4db9f6d8afa7887ae9d2081c2c890d0 Mon Sep 17 00:00:00 2001 From: Darioush Jalali Date: Thu, 1 Aug 2024 07:06:43 -0700 Subject: [PATCH 03/39] miner: remove outdated comment (#30248) --- miner/worker.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 5dc3e2056b..9aae6e1609 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -205,8 +205,7 @@ func (miner *Miner) prepareWork(genParams *generateParams) (*environment, error) // makeEnv creates a new environment for the sealing block. func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase common.Address) (*environment, error) { - // Retrieve the parent state to execute on top and start a prefetcher for - // the miner to speed block sealing up a bit. + // Retrieve the parent state to execute on top. state, err := miner.chain.StateAt(parent.Root) if err != nil { return nil, err From dad8f237ffa5da2c2471f2d9f32d2bc5b580f667 Mon Sep 17 00:00:00 2001 From: Seungmin Kim Date: Thu, 1 Aug 2024 23:10:43 +0900 Subject: [PATCH 04/39] eth/downloader: correct sync mode logging to show old mode (#30219) This PR fixes an issue in the setMode method of beaconBackfiller where the log message was not displaying the previous mode correctly. The log message now shows both the old and new sync modes. --- eth/downloader/beaconsync.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index 57c6eee40a..5ef3f91dc7 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -123,7 +123,8 @@ func (b *beaconBackfiller) resume() { func (b *beaconBackfiller) setMode(mode SyncMode) { // Update the old sync mode and track if it was changed b.lock.Lock() - updated := b.syncMode != mode + oldMode := b.syncMode + updated := oldMode != mode filling := b.filling b.syncMode = mode b.lock.Unlock() @@ -133,8 +134,8 @@ func (b *beaconBackfiller) setMode(mode SyncMode) { if !updated || !filling { return } - log.Error("Downloader sync mode changed mid-run", "old", mode.String(), "new", mode.String()) - b.suspend() + log.Error("Downloader sync mode changed mid-run", "old", oldMode.String(), "new", mode.String()) + b.suspend() b.resume() } From b635089c7c7b4d9ae840e2052cf9f086109242ee Mon Sep 17 00:00:00 2001 From: Icarus Wu Date: Thu, 1 Aug 2024 22:25:55 +0800 Subject: [PATCH 05/39] all: remove deprecated protobuf dependencies (#30232) The package `github.com/golang/protobuf/proto` is deprecated in favor `google.golang.org/protobuf/proto`. We should update the codes to recommended package. Signed-off-by: Icarus Wu --- .golangci.yml | 4 ---- Makefile | 2 +- accounts/usbwallet/trezor.go | 2 +- accounts/usbwallet/trezor/trezor.go | 6 +++--- build/tools/tools.go | 2 +- go.mod | 4 ++-- go.sum | 2 ++ 7 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 2132f5403a..75452472d0 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -64,10 +64,6 @@ issues: text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.' - path: core/vm/contracts.go text: 'SA1019: "golang.org/x/crypto/ripemd160" is deprecated: RIPEMD-160 is a legacy hash and should not be used for new applications.' - - path: accounts/usbwallet/trezor.go - text: 'SA1019: "github.com/golang/protobuf/proto" is deprecated: Use the "google.golang.org/protobuf/proto" package instead.' - - path: accounts/usbwallet/trezor/ - text: 'SA1019: "github.com/golang/protobuf/proto" is deprecated: Use the "google.golang.org/protobuf/proto" package instead.' exclude: - 'SA1019: event.TypeMux is deprecated: use Feed' - 'SA1019: strings.Title is deprecated' diff --git a/Makefile b/Makefile index 857cb8c978..f4932165a4 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ clean: devtools: env GOBIN= go install golang.org/x/tools/cmd/stringer@latest env GOBIN= go install github.com/fjl/gencodec@latest - env GOBIN= go install github.com/golang/protobuf/protoc-gen-go@latest + env GOBIN= go install google.golang.org/protobuf/cmd/protoc-gen-go@latest env GOBIN= go install ./cmd/abigen @type "solc" 2> /dev/null || echo 'Please install solc' @type "protoc" 2> /dev/null || echo 'Please install protoc' diff --git a/accounts/usbwallet/trezor.go b/accounts/usbwallet/trezor.go index 9644dc4e02..1c4270d255 100644 --- a/accounts/usbwallet/trezor.go +++ b/accounts/usbwallet/trezor.go @@ -33,7 +33,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" - "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/proto" ) // ErrTrezorPINNeeded is returned if opening the trezor requires a PIN code. In diff --git a/accounts/usbwallet/trezor/trezor.go b/accounts/usbwallet/trezor/trezor.go index 4324258314..93aee3c289 100644 --- a/accounts/usbwallet/trezor/trezor.go +++ b/accounts/usbwallet/trezor/trezor.go @@ -39,8 +39,8 @@ // - Download the latest protoc https://github.com/protocolbuffers/protobuf/releases // - Build with the usual `./configure && make` and ensure it's on your $PATH // - Delete all the .proto and .pb.go files, pull in fresh ones from Trezor -// - Grab the latest Go plugin `go get -u github.com/golang/protobuf/protoc-gen-go` -// - Vendor in the latest Go plugin `govendor fetch github.com/golang/protobuf/...` +// - Grab the latest Go plugin `go get -u google.golang.org/protobuf/cmd/protoc-gen-go` +// - Vendor in the latest Go plugin `govendor fetch google.golang.org/protobuf/...` //go:generate protoc -I/usr/local/include:. --go_out=paths=source_relative:. messages.proto messages-common.proto messages-management.proto messages-ethereum.proto @@ -50,7 +50,7 @@ package trezor import ( "reflect" - "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/proto" ) // Type returns the protocol buffer type number of a specific message. If the diff --git a/build/tools/tools.go b/build/tools/tools.go index 506e26eeff..e9e2241d2f 100644 --- a/build/tools/tools.go +++ b/build/tools/tools.go @@ -22,6 +22,6 @@ package tools import ( // Tool imports for go:generate. _ "github.com/fjl/gencodec" - _ "github.com/golang/protobuf/protoc-gen-go" _ "golang.org/x/tools/cmd/stringer" + _ "google.golang.org/protobuf/cmd/protoc-gen-go" ) diff --git a/go.mod b/go.mod index db5c713498..5a0314d635 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,6 @@ require ( github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff github.com/gofrs/flock v0.8.1 github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/golang/protobuf v1.5.4 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb github.com/google/gofuzz v1.2.0 github.com/google/uuid v1.3.0 @@ -74,7 +73,7 @@ require ( golang.org/x/text v0.14.0 golang.org/x/time v0.5.0 golang.org/x/tools v0.20.0 - google.golang.org/protobuf v1.33.0 + google.golang.org/protobuf v1.34.2 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -112,6 +111,7 @@ require ( github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/go.sum b/go.sum index e3cf5f3d63..465e98f337 100644 --- a/go.sum +++ b/go.sum @@ -842,6 +842,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 67b81371008d147aa2ccebd3fa9655fe042b0f14 Mon Sep 17 00:00:00 2001 From: ysh0566 Date: Fri, 2 Aug 2024 02:09:04 +0800 Subject: [PATCH 06/39] accounts/abi/bind: add accessList support to base bond contract (#30195) Adding the correct accessList parameter when calling a contract can reduce gas consumption. However, the current version only allows adding the accessList manually when constructing the transaction. This PR can provide convenience for saving gas. --- accounts/abi/bind/base.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/accounts/abi/bind/base.go b/accounts/abi/bind/base.go index c8972a9dff..0504089c71 100644 --- a/accounts/abi/bind/base.go +++ b/accounts/abi/bind/base.go @@ -59,11 +59,12 @@ type TransactOpts struct { Nonce *big.Int // Nonce to use for the transaction execution (nil = use pending state) Signer SignerFn // Method to use for signing the transaction (mandatory) - Value *big.Int // Funds to transfer along the transaction (nil = 0 = no funds) - GasPrice *big.Int // Gas price to use for the transaction execution (nil = gas price oracle) - GasFeeCap *big.Int // Gas fee cap to use for the 1559 transaction execution (nil = gas price oracle) - GasTipCap *big.Int // Gas priority fee cap to use for the 1559 transaction execution (nil = gas price oracle) - GasLimit uint64 // Gas limit to set for the transaction execution (0 = estimate) + Value *big.Int // Funds to transfer along the transaction (nil = 0 = no funds) + GasPrice *big.Int // Gas price to use for the transaction execution (nil = gas price oracle) + GasFeeCap *big.Int // Gas fee cap to use for the 1559 transaction execution (nil = gas price oracle) + GasTipCap *big.Int // Gas priority fee cap to use for the 1559 transaction execution (nil = gas price oracle) + GasLimit uint64 // Gas limit to set for the transaction execution (0 = estimate) + AccessList types.AccessList // Access list to set for the transaction execution (nil = no access list) Context context.Context // Network context to support cancellation and timeouts (nil = no timeout) @@ -300,20 +301,21 @@ func (c *BoundContract) createDynamicTx(opts *TransactOpts, contract *common.Add return nil, err } baseTx := &types.DynamicFeeTx{ - To: contract, - Nonce: nonce, - GasFeeCap: gasFeeCap, - GasTipCap: gasTipCap, - Gas: gasLimit, - Value: value, - Data: input, + To: contract, + Nonce: nonce, + GasFeeCap: gasFeeCap, + GasTipCap: gasTipCap, + Gas: gasLimit, + Value: value, + Data: input, + AccessList: opts.AccessList, } return types.NewTx(baseTx), nil } func (c *BoundContract) createLegacyTx(opts *TransactOpts, contract *common.Address, input []byte) (*types.Transaction, error) { - if opts.GasFeeCap != nil || opts.GasTipCap != nil { - return nil, errors.New("maxFeePerGas or maxPriorityFeePerGas specified but london is not active yet") + if opts.GasFeeCap != nil || opts.GasTipCap != nil || opts.AccessList != nil { + return nil, errors.New("maxFeePerGas or maxPriorityFeePerGas or accessList specified but london is not active yet") } // Normalize value value := opts.Value From e4675771eda550e7eeb63a8884816982c1980644 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 2 Aug 2024 06:44:03 +0200 Subject: [PATCH 07/39] internal/debug: remove memsize (#30253) Removing because memsize will very likely be broken by Go 1.23. See https://github.com/fjl/memsize/issues/4 --- cmd/geth/main.go | 2 -- go.mod | 1 - go.sum | 2 -- internal/debug/flags.go | 4 ---- 4 files changed, 9 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index f6bb09ee54..a37bb5f987 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -360,8 +360,6 @@ func geth(ctx *cli.Context) error { // it unlocks any requested accounts, and starts the RPC/IPC interfaces and the // miner. func startNode(ctx *cli.Context, stack *node.Node, isConsole bool) { - debug.Memsize.Add("node", stack) - // Start up the node itself utils.StartNode(ctx, stack, isConsole) diff --git a/go.mod b/go.mod index 5a0314d635..3e9837186d 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/fatih/color v1.16.0 github.com/ferranbt/fastssz v0.1.2 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e - github.com/fjl/memsize v0.0.2 github.com/fsnotify/fsnotify v1.6.0 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff github.com/gofrs/flock v0.8.1 diff --git a/go.sum b/go.sum index 465e98f337..d8e7a540c7 100644 --- a/go.sum +++ b/go.sum @@ -178,8 +178,6 @@ github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNP github.com/ferranbt/fastssz v0.1.2/go.mod h1:X5UPrE2u1UJjxHA8X54u04SBwdAQjG2sFtWs39YxyWs= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= -github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= -github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 19222c8325..0e05975c7e 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -31,15 +31,12 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/metrics/exp" - "github.com/fjl/memsize/memsizeui" "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" "github.com/urfave/cli/v2" "gopkg.in/natefinch/lumberjack.v2" ) -var Memsize memsizeui.Handler - var ( verbosityFlag = &cli.IntFlag{ Name: "verbosity", @@ -313,7 +310,6 @@ func StartPProf(address string, withMetrics bool) { if withMetrics { exp.Exp(metrics.DefaultRegistry) } - http.Handle("/memsize/", http.StripPrefix("/memsize", &Memsize)) log.Info("Starting pprof server", "addr", fmt.Sprintf("http://%s/debug/pprof", address)) go func() { if err := http.ListenAndServe(address, nil); err != nil { From 16cf5c5fed9f8f22ef31fcdb0057aa4ad4ea3a12 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 2 Aug 2024 17:36:28 +0200 Subject: [PATCH 08/39] eth/downloader: gofmt (#30261) Fixes a regression introduced in https://github.com/ethereum/go-ethereum/pull/30219 --- eth/downloader/beaconsync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index 5ef3f91dc7..e682536e07 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -135,7 +135,7 @@ func (b *beaconBackfiller) setMode(mode SyncMode) { return } log.Error("Downloader sync mode changed mid-run", "old", oldMode.String(), "new", mode.String()) - b.suspend() + b.suspend() b.resume() } From 142c94d62842c7801e8f4d71f080ae156ecd1f2b Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Fri, 2 Aug 2024 09:49:01 -0600 Subject: [PATCH 09/39] cmd/evm: don't overwrite sender account (#30259) Fixes #30254 It seems like the removed CreateAccount call is very old and not needed anymore. After removing it, setting a sender that does not exist in the state doesn't seem to cause an issue. --- cmd/evm/runner.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index f179e733e6..d06f85ed59 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -162,7 +162,6 @@ func runCmd(ctx *cli.Context) error { if ctx.String(SenderFlag.Name) != "" { sender = common.HexToAddress(ctx.String(SenderFlag.Name)) } - statedb.CreateAccount(sender) if ctx.String(ReceiverFlag.Name) != "" { receiver = common.HexToAddress(ctx.String(ReceiverFlag.Name)) From 10586952dfad6d57ed0b24500f572f7889c8fa97 Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 6 Aug 2024 01:14:22 +0800 Subject: [PATCH 10/39] eth/catalyst: get params.ExcessBlobGas but check with params.BlobGasUsed (#30267) Seems it is checked with the wrong argument Signed-off-by: jsvisa --- eth/catalyst/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 00cce259c8..5a001e1ee8 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -546,7 +546,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe bgu = strconv.Itoa(int(*params.BlobGasUsed)) } ebg := "nil" - if params.BlobGasUsed != nil { + if params.ExcessBlobGas != nil { ebg = strconv.Itoa(int(*params.ExcessBlobGas)) } log.Warn("Invalid NewPayload params", From cf8aa31e3edc6fc652963c5d5dee869cd69f077b Mon Sep 17 00:00:00 2001 From: stevemilk Date: Tue, 6 Aug 2024 19:49:48 +0800 Subject: [PATCH 11/39] params: remove unused les parameters (#30268) --- params/network_params.go | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/params/network_params.go b/params/network_params.go index 9311b5e2d5..61bd6b2f42 100644 --- a/params/network_params.go +++ b/params/network_params.go @@ -24,44 +24,13 @@ const ( // contains on the server side. BloomBitsBlocks uint64 = 4096 - // BloomBitsBlocksClient is the number of blocks a single bloom bit section vector - // contains on the light client side - BloomBitsBlocksClient uint64 = 32768 - // BloomConfirms is the number of confirmation blocks before a bloom section is // considered probably final and its rotated bits are calculated. BloomConfirms = 256 - // CHTFrequency is the block frequency for creating CHTs - CHTFrequency = 32768 - - // BloomTrieFrequency is the block frequency for creating BloomTrie on both - // server/client sides. - BloomTrieFrequency = 32768 - - // HelperTrieConfirmations is the number of confirmations before a client is expected - // to have the given HelperTrie available. - HelperTrieConfirmations = 2048 - - // HelperTrieProcessConfirmations is the number of confirmations before a HelperTrie - // is generated - HelperTrieProcessConfirmations = 256 - - // CheckpointFrequency is the block frequency for creating checkpoint - CheckpointFrequency = 32768 - - // CheckpointProcessConfirmations is the number before a checkpoint is generated - CheckpointProcessConfirmations = 256 - // FullImmutabilityThreshold is the number of blocks after which a chain segment is // considered immutable (i.e. soft finality). It is used by the downloader as a // hard limit against deep ancestors, by the blockchain against deep reorgs, by // the freezer as the cutoff threshold and by clique as the snapshot trust limit. FullImmutabilityThreshold = 90000 - - // LightImmutabilityThreshold is the number of blocks after which a header chain - // segment is considered immutable for light client(i.e. soft finality). It is used by - // the downloader as a hard limit against deep ancestors, by the blockchain against deep - // reorgs, by the light pruner as the pruning validity guarantee. - LightImmutabilityThreshold = 30000 ) From dbc1d04f5e4a6e466e32db08ed03678624a93307 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 6 Aug 2024 05:51:48 -0600 Subject: [PATCH 12/39] core/vm/runtime: ensure tracer benchmark calls `OnTxStart` (#30257) The struct-based tracing added in #29189 seems to have caused an issue with the benchmark `BenchmarkTracerStepVsCallFrame`. On master we see the following panic: ```console BenchmarkTracerStepVsCallFrame panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x2 addr=0x40 pc=0x1019782f0] goroutine 37 [running]: github.com/ethereum/go-ethereum/eth/tracers/js.(*jsTracer).OnOpcode(0x140004c4000, 0x0, 0x10?, 0x989680, 0x1, {0x101ea2298, 0x1400000e258}, {0x1400000e258?, 0x14000155928?, 0x10173020c?}, ...) /Users/matt/dev/go-ethereum/eth/tracers/js/goja.go:328 +0x140 github.com/ethereum/go-ethereum/core/vm.(*EVMInterpreter).Run(0x14000307da0, 0x140003cc0d0, {0x0, 0x0, 0x0}, 0x0) ... FAIL github.com/ethereum/go-ethereum/core/vm/runtime 0.420s FAIL ``` The issue seems to be that `OnOpcode` expects that `OnTxStart` has already been called to initialize the `env` value in the tracer. The JS tracer uses it in `OnOpcode` for the `GetRefund()` method. This patch resolves the issue by reusing the `Call` method already defined in `runtime_test.go` which correctly calls `OnTxStart`. --- core/vm/runtime/runtime_test.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 04abc5480e..a49eca25d3 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -38,7 +38,6 @@ import ( // force-load js tracers to trigger registration _ "github.com/ethereum/go-ethereum/eth/tracers/js" - "github.com/holiman/uint256" ) func TestDefaults(t *testing.T) { @@ -339,11 +338,7 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode Tracer: tracer.Hooks, } } - var ( - destination = common.BytesToAddress([]byte("contract")) - vmenv = NewEnv(cfg) - sender = vm.AccountRef(cfg.Origin) - ) + destination := common.BytesToAddress([]byte("contract")) cfg.State.CreateAccount(destination) eoa := common.HexToAddress("E0") { @@ -363,12 +358,12 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode //cfg.State.CreateAccount(cfg.Origin) // set the receiver's (the executing contract) code for execution. cfg.State.SetCode(destination, code) - vmenv.Call(sender, destination, nil, gas, uint256.MustFromBig(cfg.Value)) + Call(destination, nil, cfg) b.Run(name, func(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - vmenv.Call(sender, destination, nil, gas, uint256.MustFromBig(cfg.Value)) + Call(destination, nil, cfg) } }) } From e9981bc6f72229f2bcc87b6721d2c06a195332b9 Mon Sep 17 00:00:00 2001 From: Zhihao Lin <3955922+kkqy@users.noreply.github.com> Date: Tue, 6 Aug 2024 20:14:37 +0800 Subject: [PATCH 13/39] ethclient: support networkID in hex format (#30263) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some chains’ network IDs use hexadecimal such as Optimism ("0xa" instead of "10"), so when converting the string to big.Int, we cannot specify base 10; otherwise, it will encounter errors with hexadecimal network IDs. --- ethclient/ethclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 390f085677..f769ee8475 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -359,7 +359,7 @@ func (ec *Client) NetworkID(ctx context.Context) (*big.Int, error) { if err := ec.c.CallContext(ctx, &ver, "net_version"); err != nil { return nil, err } - if _, ok := version.SetString(ver, 10); !ok { + if _, ok := version.SetString(ver, 0); !ok { return nil, fmt.Errorf("invalid net_version result %q", ver) } return version, nil From b37ac5c1027847de2eb65162a7251b37b9466793 Mon Sep 17 00:00:00 2001 From: lmittmann <3458786+lmittmann@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:38:47 +0200 Subject: [PATCH 14/39] core/vm: improved stack swap performance (#30249) This PR adds the methods `Stack.swap1..16()` that faster than `Stack.swap(1..16)`. Co-authored-by: lmittmann --- core/vm/instructions.go | 90 +++++++++++++++++++++++++++++---- core/vm/jump_table.go | 32 ++++++------ core/vm/runtime/runtime_test.go | 29 +++++++++++ core/vm/stack.go | 51 +++++++++++++++++-- 4 files changed, 173 insertions(+), 29 deletions(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 9ec4544643..1e8be295d8 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -583,6 +583,86 @@ func opGas(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte return nil, nil } +func opSwap1(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap1() + return nil, nil +} + +func opSwap2(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap2() + return nil, nil +} + +func opSwap3(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap3() + return nil, nil +} + +func opSwap4(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap4() + return nil, nil +} + +func opSwap5(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap5() + return nil, nil +} + +func opSwap6(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap6() + return nil, nil +} + +func opSwap7(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap7() + return nil, nil +} + +func opSwap8(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap8() + return nil, nil +} + +func opSwap9(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap9() + return nil, nil +} + +func opSwap10(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap10() + return nil, nil +} + +func opSwap11(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap11() + return nil, nil +} + +func opSwap12(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap12() + return nil, nil +} + +func opSwap13(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap13() + return nil, nil +} + +func opSwap14(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap14() + return nil, nil +} + +func opSwap15(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap15() + return nil, nil +} + +func opSwap16(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap16() + return nil, nil +} + func opCreate(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { if interpreter.readOnly { return nil, ErrWriteProtection @@ -923,13 +1003,3 @@ func makeDup(size int64) executionFunc { return nil, nil } } - -// make swap instruction function -func makeSwap(size int64) executionFunc { - // switch n + 1 otherwise n would be swapped with n - size++ - return func(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { - scope.Stack.swap(int(size)) - return nil, nil - } -} diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index 5624f47ba7..6b2950194d 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -892,97 +892,97 @@ func newFrontierInstructionSet() JumpTable { maxStack: maxDupStack(16), }, SWAP1: { - execute: makeSwap(1), + execute: opSwap1, constantGas: GasFastestStep, minStack: minSwapStack(2), maxStack: maxSwapStack(2), }, SWAP2: { - execute: makeSwap(2), + execute: opSwap2, constantGas: GasFastestStep, minStack: minSwapStack(3), maxStack: maxSwapStack(3), }, SWAP3: { - execute: makeSwap(3), + execute: opSwap3, constantGas: GasFastestStep, minStack: minSwapStack(4), maxStack: maxSwapStack(4), }, SWAP4: { - execute: makeSwap(4), + execute: opSwap4, constantGas: GasFastestStep, minStack: minSwapStack(5), maxStack: maxSwapStack(5), }, SWAP5: { - execute: makeSwap(5), + execute: opSwap5, constantGas: GasFastestStep, minStack: minSwapStack(6), maxStack: maxSwapStack(6), }, SWAP6: { - execute: makeSwap(6), + execute: opSwap6, constantGas: GasFastestStep, minStack: minSwapStack(7), maxStack: maxSwapStack(7), }, SWAP7: { - execute: makeSwap(7), + execute: opSwap7, constantGas: GasFastestStep, minStack: minSwapStack(8), maxStack: maxSwapStack(8), }, SWAP8: { - execute: makeSwap(8), + execute: opSwap8, constantGas: GasFastestStep, minStack: minSwapStack(9), maxStack: maxSwapStack(9), }, SWAP9: { - execute: makeSwap(9), + execute: opSwap9, constantGas: GasFastestStep, minStack: minSwapStack(10), maxStack: maxSwapStack(10), }, SWAP10: { - execute: makeSwap(10), + execute: opSwap10, constantGas: GasFastestStep, minStack: minSwapStack(11), maxStack: maxSwapStack(11), }, SWAP11: { - execute: makeSwap(11), + execute: opSwap11, constantGas: GasFastestStep, minStack: minSwapStack(12), maxStack: maxSwapStack(12), }, SWAP12: { - execute: makeSwap(12), + execute: opSwap12, constantGas: GasFastestStep, minStack: minSwapStack(13), maxStack: maxSwapStack(13), }, SWAP13: { - execute: makeSwap(13), + execute: opSwap13, constantGas: GasFastestStep, minStack: minSwapStack(14), maxStack: maxSwapStack(14), }, SWAP14: { - execute: makeSwap(14), + execute: opSwap14, constantGas: GasFastestStep, minStack: minSwapStack(15), maxStack: maxSwapStack(15), }, SWAP15: { - execute: makeSwap(15), + execute: opSwap15, constantGas: GasFastestStep, minStack: minSwapStack(16), maxStack: maxSwapStack(16), }, SWAP16: { - execute: makeSwap(16), + execute: opSwap16, constantGas: GasFastestStep, minStack: minSwapStack(17), maxStack: maxSwapStack(17), diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index a49eca25d3..f52484606b 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -212,6 +212,35 @@ func BenchmarkEVM_CREATE2_1200(bench *testing.B) { benchmarkEVM_Create(bench, "5b5862124f80600080f5600152600056") } +func BenchmarkEVM_SWAP1(b *testing.B) { + // returns a contract that does n swaps (SWAP1) + swapContract := func(n uint64) []byte { + contract := []byte{ + byte(vm.PUSH0), // PUSH0 + byte(vm.PUSH0), // PUSH0 + } + for i := uint64(0); i < n; i++ { + contract = append(contract, byte(vm.SWAP1)) + } + return contract + } + + state, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + contractAddr := common.BytesToAddress([]byte("contract")) + + b.Run("10k", func(b *testing.B) { + contractCode := swapContract(10_000) + state.SetCode(contractAddr, contractCode) + + for i := 0; i < b.N; i++ { + _, _, err := Call(contractAddr, []byte{}, &Config{State: state}) + if err != nil { + b.Fatal(err) + } + } + }) +} + func fakeHeader(n uint64, parentHash common.Hash) *types.Header { header := types.Header{ Coinbase: common.HexToAddress("0x00000000000000000000000000000000deadbeef"), diff --git a/core/vm/stack.go b/core/vm/stack.go index e1a957e244..879dc9aa6d 100644 --- a/core/vm/stack.go +++ b/core/vm/stack.go @@ -30,7 +30,7 @@ var stackPool = sync.Pool{ // Stack is an object for basic stack operations. Items popped to the stack are // expected to be changed and modified. stack does not take care of adding newly -// initialised objects. +// initialized objects. type Stack struct { data []uint256.Int } @@ -64,8 +64,53 @@ func (st *Stack) len() int { return len(st.data) } -func (st *Stack) swap(n int) { - st.data[st.len()-n], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-n] +func (st *Stack) swap1() { + st.data[st.len()-2], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-2] +} +func (st *Stack) swap2() { + st.data[st.len()-3], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-3] +} +func (st *Stack) swap3() { + st.data[st.len()-4], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-4] +} +func (st *Stack) swap4() { + st.data[st.len()-5], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-5] +} +func (st *Stack) swap5() { + st.data[st.len()-6], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-6] +} +func (st *Stack) swap6() { + st.data[st.len()-7], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-7] +} +func (st *Stack) swap7() { + st.data[st.len()-8], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-8] +} +func (st *Stack) swap8() { + st.data[st.len()-9], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-9] +} +func (st *Stack) swap9() { + st.data[st.len()-10], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-10] +} +func (st *Stack) swap10() { + st.data[st.len()-11], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-11] +} +func (st *Stack) swap11() { + st.data[st.len()-12], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-12] +} +func (st *Stack) swap12() { + st.data[st.len()-13], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-13] +} +func (st *Stack) swap13() { + st.data[st.len()-14], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-14] +} +func (st *Stack) swap14() { + st.data[st.len()-15], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-15] +} +func (st *Stack) swap15() { + st.data[st.len()-16], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-16] +} +func (st *Stack) swap16() { + st.data[st.len()-17], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-17] } func (st *Stack) dup(n int) { From 978041feeaffc3f91afd98c8495a63bfff4b12f4 Mon Sep 17 00:00:00 2001 From: llkhacquan <3724362+llkhacquan@users.noreply.github.com> Date: Thu, 8 Aug 2024 14:13:18 +0700 Subject: [PATCH 15/39] signer/core: improve performance of isPrimitiveTypeValid function (#30274) (#30277) Precomputes valid primitive types into a map to use for validation, thus removing sprintf. --- signer/core/apitypes/types.go | 48 ++++++++++++++++------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 73243b16a1..e886d7fc44 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -843,39 +843,35 @@ func (t Types) validate() error { return nil } -// Checks if the primitive value is valid -func isPrimitiveTypeValid(primitiveType string) bool { - if primitiveType == "address" || - primitiveType == "address[]" || - primitiveType == "bool" || - primitiveType == "bool[]" || - primitiveType == "string" || - primitiveType == "string[]" || - primitiveType == "bytes" || - primitiveType == "bytes[]" || - primitiveType == "int" || - primitiveType == "int[]" || - primitiveType == "uint" || - primitiveType == "uint[]" { - return true +var validPrimitiveTypes = map[string]struct{}{} + +// build the set of valid primitive types +func init() { + // Types those are trivially valid + for _, t := range []string{ + "address", "address[]", "bool", "bool[]", "string", "string[]", + "bytes", "bytes[]", "int", "int[]", "uint", "uint[]", + } { + validPrimitiveTypes[t] = struct{}{} } // For 'bytesN', 'bytesN[]', we allow N from 1 to 32 for n := 1; n <= 32; n++ { - // e.g. 'bytes28' or 'bytes28[]' - if primitiveType == fmt.Sprintf("bytes%d", n) || primitiveType == fmt.Sprintf("bytes%d[]", n) { - return true - } + validPrimitiveTypes[fmt.Sprintf("bytes%d", n)] = struct{}{} + validPrimitiveTypes[fmt.Sprintf("bytes%d[]", n)] = struct{}{} } // For 'intN','intN[]' and 'uintN','uintN[]' we allow N in increments of 8, from 8 up to 256 for n := 8; n <= 256; n += 8 { - if primitiveType == fmt.Sprintf("int%d", n) || primitiveType == fmt.Sprintf("int%d[]", n) { - return true - } - if primitiveType == fmt.Sprintf("uint%d", n) || primitiveType == fmt.Sprintf("uint%d[]", n) { - return true - } + validPrimitiveTypes[fmt.Sprintf("int%d", n)] = struct{}{} + validPrimitiveTypes[fmt.Sprintf("int%d[]", n)] = struct{}{} + validPrimitiveTypes[fmt.Sprintf("uint%d", n)] = struct{}{} + validPrimitiveTypes[fmt.Sprintf("uint%d[]", n)] = struct{}{} } - return false +} + +// Checks if the primitive value is valid +func isPrimitiveTypeValid(primitiveType string) bool { + _, ok := validPrimitiveTypes[primitiveType] + return ok } // validate checks if the given domain is valid, i.e. contains at least From 4a3aed380e91fc3e01ccc35e367865de89adb508 Mon Sep 17 00:00:00 2001 From: lmittmann <3458786+lmittmann@users.noreply.github.com> Date: Thu, 8 Aug 2024 10:27:38 +0200 Subject: [PATCH 16/39] core/vm: use uint64 in memory for indices everywhere (#30252) Consistently use `uint64` for indices in `Memory` and drop lots of type conversions from `uint64` to `int64`. --------- Co-authored-by: lmittmann --- core/vm/instructions.go | 22 +++++++++++----------- core/vm/memory.go | 21 +++++++-------------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 1e8be295d8..2e0f4c40ab 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -232,7 +232,7 @@ func opSAR(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte func opKeccak256(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { offset, size := scope.Stack.pop(), scope.Stack.peek() - data := scope.Memory.GetPtr(int64(offset.Uint64()), int64(size.Uint64())) + data := scope.Memory.GetPtr(offset.Uint64(), size.Uint64()) if interpreter.hasher == nil { interpreter.hasher = crypto.NewKeccakState() @@ -502,7 +502,7 @@ func opPop(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte func opMload(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { v := scope.Stack.peek() - offset := int64(v.Uint64()) + offset := v.Uint64() v.SetBytes(scope.Memory.GetPtr(offset, 32)) return nil, nil } @@ -670,7 +670,7 @@ func opCreate(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]b var ( value = scope.Stack.pop() offset, size = scope.Stack.pop(), scope.Stack.pop() - input = scope.Memory.GetCopy(int64(offset.Uint64()), int64(size.Uint64())) + input = scope.Memory.GetCopy(offset.Uint64(), size.Uint64()) gas = scope.Contract.Gas ) if interpreter.evm.chainRules.IsEIP150 { @@ -714,7 +714,7 @@ func opCreate2(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([] endowment = scope.Stack.pop() offset, size = scope.Stack.pop(), scope.Stack.pop() salt = scope.Stack.pop() - input = scope.Memory.GetCopy(int64(offset.Uint64()), int64(size.Uint64())) + input = scope.Memory.GetCopy(offset.Uint64(), size.Uint64()) gas = scope.Contract.Gas ) @@ -752,7 +752,7 @@ func opCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byt addr, value, inOffset, inSize, retOffset, retSize := stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() toAddr := common.Address(addr.Bytes20()) // Get the arguments from the memory. - args := scope.Memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) + args := scope.Memory.GetPtr(inOffset.Uint64(), inSize.Uint64()) if interpreter.readOnly && !value.IsZero() { return nil, ErrWriteProtection @@ -788,7 +788,7 @@ func opCallCode(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ addr, value, inOffset, inSize, retOffset, retSize := stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() toAddr := common.Address(addr.Bytes20()) // Get arguments from the memory. - args := scope.Memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) + args := scope.Memory.GetPtr(inOffset.Uint64(), inSize.Uint64()) if !value.IsZero() { gas += params.CallStipend @@ -821,7 +821,7 @@ func opDelegateCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext addr, inOffset, inSize, retOffset, retSize := stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() toAddr := common.Address(addr.Bytes20()) // Get arguments from the memory. - args := scope.Memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) + args := scope.Memory.GetPtr(inOffset.Uint64(), inSize.Uint64()) ret, returnGas, err := interpreter.evm.DelegateCall(scope.Contract, toAddr, args, gas) if err != nil { @@ -850,7 +850,7 @@ func opStaticCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) addr, inOffset, inSize, retOffset, retSize := stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() toAddr := common.Address(addr.Bytes20()) // Get arguments from the memory. - args := scope.Memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) + args := scope.Memory.GetPtr(inOffset.Uint64(), inSize.Uint64()) ret, returnGas, err := interpreter.evm.StaticCall(scope.Contract, toAddr, args, gas) if err != nil { @@ -871,14 +871,14 @@ func opStaticCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) func opReturn(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { offset, size := scope.Stack.pop(), scope.Stack.pop() - ret := scope.Memory.GetPtr(int64(offset.Uint64()), int64(size.Uint64())) + ret := scope.Memory.GetPtr(offset.Uint64(), size.Uint64()) return ret, errStopToken } func opRevert(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { offset, size := scope.Stack.pop(), scope.Stack.pop() - ret := scope.Memory.GetPtr(int64(offset.Uint64()), int64(size.Uint64())) + ret := scope.Memory.GetPtr(offset.Uint64(), size.Uint64()) interpreter.returnData = ret return ret, ErrExecutionReverted @@ -947,7 +947,7 @@ func makeLog(size int) executionFunc { topics[i] = addr.Bytes32() } - d := scope.Memory.GetCopy(int64(mStart.Uint64()), int64(mSize.Uint64())) + d := scope.Memory.GetCopy(mStart.Uint64(), mSize.Uint64()) interpreter.evm.StateDB.AddLog(&types.Log{ Address: scope.Contract.Address(), Topics: topics, diff --git a/core/vm/memory.go b/core/vm/memory.go index e0202fd7c0..33203879ae 100644 --- a/core/vm/memory.go +++ b/core/vm/memory.go @@ -66,32 +66,25 @@ func (m *Memory) Resize(size uint64) { } // GetCopy returns offset + size as a new slice -func (m *Memory) GetCopy(offset, size int64) (cpy []byte) { +func (m *Memory) GetCopy(offset, size uint64) (cpy []byte) { if size == 0 { return nil } - if len(m.store) > int(offset) { - cpy = make([]byte, size) - copy(cpy, m.store[offset:offset+size]) - - return - } - + // memory is always resized before being accessed, no need to check bounds + cpy = make([]byte, size) + copy(cpy, m.store[offset:offset+size]) return } // GetPtr returns the offset + size -func (m *Memory) GetPtr(offset, size int64) []byte { +func (m *Memory) GetPtr(offset, size uint64) []byte { if size == 0 { return nil } - if len(m.store) > int(offset) { - return m.store[offset : offset+size] - } - - return nil + // memory is always resized before being accessed, no need to check bounds + return m.store[offset : offset+size] } // Len returns the length of the backing slice From 9ea766d6e9189e8fbec7dbcacaf4c3f5083276ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Kj=C3=A6rstad?= Date: Thu, 8 Aug 2024 13:47:43 +0200 Subject: [PATCH 17/39] build: upgrade -dlgo version to Go 1.22.6 (#30273) --- build/checksums.txt | 96 ++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index 15c74559e6..5a25229885 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -5,55 +5,55 @@ # https://github.com/ethereum/execution-spec-tests/releases/download/v2.1.0/ ca89c76851b0900bfcc3cbb9a26cbece1f3d7c64a3bed38723e914713290df6c fixtures_develop.tar.gz -# version:golang 1.22.5 +# version:golang 1.22.6 # https://go.dev/dl/ -ac9c723f224969aee624bc34fd34c9e13f2a212d75c71c807de644bb46e112f6 go1.22.5.src.tar.gz -c82ba3403c45a4aa4b84b08244656a51e55b86fb130dcc500f5291d0f3b12222 go1.22.5.aix-ppc64.tar.gz -8a8872b1bac959b3b76f2e3978c46406d22a54a99c83ca55840ca08b4f2960bc go1.22.5.darwin-amd64.pkg -95d9933cdcf45f211243c42c7705c37353cccd99f27eb4d8e2d1bf2f4165cb50 go1.22.5.darwin-amd64.tar.gz -8c943512d1fa4e849f0078b03721df02aac19d8bb872dd17ab3ee7127ae6b732 go1.22.5.darwin-arm64.pkg -4cd1bcb05be03cecb77bccd765785d5ff69d79adf4dd49790471d00c06b41133 go1.22.5.darwin-arm64.tar.gz -1f1f035e968a877cd8ed62adae6edb2feeee62470660b7587ddcb904a3877a21 go1.22.5.dragonfly-amd64.tar.gz -d660698411465531d475ec1c617fdb415df68740f3511138a8d15506665a06f9 go1.22.5.freebsd-386.tar.gz -75f43ef46c2ad46c534ded25d26fba9bef036fc07074dfa45c0b3b90856a8151 go1.22.5.freebsd-amd64.tar.gz -75614714e7e4a4dd721f0eddd6555b3f6afc4c07e59c1b9b769cf663996165f9 go1.22.5.freebsd-arm.tar.gz -1377d0d7233f1b8f4cb8e3456f2e7ed44aca4a95daab79ae09605d34aa967c6b go1.22.5.freebsd-arm64.tar.gz -07baf198587abc05ea789dbe5810a2d6612ad56a51718bbf74de2c93bdbe676a go1.22.5.freebsd-riscv64.tar.gz -c0bd4f0d44252f3ec93ca850a41b167bb868179c7c283f8af9439e73b2654b17 go1.22.5.illumos-amd64.tar.gz -3ea4c78e6fa52978ae1ed2e5927ad17495da440c9fae7787b1ebc1d0572f7f43 go1.22.5.linux-386.tar.gz -904b924d435eaea086515bc63235b192ea441bd8c9b198c507e85009e6e4c7f0 go1.22.5.linux-amd64.tar.gz -8d21325bfcf431be3660527c1a39d3d9ad71535fabdf5041c826e44e31642b5a go1.22.5.linux-arm64.tar.gz -8c4587cf3e63c9aefbcafa92818c4d9d51683af93ea687bf6c7508d6fa36f85e go1.22.5.linux-armv6l.tar.gz -780e2eeb6376a763c564f776eaac6700f33f95e29302faa54b040b19cb1f6fd2 go1.22.5.linux-loong64.tar.gz -f784aa1adfb605da3bfe8cd534b545bddae3eb893e9302f7c2f5d44656b1cae2 go1.22.5.linux-mips.tar.gz -aaa3756571467768388f2ab641a02ff54f98f1684808cda047a7be3026e4b438 go1.22.5.linux-mips64.tar.gz -b7956d925c9ef5a4dc53017feaed2d78dba5d0a1036bad5ea513f1f15ba08fbc go1.22.5.linux-mips64le.tar.gz -7baf605be9b787acd750b6b48a91818a5590ec9289b14aea5696a46b41853888 go1.22.5.linux-mipsle.tar.gz -f09b2a6c1a409662e8e8fe267e1eabeba0a1fd00eb1422fd88297b013803952e go1.22.5.linux-ppc64.tar.gz -5312bb420ac0b59175a58927e70b4660b14ab7319aab54398b6071fabcbfbb09 go1.22.5.linux-ppc64le.tar.gz -f8d0c7d96b336f4133409ff9da7241cfe91e65723c2e8e7c7f9b58a9f9603476 go1.22.5.linux-riscv64.tar.gz -24c6c5c9d515adea5d58ae78388348c97614a0c21ac4d4f4c0dab75e893b0b5d go1.22.5.linux-s390x.tar.gz -39144c62acbaa85e4f1ab57bad8f5b3dc67d6fa24b711ec1fa593f4a0ea1fe91 go1.22.5.netbsd-386.tar.gz -118f79640588eb878529b46cdf56599012da6575f0ac07069ec1e9a8e78ddd0b go1.22.5.netbsd-amd64.tar.gz -d39c2b94ae3fd0a6399e545cbecb673496293075291bd98ef15f24d21625a490 go1.22.5.netbsd-arm.tar.gz -f7fb617d10c39248996521d72370db82d50724fa894089c76ae4298fbbe1fb0b go1.22.5.netbsd-arm64.tar.gz -e0f778a34746587ae7c18e8a24cfaba1b2eaabce75d0ceb470adf576ad1cd90f go1.22.5.openbsd-386.tar.gz -b417311df26ef7ae8b34fcb991519a5c496010561c12386d9469aea03c1bdf0b go1.22.5.openbsd-amd64.tar.gz -e78e8ad05605d530a4f79e55031c7c65f2020a9d442e05d490bd08f0d947a34f go1.22.5.openbsd-arm.tar.gz -8027898948f17742717786ead2ff2e960ee1fc82995d6edbad0050d551710f59 go1.22.5.openbsd-arm64.tar.gz -99c5b81d75bcc0d83d25dedc9535682c42c0e761276c88bcc4db6340344644fd go1.22.5.openbsd-ppc64.tar.gz -30d5dacdee0481f0b8cabb75b706465e2177c3a4a1d1c46293332f4b90a3d199 go1.22.5.plan9-386.tar.gz -65628650cd7665387cfe6fa386c381f4de1ef7b03a12067ae9ccf06d2feaea2c go1.22.5.plan9-amd64.tar.gz -322541cbfc9ae95b48b9eec4eb45df48299784592e23121084f790cf1082787e go1.22.5.plan9-arm.tar.gz -87c590e3eb81fcffa3dc1524c03c2847f0890e95c2a43586e82b56c262eb03d8 go1.22.5.solaris-amd64.tar.gz -3ec89ed822b38f4483977a90913fbe39d0857f0ed16c4642dec1950ddbe8c943 go1.22.5.windows-386.msi -c44fc421075022add78fbf8db38519dd5520a11832749be2189e64b3cf4f02f9 go1.22.5.windows-386.zip -86b0299ab8cb9c44882a9080dac03f7f4d9546f51ed1ba1015599114bcbc66d0 go1.22.5.windows-amd64.msi -59968438b8d90f108fd240d4d2f95b037e59716995f7409e0a322dcb996e9f42 go1.22.5.windows-amd64.zip -013d3b300e6b8f26482d6dd17b02830b83ee63795498bd8c0c9d80bb2c4d6cf7 go1.22.5.windows-arm.msi -8cc860630a84e2dbff3e84280f46a571741f26f8a1819aa4fbcb3164fdd51312 go1.22.5.windows-arm.zip -8f90519d9f305f2caa05d1d4fb0656b50f1bf89d76e194279f480e5a484c891f go1.22.5.windows-arm64.msi -6717d5841162aa8c05f932eb74a643f1310b8a88f80f0830e86d194289734bbf go1.22.5.windows-arm64.zip +9e48d99d519882579917d8189c17e98c373ce25abaebb98772e2927088992a51 go1.22.6.src.tar.gz +eeb0cc42120cbae6d3695dae2e5420fa0e93a5db957db139b55efdb879dd9856 go1.22.6.aix-ppc64.tar.gz +b47ac340f0b072943fed1f558a26eb260cc23bd21b8af175582e9103141d465b go1.22.6.darwin-amd64.pkg +9c3c0124b01b5365f73a1489649f78f971ecf84844ad9ca58fde133096ddb61b go1.22.6.darwin-amd64.tar.gz +14d0355ec1c0eeb213a16efa8635fac1f16067ef78a8173abf9a8c7b805e551e go1.22.6.darwin-arm64.pkg +ebac39fd44fc22feed1bb519af431c84c55776e39b30f4fd62930da9c0cfd1e3 go1.22.6.darwin-arm64.tar.gz +3695b10c722a4920c8a736284f8820c142e1e752f3a87f797a45c64366f7a173 go1.22.6.dragonfly-amd64.tar.gz +a9b9570c80294a664d50b566d6bd1aa42465997d2d76a57936b32f55f5c69c63 go1.22.6.freebsd-386.tar.gz +424a5618406800365fe3ad96a795fb55ce394bea3ff48eaf56d292bf7a916d1e go1.22.6.freebsd-amd64.tar.gz +e0dce3a6dbe8e7e054d329dd4cb403935c63c0f7e22e693077aa60e12018b883 go1.22.6.freebsd-arm.tar.gz +34930b01f58889c71f7a78c51c6c3bd2ce289ac7862c76dab691303cfa935fd1 go1.22.6.freebsd-arm64.tar.gz +4c9d630e55d4d600a5b4297e59620c3bdfe63a441981682b3638e2fdda228a44 go1.22.6.freebsd-riscv64.tar.gz +9ed63feaf2ef56c56f1cf0d9d3fab4006efd22a38e2f1f5252e95c6ac09332f3 go1.22.6.illumos-amd64.tar.gz +9e680027b058beab10ce5938607660964b6d2c564bf50bdb01aa090dc5beda98 go1.22.6.linux-386.tar.gz +999805bed7d9039ec3da1a53bfbcafc13e367da52aa823cb60b68ba22d44c616 go1.22.6.linux-amd64.tar.gz +c15fa895341b8eaf7f219fada25c36a610eb042985dc1a912410c1c90098eaf2 go1.22.6.linux-arm64.tar.gz +b566484fe89a54c525dd1a4cbfec903c1f6e8f0b7b3dbaf94c79bc9145391083 go1.22.6.linux-armv6l.tar.gz +1ee6e1896aea856142d2af7045cea118995b39404aa61afd12677d023d47ee69 go1.22.6.linux-loong64.tar.gz +fdd0e1a3e178f9bc79adf6ff1e3de4554ce581b4c468fd6e113c43fbbbe1eec6 go1.22.6.linux-mips.tar.gz +d3e5a621fc5a07759e503a971af0b28ded6a7d6f5604ab511f51f930a18dd3e4 go1.22.6.linux-mips64.tar.gz +01547606c5b5c1b0e5587b3afd65172860d2f4755e523785832905759ecce2d7 go1.22.6.linux-mips64le.tar.gz +2cd771416ae03c11240cfdb551d66ab9a941077664f3727b966f94386c23b0fa go1.22.6.linux-mipsle.tar.gz +6ef61d517777925e6bdb0321ea42d5f60acc20c1314dd902b9d0bfa3a5fd4fca go1.22.6.linux-ppc64.tar.gz +9d99fce3f6f72a76630fe91ec0884dfe3db828def4713368424900fa98bb2bd6 go1.22.6.linux-ppc64le.tar.gz +30be9c9b9cc4f044d4da9a33ee601ab7b3aff4246107d323a79e08888710754e go1.22.6.linux-riscv64.tar.gz +82f3bae3ddb4ede45b848db48c5486fadb58551e74507bda45484257e7194a95 go1.22.6.linux-s390x.tar.gz +85b2eb9d40a930bd3e75d0096a6eb5847aac86c5085e6d13a5845e9ef03f8d4b go1.22.6.netbsd-386.tar.gz +6e9acbdc34fb2a942d547c47c9c1989bb6e32b4a37d57fb312499e2bb33b46b7 go1.22.6.netbsd-amd64.tar.gz +e6eff3cf0038f2a9b0c9e01e228577a783bddcd8051222a3d949e24ee392e769 go1.22.6.netbsd-arm.tar.gz +43a7e2ba22da700b844f7561e3dd5434540ed6c9781be2e9c42e8a8cbf558f8e go1.22.6.netbsd-arm64.tar.gz +a90b758ccb45d8a17af8e140fafa1e97607de5a7ecd53a4c55f69258bfb043d0 go1.22.6.openbsd-386.tar.gz +cc13436c4a644e55bedcea65981eb80ca8317b39b129f5563ab3b6da1391bd47 go1.22.6.openbsd-amd64.tar.gz +aee34f61ba2b0a8f2618f5c7065e20da7714ce7651680509eda30728fe01ee88 go1.22.6.openbsd-arm.tar.gz +c67d57daf8baada93c69c8fb02401270cd33159730b1f2d70d9e724ba1a918cf go1.22.6.openbsd-arm64.tar.gz +03e1f96002e94a6b381bcf66a0a62b9d5f63148682a780d727840ad540185c7c go1.22.6.openbsd-ppc64.tar.gz +0ac2b5bbe2c8a293d284512630e629bf0578aaa7b7b1f39ac4ee182c7924aaad go1.22.6.plan9-386.tar.gz +f9afdab8a72a8d874f023f5605482cc94160843ac768dbd840e6f772d16578c7 go1.22.6.plan9-amd64.tar.gz +4b9f01a47e6a29d57cbb3097b6770583336cef9c8f0d51d3d1451e42a851002e go1.22.6.plan9-arm.tar.gz +46c2552ac7b8d6314a52e14e0a0761aaeebdd6aba5f531de386f4cf2b66ec723 go1.22.6.solaris-amd64.tar.gz +a57821dab76af1ef7a6b62db1628f0caa74343e0c7cb829df9ce8ea0713a3e8e go1.22.6.windows-386.msi +eb734bacc9aabca1273b61dd392bb84a9bb33783f5e2fff2cd6ab9885bbefbe6 go1.22.6.windows-386.zip +1238a3e6892eb8a0eb3fe0640e18ab82ca21cc1a933f16897b2ad081f057b5da go1.22.6.windows-amd64.msi +6023083a6e4d3199b44c37e9ba7b25d9674da20fd846a35ee5f9589d81c21a6a go1.22.6.windows-amd64.zip +6791218c568a3d000cb36317506541d7fd67e7cfe613baaf361ca36cad5e2cd5 go1.22.6.windows-arm.msi +ee41ca83bb07c4fd46a1d6b2d083519bb8ca156fcd9db37ee711234d43126e2f go1.22.6.windows-arm.zip +91c6b3376612095315a0aeae4b03e3da34fabe9dfd4532d023e2a70f913cf22a go1.22.6.windows-arm64.msi +7cf55f357ba8116cd3bff992980e20a704ba451b3dab341cf1787b133d900512 go1.22.6.windows-arm64.zip # version:golangci 1.59.0 # https://github.com/golangci/golangci-lint/releases/ From d3dae66e594748c2fb608177726bd46715b0a3a6 Mon Sep 17 00:00:00 2001 From: taiking Date: Thu, 8 Aug 2024 20:50:00 +0900 Subject: [PATCH 18/39] tests: fix TransactionTest to actually run (#30272) Due to https://github.com/ethereum/tests/releases/tag/v10.1, the format of the TransactionTest changed, but it was not properly addressed, causing the test to pass unexpectedly. --------- Co-authored-by: Martin Holst Swende --- tests/transaction_test.go | 11 ++++------- tests/transaction_test_util.go | 22 +++++++++++++--------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/tests/transaction_test.go b/tests/transaction_test.go index cb0f262318..5179fc9afe 100644 --- a/tests/transaction_test.go +++ b/tests/transaction_test.go @@ -26,20 +26,17 @@ func TestTransaction(t *testing.T) { t.Parallel() txt := new(testMatcher) - // These can't be parsed, invalid hex in RLP - txt.skipLoad("^ttWrongRLP/.*") // We don't allow more than uint64 in gas amount // This is a pseudo-consensus vulnerability, but not in practice // because of the gas limit txt.skipLoad("^ttGasLimit/TransactionWithGasLimitxPriceOverflow.json") // We _do_ allow more than uint64 in gas price, as opposed to the tests // This is also not a concern, as long as tx.Cost() uses big.Int for - // calculating the final cozt - txt.skipLoad(".*TransactionWithGasPriceOverflow.*") + // calculating the final cost + txt.skipLoad("^ttGasPrice/TransactionWithGasPriceOverflow.json") - // The nonce is too large for uint64. Not a concern, it means geth won't - // accept transactions at a certain point in the distant future - txt.skipLoad("^ttNonce/TransactionWithHighNonce256.json") + // The maximum value of nonce is 2^64 - 1 + txt.skipLoad("^ttNonce/TransactionWithHighNonce64Minus1.json") // The value is larger than uint64, which according to the test is invalid. // Geth accepts it, which is not a consensus issue since we use big.Int's diff --git a/tests/transaction_test_util.go b/tests/transaction_test_util.go index 391aa57584..d3dbbd5db2 100644 --- a/tests/transaction_test_util.go +++ b/tests/transaction_test_util.go @@ -29,7 +29,11 @@ import ( // TransactionTest checks RLP decoding and sender derivation of transactions. type TransactionTest struct { - RLP hexutil.Bytes `json:"rlp"` + Txbytes hexutil.Bytes `json:"txbytes"` + Result ttResult +} + +type ttResult struct { Byzantium ttFork Constantinople ttFork Istanbul ttFork @@ -73,15 +77,15 @@ func (tt *TransactionTest) Run(config *params.ChainConfig) error { isHomestead bool isIstanbul bool }{ - {"Frontier", types.FrontierSigner{}, tt.Frontier, false, false}, - {"Homestead", types.HomesteadSigner{}, tt.Homestead, true, false}, - {"EIP150", types.HomesteadSigner{}, tt.EIP150, true, false}, - {"EIP158", types.NewEIP155Signer(config.ChainID), tt.EIP158, true, false}, - {"Byzantium", types.NewEIP155Signer(config.ChainID), tt.Byzantium, true, false}, - {"Constantinople", types.NewEIP155Signer(config.ChainID), tt.Constantinople, true, false}, - {"Istanbul", types.NewEIP155Signer(config.ChainID), tt.Istanbul, true, true}, + {"Frontier", types.FrontierSigner{}, tt.Result.Frontier, false, false}, + {"Homestead", types.HomesteadSigner{}, tt.Result.Homestead, true, false}, + {"EIP150", types.HomesteadSigner{}, tt.Result.EIP150, true, false}, + {"EIP158", types.NewEIP155Signer(config.ChainID), tt.Result.EIP158, true, false}, + {"Byzantium", types.NewEIP155Signer(config.ChainID), tt.Result.Byzantium, true, false}, + {"Constantinople", types.NewEIP155Signer(config.ChainID), tt.Result.Constantinople, true, false}, + {"Istanbul", types.NewEIP155Signer(config.ChainID), tt.Result.Istanbul, true, true}, } { - sender, txhash, err := validateTx(tt.RLP, testcase.signer, testcase.isHomestead, testcase.isIstanbul) + sender, txhash, err := validateTx(tt.Txbytes, testcase.signer, testcase.isHomestead, testcase.isIstanbul) if testcase.fork.Sender == (common.UnprefixedAddress{}) { if err == nil { From ebe31dfd5c8c053c098b65eac2ec3070b7f8e9a8 Mon Sep 17 00:00:00 2001 From: psogv0308 Date: Thu, 8 Aug 2024 22:14:00 +0900 Subject: [PATCH 19/39] eth/downloader, core/types: take withdrawals-size into account in downloader queue (#30276) Fixes a slight miscalculation in the downloader queue, which was not accurately taking block withdrawals into account when calculating the size of the items in the queue --- core/types/withdrawal.go | 7 +++++++ eth/downloader/queue.go | 1 + 2 files changed, 8 insertions(+) diff --git a/core/types/withdrawal.go b/core/types/withdrawal.go index d1ad918f98..6f99e53b56 100644 --- a/core/types/withdrawal.go +++ b/core/types/withdrawal.go @@ -18,6 +18,7 @@ package types import ( "bytes" + "reflect" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -48,6 +49,12 @@ type Withdrawals []*Withdrawal // Len returns the length of s. func (s Withdrawals) Len() int { return len(s) } +var withdrawalSize = int(reflect.TypeOf(Withdrawal{}).Size()) + +func (s Withdrawals) Size() int { + return withdrawalSize * len(s) +} + // EncodeIndex encodes the i'th withdrawal to w. Note that this does not check for errors // because we assume that *Withdrawal will only ever contain valid withdrawals that were either // constructed by decoding or via public API in this package. diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 267c23407f..5441ad1187 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -385,6 +385,7 @@ func (q *queue) Results(block bool) []*fetchResult { for _, tx := range result.Transactions { size += common.StorageSize(tx.Size()) } + size += common.StorageSize(result.Withdrawals.Size()) q.resultSize = common.StorageSize(blockCacheSizeWeight)*size + (1-common.StorageSize(blockCacheSizeWeight))*q.resultSize } From 83e70aa3d00bbee9713b31768a30a7741eee8945 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Thu, 8 Aug 2024 18:58:08 +0200 Subject: [PATCH 20/39] cmd/evm: fix evm basefee (#30281) fixes #30279 -- previously we did not use the basefee from the genesis, and instead the defaults were used from `runtime.go/setDefaults`-function --- cmd/evm/runner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index d06f85ed59..c02f9f0590 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -221,6 +221,7 @@ func runCmd(ctx *cli.Context) error { Time: genesisConfig.Timestamp, Coinbase: genesisConfig.Coinbase, BlockNumber: new(big.Int).SetUint64(genesisConfig.Number), + BaseFee: genesisConfig.BaseFee, BlobHashes: blobHashes, BlobBaseFee: blobBaseFee, EVMConfig: vm.Config{ From 811a69cd3cf77fe9b63c7dc260ff92a79c631846 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Fri, 9 Aug 2024 22:11:22 +0200 Subject: [PATCH 21/39] go.mod: update uint256 to 1.3.1 (#30280) Release notes: https://github.com/holiman/uint256/releases/tag/v1.3.1 --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3e9837186d..763210decd 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/hashicorp/go-bexpr v0.1.10 github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 github.com/holiman/bloomfilter/v2 v2.0.3 - github.com/holiman/uint256 v1.3.0 + github.com/holiman/uint256 v1.3.1 github.com/huin/goupnp v1.3.0 github.com/influxdata/influxdb-client-go/v2 v2.4.0 github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c diff --git a/go.sum b/go.sum index d8e7a540c7..562362dee6 100644 --- a/go.sum +++ b/go.sum @@ -310,8 +310,8 @@ github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8 github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= -github.com/holiman/uint256 v1.3.0 h1:4wdcm/tnd0xXdu7iS3ruNvxkWwrb4aeBQv19ayYn8F4= -github.com/holiman/uint256 v1.3.0/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= +github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= @@ -838,8 +838,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= From 32a1e0643ca5012bda851d65194fb5eb3d83591b Mon Sep 17 00:00:00 2001 From: Artyom Aminov Date: Sat, 10 Aug 2024 13:44:31 +0300 Subject: [PATCH 22/39] beacon/engine, consensus/beacon: use params.MaximumExtraDataSize instead of hard-coded value (#29721) Co-authored-by: Felix Lange Co-authored-by: Marius van der Wijden Co-authored-by: lightclient --- beacon/engine/types.go | 57 ++++++++++++++++++----------------- consensus/beacon/consensus.go | 2 +- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/beacon/engine/types.go b/beacon/engine/types.go index 1dfcf5b71a..d1b3aa22ab 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) @@ -193,21 +194,21 @@ func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) { // // and that the blockhash of the constructed block matches the parameters. Nil // Withdrawals value will propagate through the returned block. Empty -// Withdrawals value must be passed via non-nil, length 0 value in params. -func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (*types.Block, error) { - txs, err := decodeTransactions(params.Transactions) +// Withdrawals value must be passed via non-nil, length 0 value in data. +func ExecutableDataToBlock(data ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (*types.Block, error) { + txs, err := decodeTransactions(data.Transactions) if err != nil { return nil, err } - if len(params.ExtraData) > 32 { - return nil, fmt.Errorf("invalid extradata length: %v", len(params.ExtraData)) + if len(data.ExtraData) > int(params.MaximumExtraDataSize) { + return nil, fmt.Errorf("invalid extradata length: %v", len(data.ExtraData)) } - if len(params.LogsBloom) != 256 { - return nil, fmt.Errorf("invalid logsBloom length: %v", len(params.LogsBloom)) + if len(data.LogsBloom) != 256 { + return nil, fmt.Errorf("invalid logsBloom length: %v", len(data.LogsBloom)) } // Check that baseFeePerGas is not negative or too big - if params.BaseFeePerGas != nil && (params.BaseFeePerGas.Sign() == -1 || params.BaseFeePerGas.BitLen() > 256) { - return nil, fmt.Errorf("invalid baseFeePerGas: %v", params.BaseFeePerGas) + if data.BaseFeePerGas != nil && (data.BaseFeePerGas.Sign() == -1 || data.BaseFeePerGas.BitLen() > 256) { + return nil, fmt.Errorf("invalid baseFeePerGas: %v", data.BaseFeePerGas) } var blobHashes = make([]common.Hash, 0, len(txs)) for _, tx := range txs { @@ -225,34 +226,34 @@ func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash, // ExecutableData before withdrawals are enabled by marshaling // Withdrawals as the json null value. var withdrawalsRoot *common.Hash - if params.Withdrawals != nil { - h := types.DeriveSha(types.Withdrawals(params.Withdrawals), trie.NewStackTrie(nil)) + if data.Withdrawals != nil { + h := types.DeriveSha(types.Withdrawals(data.Withdrawals), trie.NewStackTrie(nil)) withdrawalsRoot = &h } header := &types.Header{ - ParentHash: params.ParentHash, + ParentHash: data.ParentHash, UncleHash: types.EmptyUncleHash, - Coinbase: params.FeeRecipient, - Root: params.StateRoot, + Coinbase: data.FeeRecipient, + Root: data.StateRoot, TxHash: types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)), - ReceiptHash: params.ReceiptsRoot, - Bloom: types.BytesToBloom(params.LogsBloom), + ReceiptHash: data.ReceiptsRoot, + Bloom: types.BytesToBloom(data.LogsBloom), Difficulty: common.Big0, - Number: new(big.Int).SetUint64(params.Number), - GasLimit: params.GasLimit, - GasUsed: params.GasUsed, - Time: params.Timestamp, - BaseFee: params.BaseFeePerGas, - Extra: params.ExtraData, - MixDigest: params.Random, + Number: new(big.Int).SetUint64(data.Number), + GasLimit: data.GasLimit, + GasUsed: data.GasUsed, + Time: data.Timestamp, + BaseFee: data.BaseFeePerGas, + Extra: data.ExtraData, + MixDigest: data.Random, WithdrawalsHash: withdrawalsRoot, - ExcessBlobGas: params.ExcessBlobGas, - BlobGasUsed: params.BlobGasUsed, + ExcessBlobGas: data.ExcessBlobGas, + BlobGasUsed: data.BlobGasUsed, ParentBeaconRoot: beaconRoot, } - block := types.NewBlockWithHeader(header).WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: params.Withdrawals}) - if block.Hash() != params.BlockHash { - return nil, fmt.Errorf("blockhash mismatch, want %x, got %x", params.BlockHash, block.Hash()) + block := types.NewBlockWithHeader(header).WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: data.Withdrawals}) + if block.Hash() != data.BlockHash { + return nil, fmt.Errorf("blockhash mismatch, want %x, got %x", data.BlockHash, block.Hash()) } return block, nil } diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index b8946e0c71..19763ed303 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -229,7 +229,7 @@ func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Blo // (c) the extradata is limited to 32 bytes func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, parent *types.Header) error { // Ensure that the header's extra-data section is of a reasonable size - if len(header.Extra) > 32 { + if len(header.Extra) > int(params.MaximumExtraDataSize) { return fmt.Errorf("extra-data longer than 32 bytes (%d)", len(header.Extra)) } // Verify the seal parts. Ensure the nonce and uncle hash are the expected value. From 33a13b6f2110807a15c443ec234580cfce94a194 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Mon, 12 Aug 2024 02:36:48 -0600 Subject: [PATCH 23/39] p2p/simulations: remove packages (#30250) Looking at the history of these packages over the past several years, there haven't been any meaningful contributions or usages: https://github.com/ethereum/go-ethereum/commits/master/p2p/simulations?before=de6d5976794a9ed3b626d4eba57bf7f0806fb970+35 Almost all of the commits are part of larger refactors or low-hanging-fruit contributions. Seems like it's not providing much value and taking up team + contributor time. --- .github/CODEOWNERS | 3 - cmd/p2psim/main.go | 443 ------- .../pipes/pipes.go => pipes/pipe.go} | 14 +- p2p/rlpx/rlpx_test.go | 2 +- p2p/simulations/README.md | 174 --- p2p/simulations/adapters/exec.go | 567 --------- p2p/simulations/adapters/inproc.go | 344 ------ p2p/simulations/adapters/inproc_test.go | 202 --- p2p/simulations/adapters/types.go | 325 ----- p2p/simulations/connect.go | 153 --- p2p/simulations/connect_test.go | 172 --- p2p/simulations/events.go | 110 -- p2p/simulations/examples/README.md | 39 - p2p/simulations/examples/ping-pong.go | 173 --- p2p/simulations/examples/ping-pong.sh | 40 - p2p/simulations/http.go | 743 ----------- p2p/simulations/http_test.go | 869 ------------- p2p/simulations/mocker.go | 197 --- p2p/simulations/mocker_test.go | 174 --- p2p/simulations/network.go | 1093 ----------------- p2p/simulations/network_test.go | 872 ------------- p2p/simulations/simulation.go | 157 --- p2p/simulations/test.go | 150 --- p2p/transport_test.go | 2 +- 24 files changed, 5 insertions(+), 7013 deletions(-) delete mode 100644 cmd/p2psim/main.go rename p2p/{simulations/pipes/pipes.go => pipes/pipe.go} (85%) delete mode 100644 p2p/simulations/README.md delete mode 100644 p2p/simulations/adapters/exec.go delete mode 100644 p2p/simulations/adapters/inproc.go delete mode 100644 p2p/simulations/adapters/inproc_test.go delete mode 100644 p2p/simulations/adapters/types.go delete mode 100644 p2p/simulations/connect.go delete mode 100644 p2p/simulations/connect_test.go delete mode 100644 p2p/simulations/events.go delete mode 100644 p2p/simulations/examples/README.md delete mode 100644 p2p/simulations/examples/ping-pong.go delete mode 100755 p2p/simulations/examples/ping-pong.sh delete mode 100644 p2p/simulations/http.go delete mode 100644 p2p/simulations/http_test.go delete mode 100644 p2p/simulations/mocker.go delete mode 100644 p2p/simulations/mocker_test.go delete mode 100644 p2p/simulations/network.go delete mode 100644 p2p/simulations/network_test.go delete mode 100644 p2p/simulations/simulation.go delete mode 100644 p2p/simulations/test.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ec2efb10e3..b50561172f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -21,7 +21,4 @@ light/ @zsfelfoldi @rjl493456442 node/ @fjl p2p/ @fjl @zsfelfoldi rpc/ @fjl @holiman -p2p/simulations @fjl -p2p/protocols @fjl -p2p/testing @fjl signer/ @holiman diff --git a/cmd/p2psim/main.go b/cmd/p2psim/main.go deleted file mode 100644 index a0f5f0d288..0000000000 --- a/cmd/p2psim/main.go +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -// p2psim provides a command-line client for a simulation HTTP API. -// -// Here is an example of creating a 2 node network with the first node -// connected to the second: -// -// $ p2psim node create -// Created node01 -// -// $ p2psim node start node01 -// Started node01 -// -// $ p2psim node create -// Created node02 -// -// $ p2psim node start node02 -// Started node02 -// -// $ p2psim node connect node01 node02 -// Connected node01 to node02 -package main - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - "strings" - "text/tabwriter" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/internal/flags" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/simulations" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" - "github.com/ethereum/go-ethereum/rpc" - "github.com/urfave/cli/v2" -) - -var client *simulations.Client - -var ( - // global command flags - apiFlag = &cli.StringFlag{ - Name: "api", - Value: "http://localhost:8888", - Usage: "simulation API URL", - EnvVars: []string{"P2PSIM_API_URL"}, - } - - // events subcommand flags - currentFlag = &cli.BoolFlag{ - Name: "current", - Usage: "get existing nodes and conns first", - } - filterFlag = &cli.StringFlag{ - Name: "filter", - Value: "", - Usage: "message filter", - } - - // node create subcommand flags - nameFlag = &cli.StringFlag{ - Name: "name", - Value: "", - Usage: "node name", - } - servicesFlag = &cli.StringFlag{ - Name: "services", - Value: "", - Usage: "node services (comma separated)", - } - keyFlag = &cli.StringFlag{ - Name: "key", - Value: "", - Usage: "node private key (hex encoded)", - } - - // node rpc subcommand flags - subscribeFlag = &cli.BoolFlag{ - Name: "subscribe", - Usage: "method is a subscription", - } -) - -func main() { - app := flags.NewApp("devp2p simulation command-line client") - app.Flags = []cli.Flag{ - apiFlag, - } - app.Before = func(ctx *cli.Context) error { - client = simulations.NewClient(ctx.String(apiFlag.Name)) - return nil - } - app.Commands = []*cli.Command{ - { - Name: "show", - Usage: "show network information", - Action: showNetwork, - }, - { - Name: "events", - Usage: "stream network events", - Action: streamNetwork, - Flags: []cli.Flag{ - currentFlag, - filterFlag, - }, - }, - { - Name: "snapshot", - Usage: "create a network snapshot to stdout", - Action: createSnapshot, - }, - { - Name: "load", - Usage: "load a network snapshot from stdin", - Action: loadSnapshot, - }, - { - Name: "node", - Usage: "manage simulation nodes", - Action: listNodes, - Subcommands: []*cli.Command{ - { - Name: "list", - Usage: "list nodes", - Action: listNodes, - }, - { - Name: "create", - Usage: "create a node", - Action: createNode, - Flags: []cli.Flag{ - nameFlag, - servicesFlag, - keyFlag, - }, - }, - { - Name: "show", - ArgsUsage: "", - Usage: "show node information", - Action: showNode, - }, - { - Name: "start", - ArgsUsage: "", - Usage: "start a node", - Action: startNode, - }, - { - Name: "stop", - ArgsUsage: "", - Usage: "stop a node", - Action: stopNode, - }, - { - Name: "connect", - ArgsUsage: " ", - Usage: "connect a node to a peer node", - Action: connectNode, - }, - { - Name: "disconnect", - ArgsUsage: " ", - Usage: "disconnect a node from a peer node", - Action: disconnectNode, - }, - { - Name: "rpc", - ArgsUsage: " []", - Usage: "call a node RPC method", - Action: rpcNode, - Flags: []cli.Flag{ - subscribeFlag, - }, - }, - }, - }, - } - if err := app.Run(os.Args); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -func showNetwork(ctx *cli.Context) error { - if ctx.NArg() != 0 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - network, err := client.GetNetwork() - if err != nil { - return err - } - w := tabwriter.NewWriter(ctx.App.Writer, 1, 2, 2, ' ', 0) - defer w.Flush() - fmt.Fprintf(w, "NODES\t%d\n", len(network.Nodes)) - fmt.Fprintf(w, "CONNS\t%d\n", len(network.Conns)) - return nil -} - -func streamNetwork(ctx *cli.Context) error { - if ctx.NArg() != 0 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - events := make(chan *simulations.Event) - sub, err := client.SubscribeNetwork(events, simulations.SubscribeOpts{ - Current: ctx.Bool(currentFlag.Name), - Filter: ctx.String(filterFlag.Name), - }) - if err != nil { - return err - } - defer sub.Unsubscribe() - enc := json.NewEncoder(ctx.App.Writer) - for { - select { - case event := <-events: - if err := enc.Encode(event); err != nil { - return err - } - case err := <-sub.Err(): - return err - } - } -} - -func createSnapshot(ctx *cli.Context) error { - if ctx.NArg() != 0 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - snap, err := client.CreateSnapshot() - if err != nil { - return err - } - return json.NewEncoder(os.Stdout).Encode(snap) -} - -func loadSnapshot(ctx *cli.Context) error { - if ctx.NArg() != 0 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - snap := &simulations.Snapshot{} - if err := json.NewDecoder(os.Stdin).Decode(snap); err != nil { - return err - } - return client.LoadSnapshot(snap) -} - -func listNodes(ctx *cli.Context) error { - if ctx.NArg() != 0 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - nodes, err := client.GetNodes() - if err != nil { - return err - } - w := tabwriter.NewWriter(ctx.App.Writer, 1, 2, 2, ' ', 0) - defer w.Flush() - fmt.Fprintf(w, "NAME\tPROTOCOLS\tID\n") - for _, node := range nodes { - fmt.Fprintf(w, "%s\t%s\t%s\n", node.Name, strings.Join(protocolList(node), ","), node.ID) - } - return nil -} - -func protocolList(node *p2p.NodeInfo) []string { - protos := make([]string, 0, len(node.Protocols)) - for name := range node.Protocols { - protos = append(protos, name) - } - return protos -} - -func createNode(ctx *cli.Context) error { - if ctx.NArg() != 0 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - config := adapters.RandomNodeConfig() - config.Name = ctx.String(nameFlag.Name) - if key := ctx.String(keyFlag.Name); key != "" { - privKey, err := crypto.HexToECDSA(key) - if err != nil { - return err - } - config.ID = enode.PubkeyToIDV4(&privKey.PublicKey) - config.PrivateKey = privKey - } - if services := ctx.String(servicesFlag.Name); services != "" { - config.Lifecycles = strings.Split(services, ",") - } - node, err := client.CreateNode(config) - if err != nil { - return err - } - fmt.Fprintln(ctx.App.Writer, "Created", node.Name) - return nil -} - -func showNode(ctx *cli.Context) error { - if ctx.NArg() != 1 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - nodeName := ctx.Args().First() - node, err := client.GetNode(nodeName) - if err != nil { - return err - } - w := tabwriter.NewWriter(ctx.App.Writer, 1, 2, 2, ' ', 0) - defer w.Flush() - fmt.Fprintf(w, "NAME\t%s\n", node.Name) - fmt.Fprintf(w, "PROTOCOLS\t%s\n", strings.Join(protocolList(node), ",")) - fmt.Fprintf(w, "ID\t%s\n", node.ID) - fmt.Fprintf(w, "ENODE\t%s\n", node.Enode) - for name, proto := range node.Protocols { - fmt.Fprintln(w) - fmt.Fprintf(w, "--- PROTOCOL INFO: %s\n", name) - fmt.Fprintf(w, "%v\n", proto) - fmt.Fprintf(w, "---\n") - } - return nil -} - -func startNode(ctx *cli.Context) error { - if ctx.NArg() != 1 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - nodeName := ctx.Args().First() - if err := client.StartNode(nodeName); err != nil { - return err - } - fmt.Fprintln(ctx.App.Writer, "Started", nodeName) - return nil -} - -func stopNode(ctx *cli.Context) error { - if ctx.NArg() != 1 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - nodeName := ctx.Args().First() - if err := client.StopNode(nodeName); err != nil { - return err - } - fmt.Fprintln(ctx.App.Writer, "Stopped", nodeName) - return nil -} - -func connectNode(ctx *cli.Context) error { - if ctx.NArg() != 2 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - args := ctx.Args() - nodeName := args.Get(0) - peerName := args.Get(1) - if err := client.ConnectNode(nodeName, peerName); err != nil { - return err - } - fmt.Fprintln(ctx.App.Writer, "Connected", nodeName, "to", peerName) - return nil -} - -func disconnectNode(ctx *cli.Context) error { - args := ctx.Args() - if args.Len() != 2 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - nodeName := args.Get(0) - peerName := args.Get(1) - if err := client.DisconnectNode(nodeName, peerName); err != nil { - return err - } - fmt.Fprintln(ctx.App.Writer, "Disconnected", nodeName, "from", peerName) - return nil -} - -func rpcNode(ctx *cli.Context) error { - args := ctx.Args() - if args.Len() < 2 { - return cli.ShowCommandHelp(ctx, ctx.Command.Name) - } - nodeName := args.Get(0) - method := args.Get(1) - rpcClient, err := client.RPCClient(context.Background(), nodeName) - if err != nil { - return err - } - if ctx.Bool(subscribeFlag.Name) { - return rpcSubscribe(rpcClient, ctx.App.Writer, method, args.Slice()[3:]...) - } - var result interface{} - params := make([]interface{}, len(args.Slice()[3:])) - for i, v := range args.Slice()[3:] { - params[i] = v - } - if err := rpcClient.Call(&result, method, params...); err != nil { - return err - } - return json.NewEncoder(ctx.App.Writer).Encode(result) -} - -func rpcSubscribe(client *rpc.Client, out io.Writer, method string, args ...string) error { - namespace, method, _ := strings.Cut(method, "_") - ch := make(chan interface{}) - subArgs := make([]interface{}, len(args)+1) - subArgs[0] = method - for i, v := range args { - subArgs[i+1] = v - } - sub, err := client.Subscribe(context.Background(), namespace, ch, subArgs...) - if err != nil { - return err - } - defer sub.Unsubscribe() - enc := json.NewEncoder(out) - for { - select { - case v := <-ch: - if err := enc.Encode(v); err != nil { - return err - } - case err := <-sub.Err(): - return err - } - } -} diff --git a/p2p/simulations/pipes/pipes.go b/p2p/pipes/pipe.go similarity index 85% rename from p2p/simulations/pipes/pipes.go rename to p2p/pipes/pipe.go index ec277c0d14..cf1f3e2a80 100644 --- a/p2p/simulations/pipes/pipes.go +++ b/p2p/pipes/pipe.go @@ -1,4 +1,4 @@ -// Copyright 2018 The go-ethereum Authors +// Copyright 2024 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -16,17 +16,9 @@ package pipes -import ( - "net" -) +import "net" -// NetPipe wraps net.Pipe in a signature returning an error -func NetPipe() (net.Conn, net.Conn, error) { - p1, p2 := net.Pipe() - return p1, p2, nil -} - -// TCPPipe creates an in process full duplex pipe based on a localhost TCP socket +// TCPPipe creates an in process full duplex pipe based on a localhost TCP socket. func TCPPipe() (net.Conn, net.Conn, error) { l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { diff --git a/p2p/rlpx/rlpx_test.go b/p2p/rlpx/rlpx_test.go index 136cb1b5bf..27d51546e7 100644 --- a/p2p/rlpx/rlpx_test.go +++ b/p2p/rlpx/rlpx_test.go @@ -31,7 +31,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/ecies" - "github.com/ethereum/go-ethereum/p2p/simulations/pipes" + "github.com/ethereum/go-ethereum/p2p/pipes" "github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/assert" ) diff --git a/p2p/simulations/README.md b/p2p/simulations/README.md deleted file mode 100644 index 1f9f72dcda..0000000000 --- a/p2p/simulations/README.md +++ /dev/null @@ -1,174 +0,0 @@ -# devp2p Simulations - -The `p2p/simulations` package implements a simulation framework that supports -creating a collection of devp2p nodes, connecting them to form a -simulation network, performing simulation actions in that network and then -extracting useful information. - -## Nodes - -Each node in a simulation network runs multiple services by wrapping a collection -of objects which implement the `node.Service` interface meaning they: - -* can be started and stopped -* run p2p protocols -* expose RPC APIs - -This means that any object which implements the `node.Service` interface can be -used to run a node in the simulation. - -## Services - -Before running a simulation, a set of service initializers must be registered -which can then be used to run nodes in the network. - -A service initializer is a function with the following signature: - -```go -func(ctx *adapters.ServiceContext) (node.Service, error) -``` - -These initializers should be registered by calling the `adapters.RegisterServices` -function in an `init()` hook: - -```go -func init() { - adapters.RegisterServices(adapters.Services{ - "service1": initService1, - "service2": initService2, - }) -} -``` - -## Node Adapters - -The simulation framework includes multiple "node adapters" which are -responsible for creating an environment in which a node runs. - -### SimAdapter - -The `SimAdapter` runs nodes in-memory, connecting them using an in-memory, -synchronous `net.Pipe` and connecting to their RPC server using an in-memory -`rpc.Client`. - -### ExecAdapter - -The `ExecAdapter` runs nodes as child processes of the running simulation. - -It does this by executing the binary which is running the simulation but -setting `argv[0]` (i.e. the program name) to `p2p-node` which is then -detected by an init hook in the child process which runs the `node.Service` -using the devp2p node stack rather than executing `main()`. - -The nodes listen for devp2p connections and WebSocket RPC clients on random -localhost ports. - -## Network - -A simulation network is created with an ID and default service. The default -service is used if a node is created without an explicit service. The -network has exposed methods for creating, starting, stopping, connecting -and disconnecting nodes. It also emits events when certain actions occur. - -### Events - -A simulation network emits the following events: - -* node event - when nodes are created / started / stopped -* connection event - when nodes are connected / disconnected -* message event - when a protocol message is sent between two nodes - -The events have a "control" flag which when set indicates that the event is the -outcome of a controlled simulation action (e.g. creating a node or explicitly -connecting two nodes). - -This is in contrast to a non-control event, otherwise called a "live" event, -which is the outcome of something happening in the network as a result of a -control event (e.g. a node actually started up or a connection was actually -established between two nodes). - -Live events are detected by the simulation network by subscribing to node peer -events via RPC when the nodes start up. - -## Testing Framework - -The `Simulation` type can be used in tests to perform actions in a simulation -network and then wait for expectations to be met. - -With a running simulation network, the `Simulation.Run` method can be called -with a `Step` which has the following fields: - -* `Action` - a function that performs some action in the network - -* `Expect` - an expectation function which returns whether or not a - given node meets the expectation - -* `Trigger` - a channel that receives node IDs which then trigger a check - of the expectation function to be performed against that node - -As a concrete example, consider a simulated network of Ethereum nodes. An -`Action` could be the sending of a transaction, `Expect` it being included in -a block, and `Trigger` a check for every block that is mined. - -On return, the `Simulation.Run` method returns a `StepResult` which can be used -to determine if all nodes met the expectation, how long it took them to meet -the expectation and what network events were emitted during the step run. - -## HTTP API - -The simulation framework includes a HTTP API that can be used to control the -simulation. - -The API is initialised with a particular node adapter and has the following -endpoints: - -``` -OPTIONS / Response 200 with "Access-Control-Allow-Headers"" header set to "Content-Type"" -GET / Get network information -POST /start Start all nodes in the network -POST /stop Stop all nodes in the network -POST /mocker/start Start the mocker node simulation -POST /mocker/stop Stop the mocker node simulation -GET /mocker Get a list of available mockers -POST /reset Reset all properties of a network to initial (empty) state -GET /events Stream network events -GET /snapshot Take a network snapshot -POST /snapshot Load a network snapshot -POST /nodes Create a node -GET /nodes Get all nodes in the network -GET /nodes/:nodeid Get node information -POST /nodes/:nodeid/start Start a node -POST /nodes/:nodeid/stop Stop a node -POST /nodes/:nodeid/conn/:peerid Connect two nodes -DELETE /nodes/:nodeid/conn/:peerid Disconnect two nodes -GET /nodes/:nodeid/rpc Make RPC requests to a node via WebSocket -``` - -For convenience, `nodeid` in the URL can be the name of a node rather than its -ID. - -## Command line client - -`p2psim` is a command line client for the HTTP API, located in -`cmd/p2psim`. - -It provides the following commands: - -``` -p2psim show -p2psim events [--current] [--filter=FILTER] -p2psim snapshot -p2psim load -p2psim node create [--name=NAME] [--services=SERVICES] [--key=KEY] -p2psim node list -p2psim node show -p2psim node start -p2psim node stop -p2psim node connect -p2psim node disconnect -p2psim node rpc [] [--subscribe] -``` - -## Example - -See [p2p/simulations/examples/README.md](examples/README.md). diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go deleted file mode 100644 index 6307b90bf8..0000000000 --- a/p2p/simulations/adapters/exec.go +++ /dev/null @@ -1,567 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package adapters - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "log/slog" - "net" - "net/http" - "os" - "os/exec" - "os/signal" - "path/filepath" - "strings" - "sync" - "syscall" - "time" - - "github.com/ethereum/go-ethereum/internal/reexec" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/rpc" - "github.com/gorilla/websocket" -) - -func init() { - // Register a reexec function to start a simulation node when the current binary is - // executed as "p2p-node" (rather than whatever the main() function would normally do). - reexec.Register("p2p-node", execP2PNode) -} - -// ExecAdapter is a NodeAdapter which runs simulation nodes by executing the current binary -// as a child process. -type ExecAdapter struct { - // BaseDir is the directory under which the data directories for each - // simulation node are created. - BaseDir string - - nodes map[enode.ID]*ExecNode -} - -// NewExecAdapter returns an ExecAdapter which stores node data in -// subdirectories of the given base directory -func NewExecAdapter(baseDir string) *ExecAdapter { - return &ExecAdapter{ - BaseDir: baseDir, - nodes: make(map[enode.ID]*ExecNode), - } -} - -// Name returns the name of the adapter for logging purposes -func (e *ExecAdapter) Name() string { - return "exec-adapter" -} - -// NewNode returns a new ExecNode using the given config -func (e *ExecAdapter) NewNode(config *NodeConfig) (Node, error) { - if len(config.Lifecycles) == 0 { - return nil, errors.New("node must have at least one service lifecycle") - } - for _, service := range config.Lifecycles { - if _, exists := lifecycleConstructorFuncs[service]; !exists { - return nil, fmt.Errorf("unknown node service %q", service) - } - } - - // create the node directory using the first 12 characters of the ID - // as Unix socket paths cannot be longer than 256 characters - dir := filepath.Join(e.BaseDir, config.ID.String()[:12]) - if err := os.Mkdir(dir, 0755); err != nil { - return nil, fmt.Errorf("error creating node directory: %s", err) - } - - err := config.initDummyEnode() - if err != nil { - return nil, err - } - - // generate the config - conf := &execNodeConfig{ - Stack: node.DefaultConfig, - Node: config, - } - if config.DataDir != "" { - conf.Stack.DataDir = config.DataDir - } else { - conf.Stack.DataDir = filepath.Join(dir, "data") - } - - // these parameters are crucial for execadapter node to run correctly - conf.Stack.WSHost = "127.0.0.1" - conf.Stack.WSPort = 0 - conf.Stack.WSOrigins = []string{"*"} - conf.Stack.WSExposeAll = true - conf.Stack.P2P.EnableMsgEvents = config.EnableMsgEvents - conf.Stack.P2P.NoDiscovery = true - conf.Stack.P2P.NAT = nil - - // Listen on a localhost port, which we set when we - // initialise NodeConfig (usually a random port) - conf.Stack.P2P.ListenAddr = fmt.Sprintf(":%d", config.Port) - - node := &ExecNode{ - ID: config.ID, - Dir: dir, - Config: conf, - adapter: e, - } - node.newCmd = node.execCommand - e.nodes[node.ID] = node - return node, nil -} - -// ExecNode starts a simulation node by exec'ing the current binary and -// running the configured services -type ExecNode struct { - ID enode.ID - Dir string - Config *execNodeConfig - Cmd *exec.Cmd - Info *p2p.NodeInfo - - adapter *ExecAdapter - client *rpc.Client - wsAddr string - newCmd func() *exec.Cmd -} - -// Addr returns the node's enode URL -func (n *ExecNode) Addr() []byte { - if n.Info == nil { - return nil - } - return []byte(n.Info.Enode) -} - -// Client returns an rpc.Client which can be used to communicate with the -// underlying services (it is set once the node has started) -func (n *ExecNode) Client() (*rpc.Client, error) { - return n.client, nil -} - -// Start exec's the node passing the ID and service as command line arguments -// and the node config encoded as JSON in an environment variable. -func (n *ExecNode) Start(snapshots map[string][]byte) (err error) { - if n.Cmd != nil { - return errors.New("already started") - } - defer func() { - if err != nil { - n.Stop() - } - }() - - // encode a copy of the config containing the snapshot - confCopy := *n.Config - confCopy.Snapshots = snapshots - confCopy.PeerAddrs = make(map[string]string) - for id, node := range n.adapter.nodes { - confCopy.PeerAddrs[id.String()] = node.wsAddr - } - confData, err := json.Marshal(confCopy) - if err != nil { - return fmt.Errorf("error generating node config: %s", err) - } - // expose the admin namespace via websocket if it's not enabled - exposed := confCopy.Stack.WSExposeAll - if !exposed { - for _, api := range confCopy.Stack.WSModules { - if api == "admin" { - exposed = true - break - } - } - } - if !exposed { - confCopy.Stack.WSModules = append(confCopy.Stack.WSModules, "admin") - } - // start the one-shot server that waits for startup information - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - statusURL, statusC := n.waitForStartupJSON(ctx) - - // start the node - cmd := n.newCmd() - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Env = append(os.Environ(), - envStatusURL+"="+statusURL, - envNodeConfig+"="+string(confData), - ) - if err := cmd.Start(); err != nil { - return fmt.Errorf("error starting node: %s", err) - } - n.Cmd = cmd - - // Wait for the node to start. - status := <-statusC - if status.Err != "" { - return errors.New(status.Err) - } - client, err := rpc.DialWebsocket(ctx, status.WSEndpoint, "") - if err != nil { - return fmt.Errorf("can't connect to RPC server: %v", err) - } - - // Node ready :) - n.client = client - n.wsAddr = status.WSEndpoint - n.Info = status.NodeInfo - return nil -} - -// waitForStartupJSON runs a one-shot HTTP server to receive a startup report. -func (n *ExecNode) waitForStartupJSON(ctx context.Context) (string, chan nodeStartupJSON) { - var ( - ch = make(chan nodeStartupJSON, 1) - quitOnce sync.Once - srv http.Server - ) - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - ch <- nodeStartupJSON{Err: err.Error()} - return "", ch - } - quit := func(status nodeStartupJSON) { - quitOnce.Do(func() { - l.Close() - ch <- status - }) - } - srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var status nodeStartupJSON - if err := json.NewDecoder(r.Body).Decode(&status); err != nil { - status.Err = fmt.Sprintf("can't decode startup report: %v", err) - } - quit(status) - }) - // Run the HTTP server, but don't wait forever and shut it down - // if the context is canceled. - go srv.Serve(l) - go func() { - <-ctx.Done() - quit(nodeStartupJSON{Err: "didn't get startup report"}) - }() - - url := "http://" + l.Addr().String() - return url, ch -} - -// execCommand returns a command which runs the node locally by exec'ing -// the current binary but setting argv[0] to "p2p-node" so that the child -// runs execP2PNode -func (n *ExecNode) execCommand() *exec.Cmd { - return &exec.Cmd{ - Path: reexec.Self(), - Args: []string{"p2p-node", strings.Join(n.Config.Node.Lifecycles, ","), n.ID.String()}, - } -} - -// Stop stops the node by first sending SIGTERM and then SIGKILL if the node -// doesn't stop within 5s -func (n *ExecNode) Stop() error { - if n.Cmd == nil { - return nil - } - defer func() { - n.Cmd = nil - }() - - if n.client != nil { - n.client.Close() - n.client = nil - n.wsAddr = "" - n.Info = nil - } - - if err := n.Cmd.Process.Signal(syscall.SIGTERM); err != nil { - return n.Cmd.Process.Kill() - } - waitErr := make(chan error, 1) - go func() { - waitErr <- n.Cmd.Wait() - }() - timer := time.NewTimer(5 * time.Second) - defer timer.Stop() - - select { - case err := <-waitErr: - return err - case <-timer.C: - return n.Cmd.Process.Kill() - } -} - -// NodeInfo returns information about the node -func (n *ExecNode) NodeInfo() *p2p.NodeInfo { - info := &p2p.NodeInfo{ - ID: n.ID.String(), - } - if n.client != nil { - n.client.Call(&info, "admin_nodeInfo") - } - return info -} - -// ServeRPC serves RPC requests over the given connection by dialling the -// node's WebSocket address and joining the two connections -func (n *ExecNode) ServeRPC(clientConn *websocket.Conn) error { - conn, _, err := websocket.DefaultDialer.Dial(n.wsAddr, nil) - if err != nil { - return err - } - var wg sync.WaitGroup - wg.Add(2) - go wsCopy(&wg, conn, clientConn) - go wsCopy(&wg, clientConn, conn) - wg.Wait() - conn.Close() - return nil -} - -func wsCopy(wg *sync.WaitGroup, src, dst *websocket.Conn) { - defer wg.Done() - for { - msgType, r, err := src.NextReader() - if err != nil { - return - } - w, err := dst.NextWriter(msgType) - if err != nil { - return - } - if _, err = io.Copy(w, r); err != nil { - return - } - } -} - -// Snapshots creates snapshots of the services by calling the -// simulation_snapshot RPC method -func (n *ExecNode) Snapshots() (map[string][]byte, error) { - if n.client == nil { - return nil, errors.New("RPC not started") - } - var snapshots map[string][]byte - return snapshots, n.client.Call(&snapshots, "simulation_snapshot") -} - -// execNodeConfig is used to serialize the node configuration so it can be -// passed to the child process as a JSON encoded environment variable -type execNodeConfig struct { - Stack node.Config `json:"stack"` - Node *NodeConfig `json:"node"` - Snapshots map[string][]byte `json:"snapshots,omitempty"` - PeerAddrs map[string]string `json:"peer_addrs,omitempty"` -} - -func initLogging() { - // Initialize the logging by default first. - var innerHandler slog.Handler - innerHandler = slog.NewTextHandler(os.Stderr, nil) - glogger := log.NewGlogHandler(innerHandler) - glogger.Verbosity(log.LevelInfo) - log.SetDefault(log.NewLogger(glogger)) - - confEnv := os.Getenv(envNodeConfig) - if confEnv == "" { - return - } - var conf execNodeConfig - if err := json.Unmarshal([]byte(confEnv), &conf); err != nil { - return - } - var writer = os.Stderr - if conf.Node.LogFile != "" { - logWriter, err := os.Create(conf.Node.LogFile) - if err != nil { - return - } - writer = logWriter - } - var verbosity = log.LevelInfo - if conf.Node.LogVerbosity <= log.LevelTrace && conf.Node.LogVerbosity >= log.LevelCrit { - verbosity = log.FromLegacyLevel(int(conf.Node.LogVerbosity)) - } - // Reinitialize the logger - innerHandler = log.NewTerminalHandler(writer, true) - glogger = log.NewGlogHandler(innerHandler) - glogger.Verbosity(verbosity) - log.SetDefault(log.NewLogger(glogger)) -} - -// execP2PNode starts a simulation node when the current binary is executed with -// argv[0] being "p2p-node", reading the service / ID from argv[1] / argv[2] -// and the node config from an environment variable. -func execP2PNode() { - initLogging() - - statusURL := os.Getenv(envStatusURL) - if statusURL == "" { - log.Crit("missing " + envStatusURL) - } - - // Start the node and gather startup report. - var status nodeStartupJSON - stack, stackErr := startExecNodeStack() - if stackErr != nil { - status.Err = stackErr.Error() - } else { - status.WSEndpoint = stack.WSEndpoint() - status.NodeInfo = stack.Server().NodeInfo() - } - - // Send status to the host. - statusJSON, _ := json.Marshal(status) - resp, err := http.Post(statusURL, "application/json", bytes.NewReader(statusJSON)) - if err != nil { - log.Crit("Can't post startup info", "url", statusURL, "err", err) - } - resp.Body.Close() - if stackErr != nil { - os.Exit(1) - } - - // Stop the stack if we get a SIGTERM signal. - go func() { - sigc := make(chan os.Signal, 1) - signal.Notify(sigc, syscall.SIGTERM) - defer signal.Stop(sigc) - <-sigc - log.Info("Received SIGTERM, shutting down...") - stack.Close() - }() - stack.Wait() // Wait for the stack to exit. -} - -func startExecNodeStack() (*node.Node, error) { - // read the services from argv - serviceNames := strings.Split(os.Args[1], ",") - - // decode the config - confEnv := os.Getenv(envNodeConfig) - if confEnv == "" { - return nil, errors.New("missing " + envNodeConfig) - } - var conf execNodeConfig - if err := json.Unmarshal([]byte(confEnv), &conf); err != nil { - return nil, fmt.Errorf("error decoding %s: %v", envNodeConfig, err) - } - - // create enode record - nodeTcpConn, _ := net.ResolveTCPAddr("tcp", conf.Stack.P2P.ListenAddr) - if nodeTcpConn.IP == nil { - nodeTcpConn.IP = net.IPv4(127, 0, 0, 1) - } - conf.Node.initEnode(nodeTcpConn.IP, nodeTcpConn.Port, nodeTcpConn.Port) - conf.Stack.P2P.PrivateKey = conf.Node.PrivateKey - conf.Stack.Logger = log.New("node.id", conf.Node.ID.String()) - - // initialize the devp2p stack - stack, err := node.New(&conf.Stack) - if err != nil { - return nil, fmt.Errorf("error creating node stack: %v", err) - } - - // Register the services, collecting them into a map so they can - // be accessed by the snapshot API. - services := make(map[string]node.Lifecycle, len(serviceNames)) - for _, name := range serviceNames { - lifecycleFunc, exists := lifecycleConstructorFuncs[name] - if !exists { - return nil, fmt.Errorf("unknown node service %q", err) - } - ctx := &ServiceContext{ - RPCDialer: &wsRPCDialer{addrs: conf.PeerAddrs}, - Config: conf.Node, - } - if conf.Snapshots != nil { - ctx.Snapshot = conf.Snapshots[name] - } - service, err := lifecycleFunc(ctx, stack) - if err != nil { - return nil, err - } - services[name] = service - } - - // Add the snapshot API. - stack.RegisterAPIs([]rpc.API{{ - Namespace: "simulation", - Service: SnapshotAPI{services}, - }}) - - if err = stack.Start(); err != nil { - err = fmt.Errorf("error starting stack: %v", err) - } - return stack, err -} - -const ( - envStatusURL = "_P2P_STATUS_URL" - envNodeConfig = "_P2P_NODE_CONFIG" -) - -// nodeStartupJSON is sent to the simulation host after startup. -type nodeStartupJSON struct { - Err string - WSEndpoint string - NodeInfo *p2p.NodeInfo -} - -// SnapshotAPI provides an RPC method to create snapshots of services -type SnapshotAPI struct { - services map[string]node.Lifecycle -} - -func (api SnapshotAPI) Snapshot() (map[string][]byte, error) { - snapshots := make(map[string][]byte) - for name, service := range api.services { - if s, ok := service.(interface { - Snapshot() ([]byte, error) - }); ok { - snap, err := s.Snapshot() - if err != nil { - return nil, err - } - snapshots[name] = snap - } - } - return snapshots, nil -} - -type wsRPCDialer struct { - addrs map[string]string -} - -// DialRPC implements the RPCDialer interface by creating a WebSocket RPC -// client of the given node -func (w *wsRPCDialer) DialRPC(id enode.ID) (*rpc.Client, error) { - addr, ok := w.addrs[id.String()] - if !ok { - return nil, fmt.Errorf("unknown node: %s", id) - } - return rpc.DialWebsocket(context.Background(), addr, "http://localhost") -} diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go deleted file mode 100644 index 0efe9744a5..0000000000 --- a/p2p/simulations/adapters/inproc.go +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package adapters - -import ( - "context" - "errors" - "fmt" - "maps" - "math" - "net" - "sync" - - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/simulations/pipes" - "github.com/ethereum/go-ethereum/rpc" - "github.com/gorilla/websocket" -) - -// SimAdapter is a NodeAdapter which creates in-memory simulation nodes and -// connects them using net.Pipe -type SimAdapter struct { - pipe func() (net.Conn, net.Conn, error) - mtx sync.RWMutex - nodes map[enode.ID]*SimNode - lifecycles LifecycleConstructors -} - -// NewSimAdapter creates a SimAdapter which is capable of running in-memory -// simulation nodes running any of the given services (the services to run on a -// particular node are passed to the NewNode function in the NodeConfig) -// the adapter uses a net.Pipe for in-memory simulated network connections -func NewSimAdapter(services LifecycleConstructors) *SimAdapter { - return &SimAdapter{ - pipe: pipes.NetPipe, - nodes: make(map[enode.ID]*SimNode), - lifecycles: services, - } -} - -// Name returns the name of the adapter for logging purposes -func (s *SimAdapter) Name() string { - return "sim-adapter" -} - -// NewNode returns a new SimNode using the given config -func (s *SimAdapter) NewNode(config *NodeConfig) (Node, error) { - s.mtx.Lock() - defer s.mtx.Unlock() - - id := config.ID - // verify that the node has a private key in the config - if config.PrivateKey == nil { - return nil, fmt.Errorf("node is missing private key: %s", id) - } - - // check a node with the ID doesn't already exist - if _, exists := s.nodes[id]; exists { - return nil, fmt.Errorf("node already exists: %s", id) - } - - // check the services are valid - if len(config.Lifecycles) == 0 { - return nil, errors.New("node must have at least one service") - } - for _, service := range config.Lifecycles { - if _, exists := s.lifecycles[service]; !exists { - return nil, fmt.Errorf("unknown node service %q", service) - } - } - - err := config.initDummyEnode() - if err != nil { - return nil, err - } - - n, err := node.New(&node.Config{ - P2P: p2p.Config{ - PrivateKey: config.PrivateKey, - MaxPeers: math.MaxInt32, - NoDiscovery: true, - Dialer: s, - EnableMsgEvents: config.EnableMsgEvents, - }, - ExternalSigner: config.ExternalSigner, - Logger: log.New("node.id", id.String()), - }) - if err != nil { - return nil, err - } - - simNode := &SimNode{ - ID: id, - config: config, - node: n, - adapter: s, - running: make(map[string]node.Lifecycle), - } - s.nodes[id] = simNode - return simNode, nil -} - -// Dial implements the p2p.NodeDialer interface by connecting to the node using -// an in-memory net.Pipe -func (s *SimAdapter) Dial(ctx context.Context, dest *enode.Node) (conn net.Conn, err error) { - node, ok := s.GetNode(dest.ID()) - if !ok { - return nil, fmt.Errorf("unknown node: %s", dest.ID()) - } - srv := node.Server() - if srv == nil { - return nil, fmt.Errorf("node not running: %s", dest.ID()) - } - // SimAdapter.pipe is net.Pipe (NewSimAdapter) - pipe1, pipe2, err := s.pipe() - if err != nil { - return nil, err - } - // this is simulated 'listening' - // asynchronously call the dialed destination node's p2p server - // to set up connection on the 'listening' side - go srv.SetupConn(pipe1, 0, nil) - return pipe2, nil -} - -// DialRPC implements the RPCDialer interface by creating an in-memory RPC -// client of the given node -func (s *SimAdapter) DialRPC(id enode.ID) (*rpc.Client, error) { - node, ok := s.GetNode(id) - if !ok { - return nil, fmt.Errorf("unknown node: %s", id) - } - return node.node.Attach(), nil -} - -// GetNode returns the node with the given ID if it exists -func (s *SimAdapter) GetNode(id enode.ID) (*SimNode, bool) { - s.mtx.RLock() - defer s.mtx.RUnlock() - node, ok := s.nodes[id] - return node, ok -} - -// SimNode is an in-memory simulation node which connects to other nodes using -// net.Pipe (see SimAdapter.Dial), running devp2p protocols directly over that -// pipe -type SimNode struct { - lock sync.RWMutex - ID enode.ID - config *NodeConfig - adapter *SimAdapter - node *node.Node - running map[string]node.Lifecycle - client *rpc.Client - registerOnce sync.Once -} - -// Close closes the underlying node.Node to release -// acquired resources. -func (sn *SimNode) Close() error { - return sn.node.Close() -} - -// Addr returns the node's discovery address -func (sn *SimNode) Addr() []byte { - return []byte(sn.Node().String()) -} - -// Node returns a node descriptor representing the SimNode -func (sn *SimNode) Node() *enode.Node { - return sn.config.Node() -} - -// Client returns an rpc.Client which can be used to communicate with the -// underlying services (it is set once the node has started) -func (sn *SimNode) Client() (*rpc.Client, error) { - sn.lock.RLock() - defer sn.lock.RUnlock() - if sn.client == nil { - return nil, errors.New("node not started") - } - return sn.client, nil -} - -// ServeRPC serves RPC requests over the given connection by creating an -// in-memory client to the node's RPC server. -func (sn *SimNode) ServeRPC(conn *websocket.Conn) error { - handler, err := sn.node.RPCHandler() - if err != nil { - return err - } - codec := rpc.NewFuncCodec(conn, func(v any, _ bool) error { return conn.WriteJSON(v) }, conn.ReadJSON) - handler.ServeCodec(codec, 0) - return nil -} - -// Snapshots creates snapshots of the services by calling the -// simulation_snapshot RPC method -func (sn *SimNode) Snapshots() (map[string][]byte, error) { - sn.lock.RLock() - services := maps.Clone(sn.running) - sn.lock.RUnlock() - if len(services) == 0 { - return nil, errors.New("no running services") - } - snapshots := make(map[string][]byte) - for name, service := range services { - if s, ok := service.(interface { - Snapshot() ([]byte, error) - }); ok { - snap, err := s.Snapshot() - if err != nil { - return nil, err - } - snapshots[name] = snap - } - } - return snapshots, nil -} - -// Start registers the services and starts the underlying devp2p node -func (sn *SimNode) Start(snapshots map[string][]byte) error { - // ensure we only register the services once in the case of the node - // being stopped and then started again - var regErr error - sn.registerOnce.Do(func() { - for _, name := range sn.config.Lifecycles { - ctx := &ServiceContext{ - RPCDialer: sn.adapter, - Config: sn.config, - } - if snapshots != nil { - ctx.Snapshot = snapshots[name] - } - serviceFunc := sn.adapter.lifecycles[name] - service, err := serviceFunc(ctx, sn.node) - if err != nil { - regErr = err - break - } - // if the service has already been registered, don't register it again. - if _, ok := sn.running[name]; ok { - continue - } - sn.running[name] = service - } - }) - if regErr != nil { - return regErr - } - - if err := sn.node.Start(); err != nil { - return err - } - - // create an in-process RPC client - client := sn.node.Attach() - sn.lock.Lock() - sn.client = client - sn.lock.Unlock() - - return nil -} - -// Stop closes the RPC client and stops the underlying devp2p node -func (sn *SimNode) Stop() error { - sn.lock.Lock() - if sn.client != nil { - sn.client.Close() - sn.client = nil - } - sn.lock.Unlock() - return sn.node.Close() -} - -// Service returns a running service by name -func (sn *SimNode) Service(name string) node.Lifecycle { - sn.lock.RLock() - defer sn.lock.RUnlock() - return sn.running[name] -} - -// Services returns a copy of the underlying services -func (sn *SimNode) Services() []node.Lifecycle { - sn.lock.RLock() - defer sn.lock.RUnlock() - services := make([]node.Lifecycle, 0, len(sn.running)) - for _, service := range sn.running { - services = append(services, service) - } - return services -} - -// ServiceMap returns a map by names of the underlying services -func (sn *SimNode) ServiceMap() map[string]node.Lifecycle { - sn.lock.RLock() - defer sn.lock.RUnlock() - return maps.Clone(sn.running) -} - -// Server returns the underlying p2p.Server -func (sn *SimNode) Server() *p2p.Server { - return sn.node.Server() -} - -// SubscribeEvents subscribes the given channel to peer events from the -// underlying p2p.Server -func (sn *SimNode) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription { - srv := sn.Server() - if srv == nil { - panic("node not running") - } - return srv.SubscribeEvents(ch) -} - -// NodeInfo returns information about the node -func (sn *SimNode) NodeInfo() *p2p.NodeInfo { - server := sn.Server() - if server == nil { - return &p2p.NodeInfo{ - ID: sn.ID.String(), - Enode: sn.Node().String(), - } - } - return server.NodeInfo() -} diff --git a/p2p/simulations/adapters/inproc_test.go b/p2p/simulations/adapters/inproc_test.go deleted file mode 100644 index d0539ca867..0000000000 --- a/p2p/simulations/adapters/inproc_test.go +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package adapters - -import ( - "bytes" - "encoding/binary" - "fmt" - "sync" - "testing" - - "github.com/ethereum/go-ethereum/p2p/simulations/pipes" -) - -func TestTCPPipe(t *testing.T) { - c1, c2, err := pipes.TCPPipe() - if err != nil { - t.Fatal(err) - } - - msgs := 50 - size := 1024 - for i := 0; i < msgs; i++ { - msg := make([]byte, size) - binary.PutUvarint(msg, uint64(i)) - if _, err := c1.Write(msg); err != nil { - t.Fatal(err) - } - } - - for i := 0; i < msgs; i++ { - msg := make([]byte, size) - binary.PutUvarint(msg, uint64(i)) - out := make([]byte, size) - if _, err := c2.Read(out); err != nil { - t.Fatal(err) - } - if !bytes.Equal(msg, out) { - t.Fatalf("expected %#v, got %#v", msg, out) - } - } -} - -func TestTCPPipeBidirections(t *testing.T) { - c1, c2, err := pipes.TCPPipe() - if err != nil { - t.Fatal(err) - } - - msgs := 50 - size := 7 - for i := 0; i < msgs; i++ { - msg := []byte(fmt.Sprintf("ping %02d", i)) - if _, err := c1.Write(msg); err != nil { - t.Fatal(err) - } - } - - for i := 0; i < msgs; i++ { - expected := []byte(fmt.Sprintf("ping %02d", i)) - out := make([]byte, size) - if _, err := c2.Read(out); err != nil { - t.Fatal(err) - } - - if !bytes.Equal(expected, out) { - t.Fatalf("expected %#v, got %#v", expected, out) - } else { - msg := []byte(fmt.Sprintf("pong %02d", i)) - if _, err := c2.Write(msg); err != nil { - t.Fatal(err) - } - } - } - - for i := 0; i < msgs; i++ { - expected := []byte(fmt.Sprintf("pong %02d", i)) - out := make([]byte, size) - if _, err := c1.Read(out); err != nil { - t.Fatal(err) - } - if !bytes.Equal(expected, out) { - t.Fatalf("expected %#v, got %#v", expected, out) - } - } -} - -func TestNetPipe(t *testing.T) { - c1, c2, err := pipes.NetPipe() - if err != nil { - t.Fatal(err) - } - - msgs := 50 - size := 1024 - var wg sync.WaitGroup - defer wg.Wait() - - // netPipe is blocking, so writes are emitted asynchronously - wg.Add(1) - go func() { - defer wg.Done() - - for i := 0; i < msgs; i++ { - msg := make([]byte, size) - binary.PutUvarint(msg, uint64(i)) - if _, err := c1.Write(msg); err != nil { - t.Error(err) - } - } - }() - - for i := 0; i < msgs; i++ { - msg := make([]byte, size) - binary.PutUvarint(msg, uint64(i)) - out := make([]byte, size) - if _, err := c2.Read(out); err != nil { - t.Error(err) - } - if !bytes.Equal(msg, out) { - t.Errorf("expected %#v, got %#v", msg, out) - } - } -} - -func TestNetPipeBidirections(t *testing.T) { - c1, c2, err := pipes.NetPipe() - if err != nil { - t.Fatal(err) - } - - msgs := 1000 - size := 8 - pingTemplate := "ping %03d" - pongTemplate := "pong %03d" - var wg sync.WaitGroup - defer wg.Wait() - - // netPipe is blocking, so writes are emitted asynchronously - wg.Add(1) - go func() { - defer wg.Done() - - for i := 0; i < msgs; i++ { - msg := []byte(fmt.Sprintf(pingTemplate, i)) - if _, err := c1.Write(msg); err != nil { - t.Error(err) - } - } - }() - - // netPipe is blocking, so reads for pong are emitted asynchronously - wg.Add(1) - go func() { - defer wg.Done() - - for i := 0; i < msgs; i++ { - expected := []byte(fmt.Sprintf(pongTemplate, i)) - out := make([]byte, size) - if _, err := c1.Read(out); err != nil { - t.Error(err) - } - if !bytes.Equal(expected, out) { - t.Errorf("expected %#v, got %#v", expected, out) - } - } - }() - - // expect to read pings, and respond with pongs to the alternate connection - for i := 0; i < msgs; i++ { - expected := []byte(fmt.Sprintf(pingTemplate, i)) - - out := make([]byte, size) - _, err := c2.Read(out) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(expected, out) { - t.Errorf("expected %#v, got %#v", expected, out) - } else { - msg := []byte(fmt.Sprintf(pongTemplate, i)) - if _, err := c2.Write(msg); err != nil { - t.Fatal(err) - } - } - } -} diff --git a/p2p/simulations/adapters/types.go b/p2p/simulations/adapters/types.go deleted file mode 100644 index e18aaacc33..0000000000 --- a/p2p/simulations/adapters/types.go +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package adapters - -import ( - "crypto/ecdsa" - "encoding/hex" - "encoding/json" - "fmt" - "log/slog" - "net" - "os" - "strconv" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/internal/reexec" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/enr" - "github.com/ethereum/go-ethereum/rpc" - "github.com/gorilla/websocket" -) - -// Node represents a node in a simulation network which is created by a -// NodeAdapter, for example: -// -// - SimNode, an in-memory node in the same process -// - ExecNode, a child process node -type Node interface { - // Addr returns the node's address (e.g. an Enode URL) - Addr() []byte - - // Client returns the RPC client which is created once the node is - // up and running - Client() (*rpc.Client, error) - - // ServeRPC serves RPC requests over the given connection - ServeRPC(*websocket.Conn) error - - // Start starts the node with the given snapshots - Start(snapshots map[string][]byte) error - - // Stop stops the node - Stop() error - - // NodeInfo returns information about the node - NodeInfo() *p2p.NodeInfo - - // Snapshots creates snapshots of the running services - Snapshots() (map[string][]byte, error) -} - -// NodeAdapter is used to create Nodes in a simulation network -type NodeAdapter interface { - // Name returns the name of the adapter for logging purposes - Name() string - - // NewNode creates a new node with the given configuration - NewNode(config *NodeConfig) (Node, error) -} - -// NodeConfig is the configuration used to start a node in a simulation -// network -type NodeConfig struct { - // ID is the node's ID which is used to identify the node in the - // simulation network - ID enode.ID - - // PrivateKey is the node's private key which is used by the devp2p - // stack to encrypt communications - PrivateKey *ecdsa.PrivateKey - - // Enable peer events for Msgs - EnableMsgEvents bool - - // Name is a human friendly name for the node like "node01" - Name string - - // Use an existing database instead of a temporary one if non-empty - DataDir string - - // Lifecycles are the names of the service lifecycles which should be run when - // starting the node (for SimNodes it should be the names of service lifecycles - // contained in SimAdapter.lifecycles, for other nodes it should be - // service lifecycles registered by calling the RegisterLifecycle function) - Lifecycles []string - - // Properties are the names of the properties this node should hold - // within running services (e.g. "bootnode", "lightnode" or any custom values) - // These values need to be checked and acted upon by node Services - Properties []string - - // ExternalSigner specifies an external URI for a clef-type signer - ExternalSigner string - - // Enode - node *enode.Node - - // ENR Record with entries to overwrite - Record enr.Record - - // function to sanction or prevent suggesting a peer - Reachable func(id enode.ID) bool - - Port uint16 - - // LogFile is the log file name of the p2p node at runtime. - // - // The default value is empty so that the default log writer - // is the system standard output. - LogFile string - - // LogVerbosity is the log verbosity of the p2p node at runtime. - // - // The default verbosity is INFO. - LogVerbosity slog.Level -} - -// nodeConfigJSON is used to encode and decode NodeConfig as JSON by encoding -// all fields as strings -type nodeConfigJSON struct { - ID string `json:"id"` - PrivateKey string `json:"private_key"` - Name string `json:"name"` - Lifecycles []string `json:"lifecycles"` - Properties []string `json:"properties"` - EnableMsgEvents bool `json:"enable_msg_events"` - Port uint16 `json:"port"` - LogFile string `json:"logfile"` - LogVerbosity int `json:"log_verbosity"` -} - -// MarshalJSON implements the json.Marshaler interface by encoding the config -// fields as strings -func (n *NodeConfig) MarshalJSON() ([]byte, error) { - confJSON := nodeConfigJSON{ - ID: n.ID.String(), - Name: n.Name, - Lifecycles: n.Lifecycles, - Properties: n.Properties, - Port: n.Port, - EnableMsgEvents: n.EnableMsgEvents, - LogFile: n.LogFile, - LogVerbosity: int(n.LogVerbosity), - } - if n.PrivateKey != nil { - confJSON.PrivateKey = hex.EncodeToString(crypto.FromECDSA(n.PrivateKey)) - } - return json.Marshal(confJSON) -} - -// UnmarshalJSON implements the json.Unmarshaler interface by decoding the json -// string values into the config fields -func (n *NodeConfig) UnmarshalJSON(data []byte) error { - var confJSON nodeConfigJSON - if err := json.Unmarshal(data, &confJSON); err != nil { - return err - } - - if confJSON.ID != "" { - if err := n.ID.UnmarshalText([]byte(confJSON.ID)); err != nil { - return err - } - } - - if confJSON.PrivateKey != "" { - key, err := hex.DecodeString(confJSON.PrivateKey) - if err != nil { - return err - } - privKey, err := crypto.ToECDSA(key) - if err != nil { - return err - } - n.PrivateKey = privKey - } - - n.Name = confJSON.Name - n.Lifecycles = confJSON.Lifecycles - n.Properties = confJSON.Properties - n.Port = confJSON.Port - n.EnableMsgEvents = confJSON.EnableMsgEvents - n.LogFile = confJSON.LogFile - n.LogVerbosity = slog.Level(confJSON.LogVerbosity) - - return nil -} - -// Node returns the node descriptor represented by the config. -func (n *NodeConfig) Node() *enode.Node { - return n.node -} - -// RandomNodeConfig returns node configuration with a randomly generated ID and -// PrivateKey -func RandomNodeConfig() *NodeConfig { - prvkey, err := crypto.GenerateKey() - if err != nil { - panic("unable to generate key") - } - - port, err := assignTCPPort() - if err != nil { - panic("unable to assign tcp port") - } - - enodId := enode.PubkeyToIDV4(&prvkey.PublicKey) - return &NodeConfig{ - PrivateKey: prvkey, - ID: enodId, - Name: fmt.Sprintf("node_%s", enodId.String()), - Port: port, - EnableMsgEvents: true, - LogVerbosity: log.LvlInfo, - } -} - -func assignTCPPort() (uint16, error) { - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - return 0, err - } - l.Close() - _, port, err := net.SplitHostPort(l.Addr().String()) - if err != nil { - return 0, err - } - p, err := strconv.ParseUint(port, 10, 16) - if err != nil { - return 0, err - } - return uint16(p), nil -} - -// ServiceContext is a collection of options and methods which can be utilised -// when starting services -type ServiceContext struct { - RPCDialer - - Config *NodeConfig - Snapshot []byte -} - -// RPCDialer is used when initialising services which need to connect to -// other nodes in the network (for example a simulated Swarm node which needs -// to connect to a Geth node to resolve ENS names) -type RPCDialer interface { - DialRPC(id enode.ID) (*rpc.Client, error) -} - -// LifecycleConstructor allows a Lifecycle to be constructed during node start-up. -// While the service-specific package usually takes care of Lifecycle creation and registration, -// for testing purposes, it is useful to be able to construct a Lifecycle on spot. -type LifecycleConstructor func(ctx *ServiceContext, stack *node.Node) (node.Lifecycle, error) - -// LifecycleConstructors stores LifecycleConstructor functions to call during node start-up. -type LifecycleConstructors map[string]LifecycleConstructor - -// lifecycleConstructorFuncs is a map of registered services which are used to boot devp2p -// nodes -var lifecycleConstructorFuncs = make(LifecycleConstructors) - -// RegisterLifecycles registers the given Services which can then be used to -// start devp2p nodes using either the Exec or Docker adapters. -// -// It should be called in an init function so that it has the opportunity to -// execute the services before main() is called. -func RegisterLifecycles(lifecycles LifecycleConstructors) { - for name, f := range lifecycles { - if _, exists := lifecycleConstructorFuncs[name]; exists { - panic(fmt.Sprintf("node service already exists: %q", name)) - } - lifecycleConstructorFuncs[name] = f - } - - // now we have registered the services, run reexec.Init() which will - // potentially start one of the services if the current binary has - // been exec'd with argv[0] set to "p2p-node" - if reexec.Init() { - os.Exit(0) - } -} - -// adds the host part to the configuration's ENR, signs it -// creates and adds the corresponding enode object to the configuration -func (n *NodeConfig) initEnode(ip net.IP, tcpport int, udpport int) error { - enrIp := enr.IP(ip) - n.Record.Set(&enrIp) - enrTcpPort := enr.TCP(tcpport) - n.Record.Set(&enrTcpPort) - enrUdpPort := enr.UDP(udpport) - n.Record.Set(&enrUdpPort) - - err := enode.SignV4(&n.Record, n.PrivateKey) - if err != nil { - return fmt.Errorf("unable to generate ENR: %v", err) - } - nod, err := enode.New(enode.V4ID{}, &n.Record) - if err != nil { - return fmt.Errorf("unable to create enode: %v", err) - } - log.Trace("simnode new", "record", n.Record) - n.node = nod - return nil -} - -func (n *NodeConfig) initDummyEnode() error { - return n.initEnode(net.IPv4(127, 0, 0, 1), int(n.Port), 0) -} diff --git a/p2p/simulations/connect.go b/p2p/simulations/connect.go deleted file mode 100644 index ede96b34c1..0000000000 --- a/p2p/simulations/connect.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package simulations - -import ( - "errors" - "strings" - - "github.com/ethereum/go-ethereum/p2p/enode" -) - -var ( - ErrNodeNotFound = errors.New("node not found") -) - -// ConnectToLastNode connects the node with provided NodeID -// to the last node that is up, and avoiding connection to self. -// It is useful when constructing a chain network topology -// when Network adds and removes nodes dynamically. -func (net *Network) ConnectToLastNode(id enode.ID) (err error) { - net.lock.Lock() - defer net.lock.Unlock() - - ids := net.getUpNodeIDs() - l := len(ids) - if l < 2 { - return nil - } - last := ids[l-1] - if last == id { - last = ids[l-2] - } - return net.connectNotConnected(last, id) -} - -// ConnectToRandomNode connects the node with provided NodeID -// to a random node that is up. -func (net *Network) ConnectToRandomNode(id enode.ID) (err error) { - net.lock.Lock() - defer net.lock.Unlock() - - selected := net.getRandomUpNode(id) - if selected == nil { - return ErrNodeNotFound - } - return net.connectNotConnected(selected.ID(), id) -} - -// ConnectNodesFull connects all nodes one to another. -// It provides a complete connectivity in the network -// which should be rarely needed. -func (net *Network) ConnectNodesFull(ids []enode.ID) (err error) { - net.lock.Lock() - defer net.lock.Unlock() - - if ids == nil { - ids = net.getUpNodeIDs() - } - for i, lid := range ids { - for _, rid := range ids[i+1:] { - if err = net.connectNotConnected(lid, rid); err != nil { - return err - } - } - } - return nil -} - -// ConnectNodesChain connects all nodes in a chain topology. -// If ids argument is nil, all nodes that are up will be connected. -func (net *Network) ConnectNodesChain(ids []enode.ID) (err error) { - net.lock.Lock() - defer net.lock.Unlock() - - return net.connectNodesChain(ids) -} - -func (net *Network) connectNodesChain(ids []enode.ID) (err error) { - if ids == nil { - ids = net.getUpNodeIDs() - } - l := len(ids) - for i := 0; i < l-1; i++ { - if err := net.connectNotConnected(ids[i], ids[i+1]); err != nil { - return err - } - } - return nil -} - -// ConnectNodesRing connects all nodes in a ring topology. -// If ids argument is nil, all nodes that are up will be connected. -func (net *Network) ConnectNodesRing(ids []enode.ID) (err error) { - net.lock.Lock() - defer net.lock.Unlock() - - if ids == nil { - ids = net.getUpNodeIDs() - } - l := len(ids) - if l < 2 { - return nil - } - if err := net.connectNodesChain(ids); err != nil { - return err - } - return net.connectNotConnected(ids[l-1], ids[0]) -} - -// ConnectNodesStar connects all nodes into a star topology -// If ids argument is nil, all nodes that are up will be connected. -func (net *Network) ConnectNodesStar(ids []enode.ID, center enode.ID) (err error) { - net.lock.Lock() - defer net.lock.Unlock() - - if ids == nil { - ids = net.getUpNodeIDs() - } - for _, id := range ids { - if center == id { - continue - } - if err := net.connectNotConnected(center, id); err != nil { - return err - } - } - return nil -} - -func (net *Network) connectNotConnected(oneID, otherID enode.ID) error { - return ignoreAlreadyConnectedErr(net.connect(oneID, otherID)) -} - -func ignoreAlreadyConnectedErr(err error) error { - if err == nil || strings.Contains(err.Error(), "already connected") { - return nil - } - return err -} diff --git a/p2p/simulations/connect_test.go b/p2p/simulations/connect_test.go deleted file mode 100644 index 0154a18b03..0000000000 --- a/p2p/simulations/connect_test.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package simulations - -import ( - "testing" - - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" -) - -func newTestNetwork(t *testing.T, nodeCount int) (*Network, []enode.ID) { - t.Helper() - adapter := adapters.NewSimAdapter(adapters.LifecycleConstructors{ - "noopwoop": func(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { - return NewNoopService(nil), nil - }, - }) - - // create network - network := NewNetwork(adapter, &NetworkConfig{ - DefaultService: "noopwoop", - }) - - // create and start nodes - ids := make([]enode.ID, nodeCount) - for i := range ids { - conf := adapters.RandomNodeConfig() - node, err := network.NewNodeWithConfig(conf) - if err != nil { - t.Fatalf("error creating node: %s", err) - } - if err := network.Start(node.ID()); err != nil { - t.Fatalf("error starting node: %s", err) - } - ids[i] = node.ID() - } - - if len(network.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - return network, ids -} - -func TestConnectToLastNode(t *testing.T) { - net, ids := newTestNetwork(t, 10) - defer net.Shutdown() - - first := ids[0] - if err := net.ConnectToLastNode(first); err != nil { - t.Fatal(err) - } - - last := ids[len(ids)-1] - for i, id := range ids { - if id == first || id == last { - continue - } - - if net.GetConn(first, id) != nil { - t.Errorf("connection must not exist with node(ind: %v, id: %v)", i, id) - } - } - - if net.GetConn(first, last) == nil { - t.Error("first and last node must be connected") - } -} - -func TestConnectToRandomNode(t *testing.T) { - net, ids := newTestNetwork(t, 10) - defer net.Shutdown() - - err := net.ConnectToRandomNode(ids[0]) - if err != nil { - t.Fatal(err) - } - - var cc int - for i, a := range ids { - for _, b := range ids[i:] { - if net.GetConn(a, b) != nil { - cc++ - } - } - } - - if cc != 1 { - t.Errorf("expected one connection, got %v", cc) - } -} - -func TestConnectNodesFull(t *testing.T) { - tests := []struct { - name string - nodeCount int - }{ - {name: "no node", nodeCount: 0}, - {name: "single node", nodeCount: 1}, - {name: "2 nodes", nodeCount: 2}, - {name: "3 nodes", nodeCount: 3}, - {name: "even number of nodes", nodeCount: 12}, - {name: "odd number of nodes", nodeCount: 13}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - net, ids := newTestNetwork(t, test.nodeCount) - defer net.Shutdown() - - err := net.ConnectNodesFull(ids) - if err != nil { - t.Fatal(err) - } - - VerifyFull(t, net, ids) - }) - } -} - -func TestConnectNodesChain(t *testing.T) { - net, ids := newTestNetwork(t, 10) - defer net.Shutdown() - - err := net.ConnectNodesChain(ids) - if err != nil { - t.Fatal(err) - } - - VerifyChain(t, net, ids) -} - -func TestConnectNodesRing(t *testing.T) { - net, ids := newTestNetwork(t, 10) - defer net.Shutdown() - - err := net.ConnectNodesRing(ids) - if err != nil { - t.Fatal(err) - } - - VerifyRing(t, net, ids) -} - -func TestConnectNodesStar(t *testing.T) { - net, ids := newTestNetwork(t, 10) - defer net.Shutdown() - - pivotIndex := 2 - - err := net.ConnectNodesStar(ids, ids[pivotIndex]) - if err != nil { - t.Fatal(err) - } - - VerifyStar(t, net, ids, pivotIndex) -} diff --git a/p2p/simulations/events.go b/p2p/simulations/events.go deleted file mode 100644 index 1131185fb9..0000000000 --- a/p2p/simulations/events.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package simulations - -import ( - "fmt" - "time" -) - -// EventType is the type of event emitted by a simulation network -type EventType string - -const ( - // EventTypeNode is the type of event emitted when a node is either - // created, started or stopped - EventTypeNode EventType = "node" - - // EventTypeConn is the type of event emitted when a connection is - // either established or dropped between two nodes - EventTypeConn EventType = "conn" - - // EventTypeMsg is the type of event emitted when a p2p message it - // sent between two nodes - EventTypeMsg EventType = "msg" -) - -// Event is an event emitted by a simulation network -type Event struct { - // Type is the type of the event - Type EventType `json:"type"` - - // Time is the time the event happened - Time time.Time `json:"time"` - - // Control indicates whether the event is the result of a controlled - // action in the network - Control bool `json:"control"` - - // Node is set if the type is EventTypeNode - Node *Node `json:"node,omitempty"` - - // Conn is set if the type is EventTypeConn - Conn *Conn `json:"conn,omitempty"` - - // Msg is set if the type is EventTypeMsg - Msg *Msg `json:"msg,omitempty"` - - //Optionally provide data (currently for simulation frontends only) - Data interface{} `json:"data"` -} - -// NewEvent creates a new event for the given object which should be either a -// Node, Conn or Msg. -// -// The object is copied so that the event represents the state of the object -// when NewEvent is called. -func NewEvent(v interface{}) *Event { - event := &Event{Time: time.Now()} - switch v := v.(type) { - case *Node: - event.Type = EventTypeNode - event.Node = v.copy() - case *Conn: - event.Type = EventTypeConn - conn := *v - event.Conn = &conn - case *Msg: - event.Type = EventTypeMsg - msg := *v - event.Msg = &msg - default: - panic(fmt.Sprintf("invalid event type: %T", v)) - } - return event -} - -// ControlEvent creates a new control event -func ControlEvent(v interface{}) *Event { - event := NewEvent(v) - event.Control = true - return event -} - -// String returns the string representation of the event -func (e *Event) String() string { - switch e.Type { - case EventTypeNode: - return fmt.Sprintf(" id: %s up: %t", e.Node.ID().TerminalString(), e.Node.Up()) - case EventTypeConn: - return fmt.Sprintf(" nodes: %s->%s up: %t", e.Conn.One.TerminalString(), e.Conn.Other.TerminalString(), e.Conn.Up) - case EventTypeMsg: - return fmt.Sprintf(" nodes: %s->%s proto: %s, code: %d, received: %t", e.Msg.One.TerminalString(), e.Msg.Other.TerminalString(), e.Msg.Protocol, e.Msg.Code, e.Msg.Received) - default: - return "" - } -} diff --git a/p2p/simulations/examples/README.md b/p2p/simulations/examples/README.md deleted file mode 100644 index 822a48dcb6..0000000000 --- a/p2p/simulations/examples/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# devp2p simulation examples - -## ping-pong - -`ping-pong.go` implements a simulation network which contains nodes running a -simple "ping-pong" protocol where nodes send a ping message to all their -connected peers every 10s and receive pong messages in return. - -To run the simulation, run `go run ping-pong.go` in one terminal to start the -simulation API and `./ping-pong.sh` in another to start and connect the nodes: - -``` -$ go run ping-pong.go -INFO [08-15|13:53:49] using sim adapter -INFO [08-15|13:53:49] starting simulation server on 0.0.0.0:8888... -``` - -``` -$ ./ping-pong.sh ----> 13:58:12 creating 10 nodes -Created node01 -Started node01 -... -Created node10 -Started node10 ----> 13:58:13 connecting node01 to all other nodes -Connected node01 to node02 -... -Connected node01 to node10 ----> 13:58:14 done -``` - -Use the `--adapter` flag to choose the adapter type: - -``` -$ go run ping-pong.go --adapter exec -INFO [08-15|14:01:14] using exec adapter tmpdir=/var/folders/k6/wpsgfg4n23ddbc6f5cnw5qg00000gn/T/p2p-example992833779 -INFO [08-15|14:01:14] starting simulation server on 0.0.0.0:8888... -``` diff --git a/p2p/simulations/examples/ping-pong.go b/p2p/simulations/examples/ping-pong.go deleted file mode 100644 index b0b8f22fdb..0000000000 --- a/p2p/simulations/examples/ping-pong.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package main - -import ( - "flag" - "fmt" - "io" - "net/http" - "os" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/simulations" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" -) - -var adapterType = flag.String("adapter", "sim", `node adapter to use (one of "sim" or "exec")`) - -// main() starts a simulation network which contains nodes running a simple -// ping-pong protocol -func main() { - flag.Parse() - - // set the log level to Trace - log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, false))) - - // register a single ping-pong service - services := map[string]adapters.LifecycleConstructor{ - "ping-pong": func(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { - pps := newPingPongService(ctx.Config.ID) - stack.RegisterProtocols(pps.Protocols()) - return pps, nil - }, - } - adapters.RegisterLifecycles(services) - - // create the NodeAdapter - var adapter adapters.NodeAdapter - - switch *adapterType { - - case "sim": - log.Info("using sim adapter") - adapter = adapters.NewSimAdapter(services) - - case "exec": - tmpdir, err := os.MkdirTemp("", "p2p-example") - if err != nil { - log.Crit("error creating temp dir", "err", err) - } - defer os.RemoveAll(tmpdir) - log.Info("using exec adapter", "tmpdir", tmpdir) - adapter = adapters.NewExecAdapter(tmpdir) - - default: - log.Crit(fmt.Sprintf("unknown node adapter %q", *adapterType)) - } - - // start the HTTP API - log.Info("starting simulation server on 0.0.0.0:8888...") - network := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ - DefaultService: "ping-pong", - }) - if err := http.ListenAndServe(":8888", simulations.NewServer(network)); err != nil { - log.Crit("error starting simulation server", "err", err) - } -} - -// pingPongService runs a ping-pong protocol between nodes where each node -// sends a ping to all its connected peers every 10s and receives a pong in -// return -type pingPongService struct { - id enode.ID - log log.Logger - received atomic.Int64 -} - -func newPingPongService(id enode.ID) *pingPongService { - return &pingPongService{ - id: id, - log: log.New("node.id", id), - } -} - -func (p *pingPongService) Protocols() []p2p.Protocol { - return []p2p.Protocol{{ - Name: "ping-pong", - Version: 1, - Length: 2, - Run: p.Run, - NodeInfo: p.Info, - }} -} - -func (p *pingPongService) Start() error { - p.log.Info("ping-pong service starting") - return nil -} - -func (p *pingPongService) Stop() error { - p.log.Info("ping-pong service stopping") - return nil -} - -func (p *pingPongService) Info() interface{} { - return struct { - Received int64 `json:"received"` - }{ - p.received.Load(), - } -} - -const ( - pingMsgCode = iota - pongMsgCode -) - -// Run implements the ping-pong protocol which sends ping messages to the peer -// at 10s intervals, and responds to pings with pong messages. -func (p *pingPongService) Run(peer *p2p.Peer, rw p2p.MsgReadWriter) error { - log := p.log.New("peer.id", peer.ID()) - - errC := make(chan error, 1) - go func() { - for range time.Tick(10 * time.Second) { - log.Info("sending ping") - if err := p2p.Send(rw, pingMsgCode, "PING"); err != nil { - errC <- err - return - } - } - }() - go func() { - for { - msg, err := rw.ReadMsg() - if err != nil { - errC <- err - return - } - payload, err := io.ReadAll(msg.Payload) - if err != nil { - errC <- err - return - } - log.Info("received message", "msg.code", msg.Code, "msg.payload", string(payload)) - p.received.Add(1) - if msg.Code == pingMsgCode { - log.Info("sending pong") - go p2p.Send(rw, pongMsgCode, "PONG") - } - } - }() - return <-errC -} diff --git a/p2p/simulations/examples/ping-pong.sh b/p2p/simulations/examples/ping-pong.sh deleted file mode 100755 index 47936bd9a0..0000000000 --- a/p2p/simulations/examples/ping-pong.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -# Boot a ping-pong network simulation using the HTTP API started by ping-pong.go - -set -e - -main() { - if ! which p2psim &>/dev/null; then - fail "missing p2psim binary (you need to build cmd/p2psim and put it in \$PATH)" - fi - - info "creating 10 nodes" - for i in $(seq 1 10); do - p2psim node create --name "$(node_name $i)" - p2psim node start "$(node_name $i)" - done - - info "connecting node01 to all other nodes" - for i in $(seq 2 10); do - p2psim node connect "node01" "$(node_name $i)" - done - - info "done" -} - -node_name() { - local num=$1 - echo "node$(printf '%02d' $num)" -} - -info() { - echo -e "\033[1;32m---> $(date +%H:%M:%S) ${@}\033[0m" -} - -fail() { - echo -e "\033[1;31mERROR: ${@}\033[0m" >&2 - exit 1 -} - -main "$@" diff --git a/p2p/simulations/http.go b/p2p/simulations/http.go deleted file mode 100644 index 34521b4778..0000000000 --- a/p2p/simulations/http.go +++ /dev/null @@ -1,743 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package simulations - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "html" - "io" - "net/http" - "strconv" - "strings" - "sync" - - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" - "github.com/ethereum/go-ethereum/rpc" - "github.com/gorilla/websocket" - "github.com/julienschmidt/httprouter" -) - -// DefaultClient is the default simulation API client which expects the API -// to be running at http://localhost:8888 -var DefaultClient = NewClient("http://localhost:8888") - -// Client is a client for the simulation HTTP API which supports creating -// and managing simulation networks -type Client struct { - URL string - - client *http.Client -} - -// NewClient returns a new simulation API client -func NewClient(url string) *Client { - return &Client{ - URL: url, - client: http.DefaultClient, - } -} - -// GetNetwork returns details of the network -func (c *Client) GetNetwork() (*Network, error) { - network := &Network{} - return network, c.Get("/", network) -} - -// StartNetwork starts all existing nodes in the simulation network -func (c *Client) StartNetwork() error { - return c.Post("/start", nil, nil) -} - -// StopNetwork stops all existing nodes in a simulation network -func (c *Client) StopNetwork() error { - return c.Post("/stop", nil, nil) -} - -// CreateSnapshot creates a network snapshot -func (c *Client) CreateSnapshot() (*Snapshot, error) { - snap := &Snapshot{} - return snap, c.Get("/snapshot", snap) -} - -// LoadSnapshot loads a snapshot into the network -func (c *Client) LoadSnapshot(snap *Snapshot) error { - return c.Post("/snapshot", snap, nil) -} - -// SubscribeOpts is a collection of options to use when subscribing to network -// events -type SubscribeOpts struct { - // Current instructs the server to send events for existing nodes and - // connections first - Current bool - - // Filter instructs the server to only send a subset of message events - Filter string -} - -// SubscribeNetwork subscribes to network events which are sent from the server -// as a server-sent-events stream, optionally receiving events for existing -// nodes and connections and filtering message events -func (c *Client) SubscribeNetwork(events chan *Event, opts SubscribeOpts) (event.Subscription, error) { - url := fmt.Sprintf("%s/events?current=%t&filter=%s", c.URL, opts.Current, opts.Filter) - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return nil, err - } - req.Header.Set("Accept", "text/event-stream") - res, err := c.client.Do(req) - if err != nil { - return nil, err - } - if res.StatusCode != http.StatusOK { - response, _ := io.ReadAll(res.Body) - res.Body.Close() - return nil, fmt.Errorf("unexpected HTTP status: %s: %s", res.Status, response) - } - - // define a producer function to pass to event.Subscription - // which reads server-sent events from res.Body and sends - // them to the events channel - producer := func(stop <-chan struct{}) error { - defer res.Body.Close() - - // read lines from res.Body in a goroutine so that we are - // always reading from the stop channel - lines := make(chan string) - errC := make(chan error, 1) - go func() { - s := bufio.NewScanner(res.Body) - for s.Scan() { - select { - case lines <- s.Text(): - case <-stop: - return - } - } - errC <- s.Err() - }() - - // detect any lines which start with "data:", decode the data - // into an event and send it to the events channel - for { - select { - case line := <-lines: - if !strings.HasPrefix(line, "data:") { - continue - } - data := strings.TrimSpace(strings.TrimPrefix(line, "data:")) - event := &Event{} - if err := json.Unmarshal([]byte(data), event); err != nil { - return fmt.Errorf("error decoding SSE event: %s", err) - } - select { - case events <- event: - case <-stop: - return nil - } - case err := <-errC: - return err - case <-stop: - return nil - } - } - } - - return event.NewSubscription(producer), nil -} - -// GetNodes returns all nodes which exist in the network -func (c *Client) GetNodes() ([]*p2p.NodeInfo, error) { - var nodes []*p2p.NodeInfo - return nodes, c.Get("/nodes", &nodes) -} - -// CreateNode creates a node in the network using the given configuration -func (c *Client) CreateNode(config *adapters.NodeConfig) (*p2p.NodeInfo, error) { - node := &p2p.NodeInfo{} - return node, c.Post("/nodes", config, node) -} - -// GetNode returns details of a node -func (c *Client) GetNode(nodeID string) (*p2p.NodeInfo, error) { - node := &p2p.NodeInfo{} - return node, c.Get(fmt.Sprintf("/nodes/%s", nodeID), node) -} - -// StartNode starts a node -func (c *Client) StartNode(nodeID string) error { - return c.Post(fmt.Sprintf("/nodes/%s/start", nodeID), nil, nil) -} - -// StopNode stops a node -func (c *Client) StopNode(nodeID string) error { - return c.Post(fmt.Sprintf("/nodes/%s/stop", nodeID), nil, nil) -} - -// ConnectNode connects a node to a peer node -func (c *Client) ConnectNode(nodeID, peerID string) error { - return c.Post(fmt.Sprintf("/nodes/%s/conn/%s", nodeID, peerID), nil, nil) -} - -// DisconnectNode disconnects a node from a peer node -func (c *Client) DisconnectNode(nodeID, peerID string) error { - return c.Delete(fmt.Sprintf("/nodes/%s/conn/%s", nodeID, peerID)) -} - -// RPCClient returns an RPC client connected to a node -func (c *Client) RPCClient(ctx context.Context, nodeID string) (*rpc.Client, error) { - baseURL := strings.Replace(c.URL, "http", "ws", 1) - return rpc.DialWebsocket(ctx, fmt.Sprintf("%s/nodes/%s/rpc", baseURL, nodeID), "") -} - -// Get performs a HTTP GET request decoding the resulting JSON response -// into "out" -func (c *Client) Get(path string, out interface{}) error { - return c.Send(http.MethodGet, path, nil, out) -} - -// Post performs a HTTP POST request sending "in" as the JSON body and -// decoding the resulting JSON response into "out" -func (c *Client) Post(path string, in, out interface{}) error { - return c.Send(http.MethodPost, path, in, out) -} - -// Delete performs a HTTP DELETE request -func (c *Client) Delete(path string) error { - return c.Send(http.MethodDelete, path, nil, nil) -} - -// Send performs a HTTP request, sending "in" as the JSON request body and -// decoding the JSON response into "out" -func (c *Client) Send(method, path string, in, out interface{}) error { - var body []byte - if in != nil { - var err error - body, err = json.Marshal(in) - if err != nil { - return err - } - } - req, err := http.NewRequest(method, c.URL+path, bytes.NewReader(body)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - res, err := c.client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusCreated { - response, _ := io.ReadAll(res.Body) - return fmt.Errorf("unexpected HTTP status: %s: %s", res.Status, response) - } - if out != nil { - if err := json.NewDecoder(res.Body).Decode(out); err != nil { - return err - } - } - return nil -} - -// Server is an HTTP server providing an API to manage a simulation network -type Server struct { - router *httprouter.Router - network *Network - mockerStop chan struct{} // when set, stops the current mocker - mockerMtx sync.Mutex // synchronises access to the mockerStop field -} - -// NewServer returns a new simulation API server -func NewServer(network *Network) *Server { - s := &Server{ - router: httprouter.New(), - network: network, - } - - s.OPTIONS("/", s.Options) - s.GET("/", s.GetNetwork) - s.POST("/start", s.StartNetwork) - s.POST("/stop", s.StopNetwork) - s.POST("/mocker/start", s.StartMocker) - s.POST("/mocker/stop", s.StopMocker) - s.GET("/mocker", s.GetMockers) - s.POST("/reset", s.ResetNetwork) - s.GET("/events", s.StreamNetworkEvents) - s.GET("/snapshot", s.CreateSnapshot) - s.POST("/snapshot", s.LoadSnapshot) - s.POST("/nodes", s.CreateNode) - s.GET("/nodes", s.GetNodes) - s.GET("/nodes/:nodeid", s.GetNode) - s.POST("/nodes/:nodeid/start", s.StartNode) - s.POST("/nodes/:nodeid/stop", s.StopNode) - s.POST("/nodes/:nodeid/conn/:peerid", s.ConnectNode) - s.DELETE("/nodes/:nodeid/conn/:peerid", s.DisconnectNode) - s.GET("/nodes/:nodeid/rpc", s.NodeRPC) - - return s -} - -// GetNetwork returns details of the network -func (s *Server) GetNetwork(w http.ResponseWriter, req *http.Request) { - s.JSON(w, http.StatusOK, s.network) -} - -// StartNetwork starts all nodes in the network -func (s *Server) StartNetwork(w http.ResponseWriter, req *http.Request) { - if err := s.network.StartAll(); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusOK) -} - -// StopNetwork stops all nodes in the network -func (s *Server) StopNetwork(w http.ResponseWriter, req *http.Request) { - if err := s.network.StopAll(); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusOK) -} - -// StartMocker starts the mocker node simulation -func (s *Server) StartMocker(w http.ResponseWriter, req *http.Request) { - s.mockerMtx.Lock() - defer s.mockerMtx.Unlock() - if s.mockerStop != nil { - http.Error(w, "mocker already running", http.StatusInternalServerError) - return - } - mockerType := req.FormValue("mocker-type") - mockerFn := LookupMocker(mockerType) - if mockerFn == nil { - http.Error(w, fmt.Sprintf("unknown mocker type %q", html.EscapeString(mockerType)), http.StatusBadRequest) - return - } - nodeCount, err := strconv.Atoi(req.FormValue("node-count")) - if err != nil { - http.Error(w, "invalid node-count provided", http.StatusBadRequest) - return - } - s.mockerStop = make(chan struct{}) - go mockerFn(s.network, s.mockerStop, nodeCount) - - w.WriteHeader(http.StatusOK) -} - -// StopMocker stops the mocker node simulation -func (s *Server) StopMocker(w http.ResponseWriter, req *http.Request) { - s.mockerMtx.Lock() - defer s.mockerMtx.Unlock() - if s.mockerStop == nil { - http.Error(w, "stop channel not initialized", http.StatusInternalServerError) - return - } - close(s.mockerStop) - s.mockerStop = nil - - w.WriteHeader(http.StatusOK) -} - -// GetMockers returns a list of available mockers -func (s *Server) GetMockers(w http.ResponseWriter, req *http.Request) { - list := GetMockerList() - s.JSON(w, http.StatusOK, list) -} - -// ResetNetwork resets all properties of a network to its initial (empty) state -func (s *Server) ResetNetwork(w http.ResponseWriter, req *http.Request) { - s.network.Reset() - - w.WriteHeader(http.StatusOK) -} - -// StreamNetworkEvents streams network events as a server-sent-events stream -func (s *Server) StreamNetworkEvents(w http.ResponseWriter, req *http.Request) { - events := make(chan *Event) - sub := s.network.events.Subscribe(events) - defer sub.Unsubscribe() - - // write writes the given event and data to the stream like: - // - // event: - // data: - // - write := func(event, data string) { - fmt.Fprintf(w, "event: %s\n", event) - fmt.Fprintf(w, "data: %s\n\n", data) - if fw, ok := w.(http.Flusher); ok { - fw.Flush() - } - } - writeEvent := func(event *Event) error { - data, err := json.Marshal(event) - if err != nil { - return err - } - write("network", string(data)) - return nil - } - writeErr := func(err error) { - write("error", err.Error()) - } - - // check if filtering has been requested - var filters MsgFilters - if filterParam := req.URL.Query().Get("filter"); filterParam != "" { - var err error - filters, err = NewMsgFilters(filterParam) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - } - - w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") - w.WriteHeader(http.StatusOK) - fmt.Fprintf(w, "\n\n") - if fw, ok := w.(http.Flusher); ok { - fw.Flush() - } - - // optionally send the existing nodes and connections - if req.URL.Query().Get("current") == "true" { - snap, err := s.network.Snapshot() - if err != nil { - writeErr(err) - return - } - for _, node := range snap.Nodes { - event := NewEvent(&node.Node) - if err := writeEvent(event); err != nil { - writeErr(err) - return - } - } - for _, conn := range snap.Conns { - conn := conn - event := NewEvent(&conn) - if err := writeEvent(event); err != nil { - writeErr(err) - return - } - } - } - - clientGone := req.Context().Done() - for { - select { - case event := <-events: - // only send message events which match the filters - if event.Msg != nil && !filters.Match(event.Msg) { - continue - } - if err := writeEvent(event); err != nil { - writeErr(err) - return - } - case <-clientGone: - return - } - } -} - -// NewMsgFilters constructs a collection of message filters from a URL query -// parameter. -// -// The parameter is expected to be a dash-separated list of individual filters, -// each having the format ':', where is the name of a -// protocol and is a comma-separated list of message codes. -// -// A message code of '*' or '-1' is considered a wildcard and matches any code. -func NewMsgFilters(filterParam string) (MsgFilters, error) { - filters := make(MsgFilters) - for _, filter := range strings.Split(filterParam, "-") { - proto, codes, found := strings.Cut(filter, ":") - if !found || proto == "" || codes == "" { - return nil, fmt.Errorf("invalid message filter: %s", filter) - } - - for _, code := range strings.Split(codes, ",") { - if code == "*" || code == "-1" { - filters[MsgFilter{Proto: proto, Code: -1}] = struct{}{} - continue - } - n, err := strconv.ParseUint(code, 10, 64) - if err != nil { - return nil, fmt.Errorf("invalid message code: %s", code) - } - filters[MsgFilter{Proto: proto, Code: int64(n)}] = struct{}{} - } - } - return filters, nil -} - -// MsgFilters is a collection of filters which are used to filter message -// events -type MsgFilters map[MsgFilter]struct{} - -// Match checks if the given message matches any of the filters -func (m MsgFilters) Match(msg *Msg) bool { - // check if there is a wildcard filter for the message's protocol - if _, ok := m[MsgFilter{Proto: msg.Protocol, Code: -1}]; ok { - return true - } - - // check if there is a filter for the message's protocol and code - if _, ok := m[MsgFilter{Proto: msg.Protocol, Code: int64(msg.Code)}]; ok { - return true - } - - return false -} - -// MsgFilter is used to filter message events based on protocol and message -// code -type MsgFilter struct { - // Proto is matched against a message's protocol - Proto string - - // Code is matched against a message's code, with -1 matching all codes - Code int64 -} - -// CreateSnapshot creates a network snapshot -func (s *Server) CreateSnapshot(w http.ResponseWriter, req *http.Request) { - snap, err := s.network.Snapshot() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - s.JSON(w, http.StatusOK, snap) -} - -// LoadSnapshot loads a snapshot into the network -func (s *Server) LoadSnapshot(w http.ResponseWriter, req *http.Request) { - snap := &Snapshot{} - if err := json.NewDecoder(req.Body).Decode(snap); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := s.network.Load(snap); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - s.JSON(w, http.StatusOK, s.network) -} - -// CreateNode creates a node in the network using the given configuration -func (s *Server) CreateNode(w http.ResponseWriter, req *http.Request) { - config := &adapters.NodeConfig{} - - err := json.NewDecoder(req.Body).Decode(config) - if err != nil && !errors.Is(err, io.EOF) { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - node, err := s.network.NewNodeWithConfig(config) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - s.JSON(w, http.StatusCreated, node.NodeInfo()) -} - -// GetNodes returns all nodes which exist in the network -func (s *Server) GetNodes(w http.ResponseWriter, req *http.Request) { - nodes := s.network.GetNodes() - - infos := make([]*p2p.NodeInfo, len(nodes)) - for i, node := range nodes { - infos[i] = node.NodeInfo() - } - - s.JSON(w, http.StatusOK, infos) -} - -// GetNode returns details of a node -func (s *Server) GetNode(w http.ResponseWriter, req *http.Request) { - node := req.Context().Value("node").(*Node) - - s.JSON(w, http.StatusOK, node.NodeInfo()) -} - -// StartNode starts a node -func (s *Server) StartNode(w http.ResponseWriter, req *http.Request) { - node := req.Context().Value("node").(*Node) - - if err := s.network.Start(node.ID()); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - s.JSON(w, http.StatusOK, node.NodeInfo()) -} - -// StopNode stops a node -func (s *Server) StopNode(w http.ResponseWriter, req *http.Request) { - node := req.Context().Value("node").(*Node) - - if err := s.network.Stop(node.ID()); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - s.JSON(w, http.StatusOK, node.NodeInfo()) -} - -// ConnectNode connects a node to a peer node -func (s *Server) ConnectNode(w http.ResponseWriter, req *http.Request) { - node := req.Context().Value("node").(*Node) - peer := req.Context().Value("peer").(*Node) - - if err := s.network.Connect(node.ID(), peer.ID()); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - s.JSON(w, http.StatusOK, node.NodeInfo()) -} - -// DisconnectNode disconnects a node from a peer node -func (s *Server) DisconnectNode(w http.ResponseWriter, req *http.Request) { - node := req.Context().Value("node").(*Node) - peer := req.Context().Value("peer").(*Node) - - if err := s.network.Disconnect(node.ID(), peer.ID()); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - s.JSON(w, http.StatusOK, node.NodeInfo()) -} - -// Options responds to the OPTIONS HTTP method by returning a 200 OK response -// with the "Access-Control-Allow-Headers" header set to "Content-Type" -func (s *Server) Options(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Access-Control-Allow-Headers", "Content-Type") - w.WriteHeader(http.StatusOK) -} - -var wsUpgrade = websocket.Upgrader{ - CheckOrigin: func(*http.Request) bool { return true }, -} - -// NodeRPC forwards RPC requests to a node in the network via a WebSocket -// connection -func (s *Server) NodeRPC(w http.ResponseWriter, req *http.Request) { - conn, err := wsUpgrade.Upgrade(w, req, nil) - if err != nil { - return - } - defer conn.Close() - node := req.Context().Value("node").(*Node) - node.ServeRPC(conn) -} - -// ServeHTTP implements the http.Handler interface by delegating to the -// underlying httprouter.Router -func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { - s.router.ServeHTTP(w, req) -} - -// GET registers a handler for GET requests to a particular path -func (s *Server) GET(path string, handle http.HandlerFunc) { - s.router.GET(path, s.wrapHandler(handle)) -} - -// POST registers a handler for POST requests to a particular path -func (s *Server) POST(path string, handle http.HandlerFunc) { - s.router.POST(path, s.wrapHandler(handle)) -} - -// DELETE registers a handler for DELETE requests to a particular path -func (s *Server) DELETE(path string, handle http.HandlerFunc) { - s.router.DELETE(path, s.wrapHandler(handle)) -} - -// OPTIONS registers a handler for OPTIONS requests to a particular path -func (s *Server) OPTIONS(path string, handle http.HandlerFunc) { - s.router.OPTIONS("/*path", s.wrapHandler(handle)) -} - -// JSON sends "data" as a JSON HTTP response -func (s *Server) JSON(w http.ResponseWriter, status int, data interface{}) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - json.NewEncoder(w).Encode(data) -} - -// wrapHandler returns an httprouter.Handle which wraps an http.HandlerFunc by -// populating request.Context with any objects from the URL params -func (s *Server) wrapHandler(handler http.HandlerFunc) httprouter.Handle { - return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") - - ctx := req.Context() - - if id := params.ByName("nodeid"); id != "" { - var nodeID enode.ID - var node *Node - if nodeID.UnmarshalText([]byte(id)) == nil { - node = s.network.GetNode(nodeID) - } else { - node = s.network.GetNodeByName(id) - } - if node == nil { - http.NotFound(w, req) - return - } - ctx = context.WithValue(ctx, "node", node) - } - - if id := params.ByName("peerid"); id != "" { - var peerID enode.ID - var peer *Node - if peerID.UnmarshalText([]byte(id)) == nil { - peer = s.network.GetNode(peerID) - } else { - peer = s.network.GetNodeByName(id) - } - if peer == nil { - http.NotFound(w, req) - return - } - ctx = context.WithValue(ctx, "peer", peer) - } - - handler(w, req.WithContext(ctx)) - } -} diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go deleted file mode 100644 index cd03e600f3..0000000000 --- a/p2p/simulations/http_test.go +++ /dev/null @@ -1,869 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package simulations - -import ( - "context" - "flag" - "fmt" - "log/slog" - "math/rand" - "net/http/httptest" - "os" - "reflect" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" - "github.com/ethereum/go-ethereum/rpc" - "github.com/mattn/go-colorable" -) - -func TestMain(m *testing.M) { - loglevel := flag.Int("loglevel", 2, "verbosity of logs") - - flag.Parse() - log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(colorable.NewColorableStderr(), slog.Level(*loglevel), true))) - os.Exit(m.Run()) -} - -// testService implements the node.Service interface and provides protocols -// and APIs which are useful for testing nodes in a simulation network -type testService struct { - id enode.ID - - // peerCount is incremented once a peer handshake has been performed - peerCount int64 - - peers map[enode.ID]*testPeer - peersMtx sync.Mutex - - // state stores []byte which is used to test creating and loading - // snapshots - state atomic.Value -} - -func newTestService(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { - svc := &testService{ - id: ctx.Config.ID, - peers: make(map[enode.ID]*testPeer), - } - svc.state.Store(ctx.Snapshot) - - stack.RegisterProtocols(svc.Protocols()) - stack.RegisterAPIs(svc.APIs()) - return svc, nil -} - -type testPeer struct { - testReady chan struct{} - dumReady chan struct{} -} - -func (t *testService) peer(id enode.ID) *testPeer { - t.peersMtx.Lock() - defer t.peersMtx.Unlock() - if peer, ok := t.peers[id]; ok { - return peer - } - peer := &testPeer{ - testReady: make(chan struct{}), - dumReady: make(chan struct{}), - } - t.peers[id] = peer - return peer -} - -func (t *testService) Protocols() []p2p.Protocol { - return []p2p.Protocol{ - { - Name: "test", - Version: 1, - Length: 3, - Run: t.RunTest, - }, - { - Name: "dum", - Version: 1, - Length: 1, - Run: t.RunDum, - }, - { - Name: "prb", - Version: 1, - Length: 1, - Run: t.RunPrb, - }, - } -} - -func (t *testService) APIs() []rpc.API { - return []rpc.API{{ - Namespace: "test", - Version: "1.0", - Service: &TestAPI{ - state: &t.state, - peerCount: &t.peerCount, - }, - }} -} - -func (t *testService) Start() error { - return nil -} - -func (t *testService) Stop() error { - return nil -} - -// handshake performs a peer handshake by sending and expecting an empty -// message with the given code -func (t *testService) handshake(rw p2p.MsgReadWriter, code uint64) error { - errc := make(chan error, 2) - go func() { errc <- p2p.SendItems(rw, code) }() - go func() { errc <- p2p.ExpectMsg(rw, code, struct{}{}) }() - for i := 0; i < 2; i++ { - if err := <-errc; err != nil { - return err - } - } - return nil -} - -func (t *testService) RunTest(p *p2p.Peer, rw p2p.MsgReadWriter) error { - peer := t.peer(p.ID()) - - // perform three handshakes with three different message codes, - // used to test message sending and filtering - if err := t.handshake(rw, 2); err != nil { - return err - } - if err := t.handshake(rw, 1); err != nil { - return err - } - if err := t.handshake(rw, 0); err != nil { - return err - } - - // close the testReady channel so that other protocols can run - close(peer.testReady) - - // track the peer - atomic.AddInt64(&t.peerCount, 1) - defer atomic.AddInt64(&t.peerCount, -1) - - // block until the peer is dropped - for { - _, err := rw.ReadMsg() - if err != nil { - return err - } - } -} - -func (t *testService) RunDum(p *p2p.Peer, rw p2p.MsgReadWriter) error { - peer := t.peer(p.ID()) - - // wait for the test protocol to perform its handshake - <-peer.testReady - - // perform a handshake - if err := t.handshake(rw, 0); err != nil { - return err - } - - // close the dumReady channel so that other protocols can run - close(peer.dumReady) - - // block until the peer is dropped - for { - _, err := rw.ReadMsg() - if err != nil { - return err - } - } -} -func (t *testService) RunPrb(p *p2p.Peer, rw p2p.MsgReadWriter) error { - peer := t.peer(p.ID()) - - // wait for the dum protocol to perform its handshake - <-peer.dumReady - - // perform a handshake - if err := t.handshake(rw, 0); err != nil { - return err - } - - // block until the peer is dropped - for { - _, err := rw.ReadMsg() - if err != nil { - return err - } - } -} - -func (t *testService) Snapshot() ([]byte, error) { - return t.state.Load().([]byte), nil -} - -// TestAPI provides a test API to: -// * get the peer count -// * get and set an arbitrary state byte slice -// * get and increment a counter -// * subscribe to counter increment events -type TestAPI struct { - state *atomic.Value - peerCount *int64 - counter int64 - feed event.Feed -} - -func (t *TestAPI) PeerCount() int64 { - return atomic.LoadInt64(t.peerCount) -} - -func (t *TestAPI) Get() int64 { - return atomic.LoadInt64(&t.counter) -} - -func (t *TestAPI) Add(delta int64) { - atomic.AddInt64(&t.counter, delta) - t.feed.Send(delta) -} - -func (t *TestAPI) GetState() []byte { - return t.state.Load().([]byte) -} - -func (t *TestAPI) SetState(state []byte) { - t.state.Store(state) -} - -func (t *TestAPI) Events(ctx context.Context) (*rpc.Subscription, error) { - notifier, supported := rpc.NotifierFromContext(ctx) - if !supported { - return nil, rpc.ErrNotificationsUnsupported - } - - rpcSub := notifier.CreateSubscription() - - go func() { - events := make(chan int64) - sub := t.feed.Subscribe(events) - defer sub.Unsubscribe() - - for { - select { - case event := <-events: - notifier.Notify(rpcSub.ID, event) - case <-sub.Err(): - return - case <-rpcSub.Err(): - return - } - } - }() - - return rpcSub, nil -} - -var testServices = adapters.LifecycleConstructors{ - "test": newTestService, -} - -func testHTTPServer(t *testing.T) (*Network, *httptest.Server) { - t.Helper() - adapter := adapters.NewSimAdapter(testServices) - network := NewNetwork(adapter, &NetworkConfig{ - DefaultService: "test", - }) - return network, httptest.NewServer(NewServer(network)) -} - -// TestHTTPNetwork tests interacting with a simulation network using the HTTP -// API -func TestHTTPNetwork(t *testing.T) { - // start the server - network, s := testHTTPServer(t) - defer s.Close() - - // subscribe to events so we can check them later - client := NewClient(s.URL) - events := make(chan *Event, 100) - var opts SubscribeOpts - sub, err := client.SubscribeNetwork(events, opts) - if err != nil { - t.Fatalf("error subscribing to network events: %s", err) - } - defer sub.Unsubscribe() - - // check we can retrieve details about the network - gotNetwork, err := client.GetNetwork() - if err != nil { - t.Fatalf("error getting network: %s", err) - } - if gotNetwork.ID != network.ID { - t.Fatalf("expected network to have ID %q, got %q", network.ID, gotNetwork.ID) - } - - // start a simulation network - nodeIDs := startTestNetwork(t, client) - - // check we got all the events - x := &expectEvents{t, events, sub} - x.expect( - x.nodeEvent(nodeIDs[0], false), - x.nodeEvent(nodeIDs[1], false), - x.nodeEvent(nodeIDs[0], true), - x.nodeEvent(nodeIDs[1], true), - x.connEvent(nodeIDs[0], nodeIDs[1], false), - x.connEvent(nodeIDs[0], nodeIDs[1], true), - ) - - // reconnect the stream and check we get the current nodes and conns - events = make(chan *Event, 100) - opts.Current = true - sub, err = client.SubscribeNetwork(events, opts) - if err != nil { - t.Fatalf("error subscribing to network events: %s", err) - } - defer sub.Unsubscribe() - x = &expectEvents{t, events, sub} - x.expect( - x.nodeEvent(nodeIDs[0], true), - x.nodeEvent(nodeIDs[1], true), - x.connEvent(nodeIDs[0], nodeIDs[1], true), - ) -} - -func startTestNetwork(t *testing.T, client *Client) []string { - // create two nodes - nodeCount := 2 - nodeIDs := make([]string, nodeCount) - for i := 0; i < nodeCount; i++ { - config := adapters.RandomNodeConfig() - node, err := client.CreateNode(config) - if err != nil { - t.Fatalf("error creating node: %s", err) - } - nodeIDs[i] = node.ID - } - - // check both nodes exist - nodes, err := client.GetNodes() - if err != nil { - t.Fatalf("error getting nodes: %s", err) - } - if len(nodes) != nodeCount { - t.Fatalf("expected %d nodes, got %d", nodeCount, len(nodes)) - } - for i, nodeID := range nodeIDs { - if nodes[i].ID != nodeID { - t.Fatalf("expected node %d to have ID %q, got %q", i, nodeID, nodes[i].ID) - } - node, err := client.GetNode(nodeID) - if err != nil { - t.Fatalf("error getting node %d: %s", i, err) - } - if node.ID != nodeID { - t.Fatalf("expected node %d to have ID %q, got %q", i, nodeID, node.ID) - } - } - - // start both nodes - for _, nodeID := range nodeIDs { - if err := client.StartNode(nodeID); err != nil { - t.Fatalf("error starting node %q: %s", nodeID, err) - } - } - - // connect the nodes - for i := 0; i < nodeCount-1; i++ { - peerId := i + 1 - if i == nodeCount-1 { - peerId = 0 - } - if err := client.ConnectNode(nodeIDs[i], nodeIDs[peerId]); err != nil { - t.Fatalf("error connecting nodes: %s", err) - } - } - - return nodeIDs -} - -type expectEvents struct { - *testing.T - - events chan *Event - sub event.Subscription -} - -func (t *expectEvents) nodeEvent(id string, up bool) *Event { - config := &adapters.NodeConfig{ID: enode.HexID(id)} - return &Event{Type: EventTypeNode, Node: newNode(nil, config, up)} -} - -func (t *expectEvents) connEvent(one, other string, up bool) *Event { - return &Event{ - Type: EventTypeConn, - Conn: &Conn{ - One: enode.HexID(one), - Other: enode.HexID(other), - Up: up, - }, - } -} - -func (t *expectEvents) expectMsgs(expected map[MsgFilter]int) { - actual := make(map[MsgFilter]int) - timeout := time.After(10 * time.Second) -loop: - for { - select { - case event := <-t.events: - t.Logf("received %s event: %v", event.Type, event) - - if event.Type != EventTypeMsg || event.Msg.Received { - continue loop - } - if event.Msg == nil { - t.Fatal("expected event.Msg to be set") - } - filter := MsgFilter{ - Proto: event.Msg.Protocol, - Code: int64(event.Msg.Code), - } - actual[filter]++ - if actual[filter] > expected[filter] { - t.Fatalf("received too many msgs for filter: %v", filter) - } - if reflect.DeepEqual(actual, expected) { - return - } - - case err := <-t.sub.Err(): - t.Fatalf("network stream closed unexpectedly: %s", err) - - case <-timeout: - t.Fatal("timed out waiting for expected events") - } - } -} - -func (t *expectEvents) expect(events ...*Event) { - t.Helper() - timeout := time.After(10 * time.Second) - i := 0 - for { - select { - case event := <-t.events: - t.Logf("received %s event: %v", event.Type, event) - - expected := events[i] - if event.Type != expected.Type { - t.Fatalf("expected event %d to have type %q, got %q", i, expected.Type, event.Type) - } - - switch expected.Type { - case EventTypeNode: - if event.Node == nil { - t.Fatal("expected event.Node to be set") - } - if event.Node.ID() != expected.Node.ID() { - t.Fatalf("expected node event %d to have id %q, got %q", i, expected.Node.ID().TerminalString(), event.Node.ID().TerminalString()) - } - if event.Node.Up() != expected.Node.Up() { - t.Fatalf("expected node event %d to have up=%t, got up=%t", i, expected.Node.Up(), event.Node.Up()) - } - - case EventTypeConn: - if event.Conn == nil { - t.Fatal("expected event.Conn to be set") - } - if event.Conn.One != expected.Conn.One { - t.Fatalf("expected conn event %d to have one=%q, got one=%q", i, expected.Conn.One.TerminalString(), event.Conn.One.TerminalString()) - } - if event.Conn.Other != expected.Conn.Other { - t.Fatalf("expected conn event %d to have other=%q, got other=%q", i, expected.Conn.Other.TerminalString(), event.Conn.Other.TerminalString()) - } - if event.Conn.Up != expected.Conn.Up { - t.Fatalf("expected conn event %d to have up=%t, got up=%t", i, expected.Conn.Up, event.Conn.Up) - } - } - - i++ - if i == len(events) { - return - } - - case err := <-t.sub.Err(): - t.Fatalf("network stream closed unexpectedly: %s", err) - - case <-timeout: - t.Fatal("timed out waiting for expected events") - } - } -} - -// TestHTTPNodeRPC tests calling RPC methods on nodes via the HTTP API -func TestHTTPNodeRPC(t *testing.T) { - // start the server - _, s := testHTTPServer(t) - defer s.Close() - - // start a node in the network - client := NewClient(s.URL) - - config := adapters.RandomNodeConfig() - node, err := client.CreateNode(config) - if err != nil { - t.Fatalf("error creating node: %s", err) - } - if err := client.StartNode(node.ID); err != nil { - t.Fatalf("error starting node: %s", err) - } - - // create two RPC clients - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - rpcClient1, err := client.RPCClient(ctx, node.ID) - if err != nil { - t.Fatalf("error getting node RPC client: %s", err) - } - rpcClient2, err := client.RPCClient(ctx, node.ID) - if err != nil { - t.Fatalf("error getting node RPC client: %s", err) - } - - // subscribe to events using client 1 - events := make(chan int64, 1) - sub, err := rpcClient1.Subscribe(ctx, "test", events, "events") - if err != nil { - t.Fatalf("error subscribing to events: %s", err) - } - defer sub.Unsubscribe() - - // call some RPC methods using client 2 - if err := rpcClient2.CallContext(ctx, nil, "test_add", 10); err != nil { - t.Fatalf("error calling RPC method: %s", err) - } - var result int64 - if err := rpcClient2.CallContext(ctx, &result, "test_get"); err != nil { - t.Fatalf("error calling RPC method: %s", err) - } - if result != 10 { - t.Fatalf("expected result to be 10, got %d", result) - } - - // check we got an event from client 1 - select { - case event := <-events: - if event != 10 { - t.Fatalf("expected event to be 10, got %d", event) - } - case <-ctx.Done(): - t.Fatal(ctx.Err()) - } -} - -// TestHTTPSnapshot tests creating and loading network snapshots -func TestHTTPSnapshot(t *testing.T) { - // start the server - network, s := testHTTPServer(t) - defer s.Close() - - var eventsDone = make(chan struct{}, 1) - count := 1 - eventsDoneChan := make(chan *Event) - eventSub := network.Events().Subscribe(eventsDoneChan) - go func() { - defer eventSub.Unsubscribe() - for event := range eventsDoneChan { - if event.Type == EventTypeConn && !event.Control { - count-- - if count == 0 { - eventsDone <- struct{}{} - return - } - } - } - }() - - // create a two-node network - client := NewClient(s.URL) - nodeCount := 2 - nodes := make([]*p2p.NodeInfo, nodeCount) - for i := 0; i < nodeCount; i++ { - config := adapters.RandomNodeConfig() - node, err := client.CreateNode(config) - if err != nil { - t.Fatalf("error creating node: %s", err) - } - if err := client.StartNode(node.ID); err != nil { - t.Fatalf("error starting node: %s", err) - } - nodes[i] = node - } - if err := client.ConnectNode(nodes[0].ID, nodes[1].ID); err != nil { - t.Fatalf("error connecting nodes: %s", err) - } - - // store some state in the test services - states := make([]string, nodeCount) - for i, node := range nodes { - rpc, err := client.RPCClient(context.Background(), node.ID) - if err != nil { - t.Fatalf("error getting RPC client: %s", err) - } - defer rpc.Close() - state := fmt.Sprintf("%x", rand.Int()) - if err := rpc.Call(nil, "test_setState", []byte(state)); err != nil { - t.Fatalf("error setting service state: %s", err) - } - states[i] = state - } - <-eventsDone - // create a snapshot - snap, err := client.CreateSnapshot() - if err != nil { - t.Fatalf("error creating snapshot: %s", err) - } - for i, state := range states { - gotState := snap.Nodes[i].Snapshots["test"] - if string(gotState) != state { - t.Fatalf("expected snapshot state %q, got %q", state, gotState) - } - } - - // create another network - network2, s := testHTTPServer(t) - defer s.Close() - client = NewClient(s.URL) - count = 1 - eventSub = network2.Events().Subscribe(eventsDoneChan) - go func() { - defer eventSub.Unsubscribe() - for event := range eventsDoneChan { - if event.Type == EventTypeConn && !event.Control { - count-- - if count == 0 { - eventsDone <- struct{}{} - return - } - } - } - }() - - // subscribe to events so we can check them later - events := make(chan *Event, 100) - var opts SubscribeOpts - sub, err := client.SubscribeNetwork(events, opts) - if err != nil { - t.Fatalf("error subscribing to network events: %s", err) - } - defer sub.Unsubscribe() - - // load the snapshot - if err := client.LoadSnapshot(snap); err != nil { - t.Fatalf("error loading snapshot: %s", err) - } - <-eventsDone - - // check the nodes and connection exists - net, err := client.GetNetwork() - if err != nil { - t.Fatalf("error getting network: %s", err) - } - if len(net.Nodes) != nodeCount { - t.Fatalf("expected network to have %d nodes, got %d", nodeCount, len(net.Nodes)) - } - for i, node := range nodes { - id := net.Nodes[i].ID().String() - if id != node.ID { - t.Fatalf("expected node %d to have ID %s, got %s", i, node.ID, id) - } - } - if len(net.Conns) != 1 { - t.Fatalf("expected network to have 1 connection, got %d", len(net.Conns)) - } - conn := net.Conns[0] - if conn.One.String() != nodes[0].ID { - t.Fatalf("expected connection to have one=%q, got one=%q", nodes[0].ID, conn.One) - } - if conn.Other.String() != nodes[1].ID { - t.Fatalf("expected connection to have other=%q, got other=%q", nodes[1].ID, conn.Other) - } - if !conn.Up { - t.Fatal("should be up") - } - - // check the node states were restored - for i, node := range nodes { - rpc, err := client.RPCClient(context.Background(), node.ID) - if err != nil { - t.Fatalf("error getting RPC client: %s", err) - } - defer rpc.Close() - var state []byte - if err := rpc.Call(&state, "test_getState"); err != nil { - t.Fatalf("error getting service state: %s", err) - } - if string(state) != states[i] { - t.Fatalf("expected snapshot state %q, got %q", states[i], state) - } - } - - // check we got all the events - x := &expectEvents{t, events, sub} - x.expect( - x.nodeEvent(nodes[0].ID, false), - x.nodeEvent(nodes[0].ID, true), - x.nodeEvent(nodes[1].ID, false), - x.nodeEvent(nodes[1].ID, true), - x.connEvent(nodes[0].ID, nodes[1].ID, false), - x.connEvent(nodes[0].ID, nodes[1].ID, true), - ) -} - -// TestMsgFilterPassMultiple tests streaming message events using a filter -// with multiple protocols -func TestMsgFilterPassMultiple(t *testing.T) { - // start the server - _, s := testHTTPServer(t) - defer s.Close() - - // subscribe to events with a message filter - client := NewClient(s.URL) - events := make(chan *Event, 10) - opts := SubscribeOpts{ - Filter: "prb:0-test:0", - } - sub, err := client.SubscribeNetwork(events, opts) - if err != nil { - t.Fatalf("error subscribing to network events: %s", err) - } - defer sub.Unsubscribe() - - // start a simulation network - startTestNetwork(t, client) - - // check we got the expected events - x := &expectEvents{t, events, sub} - x.expectMsgs(map[MsgFilter]int{ - {"test", 0}: 2, - {"prb", 0}: 2, - }) -} - -// TestMsgFilterPassWildcard tests streaming message events using a filter -// with a code wildcard -func TestMsgFilterPassWildcard(t *testing.T) { - // start the server - _, s := testHTTPServer(t) - defer s.Close() - - // subscribe to events with a message filter - client := NewClient(s.URL) - events := make(chan *Event, 10) - opts := SubscribeOpts{ - Filter: "prb:0,2-test:*", - } - sub, err := client.SubscribeNetwork(events, opts) - if err != nil { - t.Fatalf("error subscribing to network events: %s", err) - } - defer sub.Unsubscribe() - - // start a simulation network - startTestNetwork(t, client) - - // check we got the expected events - x := &expectEvents{t, events, sub} - x.expectMsgs(map[MsgFilter]int{ - {"test", 2}: 2, - {"test", 1}: 2, - {"test", 0}: 2, - {"prb", 0}: 2, - }) -} - -// TestMsgFilterPassSingle tests streaming message events using a filter -// with a single protocol and code -func TestMsgFilterPassSingle(t *testing.T) { - // start the server - _, s := testHTTPServer(t) - defer s.Close() - - // subscribe to events with a message filter - client := NewClient(s.URL) - events := make(chan *Event, 10) - opts := SubscribeOpts{ - Filter: "dum:0", - } - sub, err := client.SubscribeNetwork(events, opts) - if err != nil { - t.Fatalf("error subscribing to network events: %s", err) - } - defer sub.Unsubscribe() - - // start a simulation network - startTestNetwork(t, client) - - // check we got the expected events - x := &expectEvents{t, events, sub} - x.expectMsgs(map[MsgFilter]int{ - {"dum", 0}: 2, - }) -} - -// TestMsgFilterFailBadParams tests streaming message events using an invalid -// filter -func TestMsgFilterFailBadParams(t *testing.T) { - // start the server - _, s := testHTTPServer(t) - defer s.Close() - - client := NewClient(s.URL) - events := make(chan *Event, 10) - opts := SubscribeOpts{ - Filter: "foo:", - } - _, err := client.SubscribeNetwork(events, opts) - if err == nil { - t.Fatalf("expected event subscription to fail but succeeded!") - } - - opts.Filter = "bzz:aa" - _, err = client.SubscribeNetwork(events, opts) - if err == nil { - t.Fatalf("expected event subscription to fail but succeeded!") - } - - opts.Filter = "invalid" - _, err = client.SubscribeNetwork(events, opts) - if err == nil { - t.Fatalf("expected event subscription to fail but succeeded!") - } -} diff --git a/p2p/simulations/mocker.go b/p2p/simulations/mocker.go deleted file mode 100644 index 8763df67ef..0000000000 --- a/p2p/simulations/mocker.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package simulations simulates p2p networks. -// A mocker simulates starting and stopping real nodes in a network. -package simulations - -import ( - "fmt" - "math/rand" - "sync" - "time" - - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" -) - -// a map of mocker names to its function -var mockerList = map[string]func(net *Network, quit chan struct{}, nodeCount int){ - "startStop": startStop, - "probabilistic": probabilistic, - "boot": boot, -} - -// LookupMocker looks a mocker by its name, returns the mockerFn -func LookupMocker(mockerType string) func(net *Network, quit chan struct{}, nodeCount int) { - return mockerList[mockerType] -} - -// GetMockerList returns a list of mockers (keys of the map) -// Useful for frontend to build available mocker selection -func GetMockerList() []string { - list := make([]string, 0, len(mockerList)) - for k := range mockerList { - list = append(list, k) - } - return list -} - -// The boot mockerFn only connects the node in a ring and doesn't do anything else -func boot(net *Network, quit chan struct{}, nodeCount int) { - _, err := connectNodesInRing(net, nodeCount) - if err != nil { - panic("Could not startup node network for mocker") - } -} - -// The startStop mockerFn stops and starts nodes in a defined period (ticker) -func startStop(net *Network, quit chan struct{}, nodeCount int) { - nodes, err := connectNodesInRing(net, nodeCount) - if err != nil { - panic("Could not startup node network for mocker") - } - var ( - tick = time.NewTicker(10 * time.Second) - timer = time.NewTimer(3 * time.Second) - ) - defer tick.Stop() - defer timer.Stop() - - for { - select { - case <-quit: - log.Info("Terminating simulation loop") - return - case <-tick.C: - id := nodes[rand.Intn(len(nodes))] - log.Info("stopping node", "id", id) - if err := net.Stop(id); err != nil { - log.Error("error stopping node", "id", id, "err", err) - return - } - - timer.Reset(3 * time.Second) - select { - case <-quit: - log.Info("Terminating simulation loop") - return - case <-timer.C: - } - - log.Debug("starting node", "id", id) - if err := net.Start(id); err != nil { - log.Error("error starting node", "id", id, "err", err) - return - } - } - } -} - -// The probabilistic mocker func has a more probabilistic pattern -// (the implementation could probably be improved): -// nodes are connected in a ring, then a varying number of random nodes is selected, -// mocker then stops and starts them in random intervals, and continues the loop -func probabilistic(net *Network, quit chan struct{}, nodeCount int) { - nodes, err := connectNodesInRing(net, nodeCount) - if err != nil { - select { - case <-quit: - //error may be due to abortion of mocking; so the quit channel is closed - return - default: - panic("Could not startup node network for mocker") - } - } - for { - select { - case <-quit: - log.Info("Terminating simulation loop") - return - default: - } - var lowid, highid int - var wg sync.WaitGroup - randWait := time.Duration(rand.Intn(5000)+1000) * time.Millisecond - rand1 := rand.Intn(nodeCount - 1) - rand2 := rand.Intn(nodeCount - 1) - if rand1 <= rand2 { - lowid = rand1 - highid = rand2 - } else if rand1 > rand2 { - highid = rand1 - lowid = rand2 - } - var steps = highid - lowid - wg.Add(steps) - for i := lowid; i < highid; i++ { - select { - case <-quit: - log.Info("Terminating simulation loop") - return - case <-time.After(randWait): - } - log.Debug(fmt.Sprintf("node %v shutting down", nodes[i])) - err := net.Stop(nodes[i]) - if err != nil { - log.Error("Error stopping node", "node", nodes[i]) - wg.Done() - continue - } - go func(id enode.ID) { - time.Sleep(randWait) - err := net.Start(id) - if err != nil { - log.Error("Error starting node", "node", id) - } - wg.Done() - }(nodes[i]) - } - wg.Wait() - } -} - -// connect nodeCount number of nodes in a ring -func connectNodesInRing(net *Network, nodeCount int) ([]enode.ID, error) { - ids := make([]enode.ID, nodeCount) - for i := 0; i < nodeCount; i++ { - conf := adapters.RandomNodeConfig() - node, err := net.NewNodeWithConfig(conf) - if err != nil { - log.Error("Error creating a node!", "err", err) - return nil, err - } - ids[i] = node.ID() - } - - for _, id := range ids { - if err := net.Start(id); err != nil { - log.Error("Error starting a node!", "err", err) - return nil, err - } - log.Debug(fmt.Sprintf("node %v starting up", id)) - } - for i, id := range ids { - peerID := ids[(i+1)%len(ids)] - if err := net.Connect(id, peerID); err != nil { - log.Error("Error connecting a node to a peer!", "err", err) - return nil, err - } - } - - return ids, nil -} diff --git a/p2p/simulations/mocker_test.go b/p2p/simulations/mocker_test.go deleted file mode 100644 index 0112ee5cfd..0000000000 --- a/p2p/simulations/mocker_test.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package simulations simulates p2p networks. -// A mocker simulates starting and stopping real nodes in a network. -package simulations - -import ( - "encoding/json" - "net/http" - "net/url" - "strconv" - "sync" - "testing" - "time" - - "github.com/ethereum/go-ethereum/p2p/enode" -) - -func TestMocker(t *testing.T) { - //start the simulation HTTP server - _, s := testHTTPServer(t) - defer s.Close() - - //create a client - client := NewClient(s.URL) - - //start the network - err := client.StartNetwork() - if err != nil { - t.Fatalf("Could not start test network: %s", err) - } - //stop the network to terminate - defer func() { - err = client.StopNetwork() - if err != nil { - t.Fatalf("Could not stop test network: %s", err) - } - }() - - //get the list of available mocker types - resp, err := http.Get(s.URL + "/mocker") - if err != nil { - t.Fatalf("Could not get mocker list: %s", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - t.Fatalf("Invalid Status Code received, expected 200, got %d", resp.StatusCode) - } - - //check the list is at least 1 in size - var mockerlist []string - err = json.NewDecoder(resp.Body).Decode(&mockerlist) - if err != nil { - t.Fatalf("Error decoding JSON mockerlist: %s", err) - } - - if len(mockerlist) < 1 { - t.Fatalf("No mockers available") - } - - nodeCount := 10 - var wg sync.WaitGroup - - events := make(chan *Event, 10) - var opts SubscribeOpts - sub, err := client.SubscribeNetwork(events, opts) - defer sub.Unsubscribe() - - // wait until all nodes are started and connected - // store every node up event in a map (value is irrelevant, mimic Set datatype) - nodemap := make(map[enode.ID]bool) - nodesComplete := false - connCount := 0 - wg.Add(1) - go func() { - defer wg.Done() - - for connCount < (nodeCount-1)*2 { - select { - case event := <-events: - if isNodeUp(event) { - //add the correspondent node ID to the map - nodemap[event.Node.Config.ID] = true - //this means all nodes got a nodeUp event, so we can continue the test - if len(nodemap) == nodeCount { - nodesComplete = true - } - } else if event.Conn != nil && nodesComplete { - connCount += 1 - } - case <-time.After(30 * time.Second): - t.Errorf("Timeout waiting for nodes being started up!") - return - } - } - }() - - //take the last element of the mockerlist as the default mocker-type to ensure one is enabled - mockertype := mockerlist[len(mockerlist)-1] - //still, use hardcoded "probabilistic" one if available ;) - for _, m := range mockerlist { - if m == "probabilistic" { - mockertype = m - break - } - } - //start the mocker with nodeCount number of nodes - resp, err = http.PostForm(s.URL+"/mocker/start", url.Values{"mocker-type": {mockertype}, "node-count": {strconv.Itoa(nodeCount)}}) - if err != nil { - t.Fatalf("Could not start mocker: %s", err) - } - resp.Body.Close() - if resp.StatusCode != 200 { - t.Fatalf("Invalid Status Code received for starting mocker, expected 200, got %d", resp.StatusCode) - } - - wg.Wait() - - //check there are nodeCount number of nodes in the network - nodesInfo, err := client.GetNodes() - if err != nil { - t.Fatalf("Could not get nodes list: %s", err) - } - - if len(nodesInfo) != nodeCount { - t.Fatalf("Expected %d number of nodes, got: %d", nodeCount, len(nodesInfo)) - } - - //stop the mocker - resp, err = http.Post(s.URL+"/mocker/stop", "", nil) - if err != nil { - t.Fatalf("Could not stop mocker: %s", err) - } - resp.Body.Close() - if resp.StatusCode != 200 { - t.Fatalf("Invalid Status Code received for stopping mocker, expected 200, got %d", resp.StatusCode) - } - - //reset the network - resp, err = http.Post(s.URL+"/reset", "", nil) - if err != nil { - t.Fatalf("Could not reset network: %s", err) - } - resp.Body.Close() - - //now the number of nodes in the network should be zero - nodesInfo, err = client.GetNodes() - if err != nil { - t.Fatalf("Could not get nodes list: %s", err) - } - - if len(nodesInfo) != 0 { - t.Fatalf("Expected empty list of nodes, got: %d", len(nodesInfo)) - } -} - -func isNodeUp(event *Event) bool { - return event.Node != nil && event.Node.Up() -} diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go deleted file mode 100644 index 2eb8333cd6..0000000000 --- a/p2p/simulations/network.go +++ /dev/null @@ -1,1093 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package simulations - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "math/rand" - "sync" - "time" - - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" -) - -var DialBanTimeout = 200 * time.Millisecond - -// NetworkConfig defines configuration options for starting a Network -type NetworkConfig struct { - ID string `json:"id"` - DefaultService string `json:"default_service,omitempty"` -} - -// Network models a p2p simulation network which consists of a collection of -// simulated nodes and the connections which exist between them. -// -// The Network has a single NodeAdapter which is responsible for actually -// starting nodes and connecting them together. -// -// The Network emits events when nodes are started and stopped, when they are -// connected and disconnected, and also when messages are sent between nodes. -type Network struct { - NetworkConfig - - Nodes []*Node `json:"nodes"` - nodeMap map[enode.ID]int - - // Maps a node property string to node indexes of all nodes that hold this property - propertyMap map[string][]int - - Conns []*Conn `json:"conns"` - connMap map[string]int - - nodeAdapter adapters.NodeAdapter - events event.Feed - lock sync.RWMutex - quitc chan struct{} -} - -// NewNetwork returns a Network which uses the given NodeAdapter and NetworkConfig -func NewNetwork(nodeAdapter adapters.NodeAdapter, conf *NetworkConfig) *Network { - return &Network{ - NetworkConfig: *conf, - nodeAdapter: nodeAdapter, - nodeMap: make(map[enode.ID]int), - propertyMap: make(map[string][]int), - connMap: make(map[string]int), - quitc: make(chan struct{}), - } -} - -// Events returns the output event feed of the Network. -func (net *Network) Events() *event.Feed { - return &net.events -} - -// NewNodeWithConfig adds a new node to the network with the given config, -// returning an error if a node with the same ID or name already exists -func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) { - net.lock.Lock() - defer net.lock.Unlock() - - if conf.Reachable == nil { - conf.Reachable = func(otherID enode.ID) bool { - _, err := net.InitConn(conf.ID, otherID) - if err != nil && bytes.Compare(conf.ID.Bytes(), otherID.Bytes()) < 0 { - return false - } - return true - } - } - - // check the node doesn't already exist - if node := net.getNode(conf.ID); node != nil { - return nil, fmt.Errorf("node with ID %q already exists", conf.ID) - } - if node := net.getNodeByName(conf.Name); node != nil { - return nil, fmt.Errorf("node with name %q already exists", conf.Name) - } - - // if no services are configured, use the default service - if len(conf.Lifecycles) == 0 { - conf.Lifecycles = []string{net.DefaultService} - } - - // use the NodeAdapter to create the node - adapterNode, err := net.nodeAdapter.NewNode(conf) - if err != nil { - return nil, err - } - node := newNode(adapterNode, conf, false) - log.Trace("Node created", "id", conf.ID) - - nodeIndex := len(net.Nodes) - net.nodeMap[conf.ID] = nodeIndex - net.Nodes = append(net.Nodes, node) - - // Register any node properties with the network-level propertyMap - for _, property := range conf.Properties { - net.propertyMap[property] = append(net.propertyMap[property], nodeIndex) - } - - // emit a "control" event - net.events.Send(ControlEvent(node)) - - return node, nil -} - -// Config returns the network configuration -func (net *Network) Config() *NetworkConfig { - return &net.NetworkConfig -} - -// StartAll starts all nodes in the network -func (net *Network) StartAll() error { - for _, node := range net.Nodes { - if node.Up() { - continue - } - if err := net.Start(node.ID()); err != nil { - return err - } - } - return nil -} - -// StopAll stops all nodes in the network -func (net *Network) StopAll() error { - for _, node := range net.Nodes { - if !node.Up() { - continue - } - if err := net.Stop(node.ID()); err != nil { - return err - } - } - return nil -} - -// Start starts the node with the given ID -func (net *Network) Start(id enode.ID) error { - return net.startWithSnapshots(id, nil) -} - -// startWithSnapshots starts the node with the given ID using the give -// snapshots -func (net *Network) startWithSnapshots(id enode.ID, snapshots map[string][]byte) error { - net.lock.Lock() - defer net.lock.Unlock() - - node := net.getNode(id) - if node == nil { - return fmt.Errorf("node %v does not exist", id) - } - if node.Up() { - return fmt.Errorf("node %v already up", id) - } - log.Trace("Starting node", "id", id, "adapter", net.nodeAdapter.Name()) - if err := node.Start(snapshots); err != nil { - log.Warn("Node startup failed", "id", id, "err", err) - return err - } - node.SetUp(true) - log.Info("Started node", "id", id) - ev := NewEvent(node) - net.events.Send(ev) - - // subscribe to peer events - client, err := node.Client() - if err != nil { - return fmt.Errorf("error getting rpc client for node %v: %s", id, err) - } - events := make(chan *p2p.PeerEvent) - sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents") - if err != nil { - return fmt.Errorf("error getting peer events for node %v: %s", id, err) - } - go net.watchPeerEvents(id, events, sub) - return nil -} - -// watchPeerEvents reads peer events from the given channel and emits -// corresponding network events -func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub event.Subscription) { - defer func() { - sub.Unsubscribe() - - // assume the node is now down - net.lock.Lock() - defer net.lock.Unlock() - - node := net.getNode(id) - if node == nil { - return - } - node.SetUp(false) - ev := NewEvent(node) - net.events.Send(ev) - }() - for { - select { - case event, ok := <-events: - if !ok { - return - } - peer := event.Peer - switch event.Type { - case p2p.PeerEventTypeAdd: - net.DidConnect(id, peer) - - case p2p.PeerEventTypeDrop: - net.DidDisconnect(id, peer) - - case p2p.PeerEventTypeMsgSend: - net.DidSend(id, peer, event.Protocol, *event.MsgCode) - - case p2p.PeerEventTypeMsgRecv: - net.DidReceive(peer, id, event.Protocol, *event.MsgCode) - } - - case err := <-sub.Err(): - if err != nil { - log.Error("Error in peer event subscription", "id", id, "err", err) - } - return - } - } -} - -// Stop stops the node with the given ID -func (net *Network) Stop(id enode.ID) error { - // IMPORTANT: node.Stop() must NOT be called under net.lock as - // node.Reachable() closure has a reference to the network and - // calls net.InitConn() what also locks the network. => DEADLOCK - // That holds until the following ticket is not resolved: - - var err error - - node, err := func() (*Node, error) { - net.lock.Lock() - defer net.lock.Unlock() - - node := net.getNode(id) - if node == nil { - return nil, fmt.Errorf("node %v does not exist", id) - } - if !node.Up() { - return nil, fmt.Errorf("node %v already down", id) - } - node.SetUp(false) - return node, nil - }() - if err != nil { - return err - } - - err = node.Stop() // must be called without net.lock - - net.lock.Lock() - defer net.lock.Unlock() - - if err != nil { - node.SetUp(true) - return err - } - log.Info("Stopped node", "id", id, "err", err) - ev := ControlEvent(node) - net.events.Send(ev) - return nil -} - -// Connect connects two nodes together by calling the "admin_addPeer" RPC -// method on the "one" node so that it connects to the "other" node -func (net *Network) Connect(oneID, otherID enode.ID) error { - net.lock.Lock() - defer net.lock.Unlock() - return net.connect(oneID, otherID) -} - -func (net *Network) connect(oneID, otherID enode.ID) error { - log.Debug("Connecting nodes with addPeer", "id", oneID, "other", otherID) - conn, err := net.initConn(oneID, otherID) - if err != nil { - return err - } - client, err := conn.one.Client() - if err != nil { - return err - } - net.events.Send(ControlEvent(conn)) - return client.Call(nil, "admin_addPeer", string(conn.other.Addr())) -} - -// Disconnect disconnects two nodes by calling the "admin_removePeer" RPC -// method on the "one" node so that it disconnects from the "other" node -func (net *Network) Disconnect(oneID, otherID enode.ID) error { - conn := net.GetConn(oneID, otherID) - if conn == nil { - return fmt.Errorf("connection between %v and %v does not exist", oneID, otherID) - } - if !conn.Up { - return fmt.Errorf("%v and %v already disconnected", oneID, otherID) - } - client, err := conn.one.Client() - if err != nil { - return err - } - net.events.Send(ControlEvent(conn)) - return client.Call(nil, "admin_removePeer", string(conn.other.Addr())) -} - -// DidConnect tracks the fact that the "one" node connected to the "other" node -func (net *Network) DidConnect(one, other enode.ID) error { - net.lock.Lock() - defer net.lock.Unlock() - conn, err := net.getOrCreateConn(one, other) - if err != nil { - return fmt.Errorf("connection between %v and %v does not exist", one, other) - } - if conn.Up { - return fmt.Errorf("%v and %v already connected", one, other) - } - conn.Up = true - net.events.Send(NewEvent(conn)) - return nil -} - -// DidDisconnect tracks the fact that the "one" node disconnected from the -// "other" node -func (net *Network) DidDisconnect(one, other enode.ID) error { - net.lock.Lock() - defer net.lock.Unlock() - conn := net.getConn(one, other) - if conn == nil { - return fmt.Errorf("connection between %v and %v does not exist", one, other) - } - if !conn.Up { - return fmt.Errorf("%v and %v already disconnected", one, other) - } - conn.Up = false - conn.initiated = time.Now().Add(-DialBanTimeout) - net.events.Send(NewEvent(conn)) - return nil -} - -// DidSend tracks the fact that "sender" sent a message to "receiver" -func (net *Network) DidSend(sender, receiver enode.ID, proto string, code uint64) error { - msg := &Msg{ - One: sender, - Other: receiver, - Protocol: proto, - Code: code, - Received: false, - } - net.events.Send(NewEvent(msg)) - return nil -} - -// DidReceive tracks the fact that "receiver" received a message from "sender" -func (net *Network) DidReceive(sender, receiver enode.ID, proto string, code uint64) error { - msg := &Msg{ - One: sender, - Other: receiver, - Protocol: proto, - Code: code, - Received: true, - } - net.events.Send(NewEvent(msg)) - return nil -} - -// GetNode gets the node with the given ID, returning nil if the node does not -// exist -func (net *Network) GetNode(id enode.ID) *Node { - net.lock.RLock() - defer net.lock.RUnlock() - return net.getNode(id) -} - -func (net *Network) getNode(id enode.ID) *Node { - i, found := net.nodeMap[id] - if !found { - return nil - } - return net.Nodes[i] -} - -// GetNodeByName gets the node with the given name, returning nil if the node does -// not exist -func (net *Network) GetNodeByName(name string) *Node { - net.lock.RLock() - defer net.lock.RUnlock() - return net.getNodeByName(name) -} - -func (net *Network) getNodeByName(name string) *Node { - for _, node := range net.Nodes { - if node.Config.Name == name { - return node - } - } - return nil -} - -// GetNodeIDs returns the IDs of all existing nodes -// Nodes can optionally be excluded by specifying their enode.ID. -func (net *Network) GetNodeIDs(excludeIDs ...enode.ID) []enode.ID { - net.lock.RLock() - defer net.lock.RUnlock() - - return net.getNodeIDs(excludeIDs) -} - -func (net *Network) getNodeIDs(excludeIDs []enode.ID) []enode.ID { - // Get all current nodeIDs - nodeIDs := make([]enode.ID, 0, len(net.nodeMap)) - for id := range net.nodeMap { - nodeIDs = append(nodeIDs, id) - } - - if len(excludeIDs) > 0 { - // Return the difference of nodeIDs and excludeIDs - return filterIDs(nodeIDs, excludeIDs) - } - return nodeIDs -} - -// GetNodes returns the existing nodes. -// Nodes can optionally be excluded by specifying their enode.ID. -func (net *Network) GetNodes(excludeIDs ...enode.ID) []*Node { - net.lock.RLock() - defer net.lock.RUnlock() - - return net.getNodes(excludeIDs) -} - -func (net *Network) getNodes(excludeIDs []enode.ID) []*Node { - if len(excludeIDs) > 0 { - nodeIDs := net.getNodeIDs(excludeIDs) - return net.getNodesByID(nodeIDs) - } - return net.Nodes -} - -// GetNodesByID returns existing nodes with the given enode.IDs. -// If a node doesn't exist with a given enode.ID, it is ignored. -func (net *Network) GetNodesByID(nodeIDs []enode.ID) []*Node { - net.lock.RLock() - defer net.lock.RUnlock() - - return net.getNodesByID(nodeIDs) -} - -func (net *Network) getNodesByID(nodeIDs []enode.ID) []*Node { - nodes := make([]*Node, 0, len(nodeIDs)) - for _, id := range nodeIDs { - node := net.getNode(id) - if node != nil { - nodes = append(nodes, node) - } - } - - return nodes -} - -// GetNodesByProperty returns existing nodes that have the given property string registered in their NodeConfig -func (net *Network) GetNodesByProperty(property string) []*Node { - net.lock.RLock() - defer net.lock.RUnlock() - - return net.getNodesByProperty(property) -} - -func (net *Network) getNodesByProperty(property string) []*Node { - nodes := make([]*Node, 0, len(net.propertyMap[property])) - for _, nodeIndex := range net.propertyMap[property] { - nodes = append(nodes, net.Nodes[nodeIndex]) - } - - return nodes -} - -// GetNodeIDsByProperty returns existing node's enode IDs that have the given property string registered in the NodeConfig -func (net *Network) GetNodeIDsByProperty(property string) []enode.ID { - net.lock.RLock() - defer net.lock.RUnlock() - - return net.getNodeIDsByProperty(property) -} - -func (net *Network) getNodeIDsByProperty(property string) []enode.ID { - nodeIDs := make([]enode.ID, 0, len(net.propertyMap[property])) - for _, nodeIndex := range net.propertyMap[property] { - node := net.Nodes[nodeIndex] - nodeIDs = append(nodeIDs, node.ID()) - } - - return nodeIDs -} - -// GetRandomUpNode returns a random node on the network, which is running. -func (net *Network) GetRandomUpNode(excludeIDs ...enode.ID) *Node { - net.lock.RLock() - defer net.lock.RUnlock() - return net.getRandomUpNode(excludeIDs...) -} - -// getRandomUpNode returns a random node on the network, which is running. -func (net *Network) getRandomUpNode(excludeIDs ...enode.ID) *Node { - return net.getRandomNode(net.getUpNodeIDs(), excludeIDs) -} - -func (net *Network) getUpNodeIDs() (ids []enode.ID) { - for _, node := range net.Nodes { - if node.Up() { - ids = append(ids, node.ID()) - } - } - return ids -} - -// GetRandomDownNode returns a random node on the network, which is stopped. -func (net *Network) GetRandomDownNode(excludeIDs ...enode.ID) *Node { - net.lock.RLock() - defer net.lock.RUnlock() - return net.getRandomNode(net.getDownNodeIDs(), excludeIDs) -} - -func (net *Network) getDownNodeIDs() (ids []enode.ID) { - for _, node := range net.Nodes { - if !node.Up() { - ids = append(ids, node.ID()) - } - } - return ids -} - -// GetRandomNode returns a random node on the network, regardless of whether it is running or not -func (net *Network) GetRandomNode(excludeIDs ...enode.ID) *Node { - net.lock.RLock() - defer net.lock.RUnlock() - return net.getRandomNode(net.getNodeIDs(nil), excludeIDs) // no need to exclude twice -} - -func (net *Network) getRandomNode(ids []enode.ID, excludeIDs []enode.ID) *Node { - filtered := filterIDs(ids, excludeIDs) - - l := len(filtered) - if l == 0 { - return nil - } - return net.getNode(filtered[rand.Intn(l)]) -} - -func filterIDs(ids []enode.ID, excludeIDs []enode.ID) []enode.ID { - exclude := make(map[enode.ID]bool) - for _, id := range excludeIDs { - exclude[id] = true - } - var filtered []enode.ID - for _, id := range ids { - if _, found := exclude[id]; !found { - filtered = append(filtered, id) - } - } - return filtered -} - -// GetConn returns the connection which exists between "one" and "other" -// regardless of which node initiated the connection -func (net *Network) GetConn(oneID, otherID enode.ID) *Conn { - net.lock.RLock() - defer net.lock.RUnlock() - return net.getConn(oneID, otherID) -} - -// GetOrCreateConn is like GetConn but creates the connection if it doesn't -// already exist -func (net *Network) GetOrCreateConn(oneID, otherID enode.ID) (*Conn, error) { - net.lock.Lock() - defer net.lock.Unlock() - return net.getOrCreateConn(oneID, otherID) -} - -func (net *Network) getOrCreateConn(oneID, otherID enode.ID) (*Conn, error) { - if conn := net.getConn(oneID, otherID); conn != nil { - return conn, nil - } - - one := net.getNode(oneID) - if one == nil { - return nil, fmt.Errorf("node %v does not exist", oneID) - } - other := net.getNode(otherID) - if other == nil { - return nil, fmt.Errorf("node %v does not exist", otherID) - } - conn := &Conn{ - One: oneID, - Other: otherID, - one: one, - other: other, - } - label := ConnLabel(oneID, otherID) - net.connMap[label] = len(net.Conns) - net.Conns = append(net.Conns, conn) - return conn, nil -} - -func (net *Network) getConn(oneID, otherID enode.ID) *Conn { - label := ConnLabel(oneID, otherID) - i, found := net.connMap[label] - if !found { - return nil - } - return net.Conns[i] -} - -// InitConn retrieves the connection model for the connection between -// peers 'oneID' and 'otherID', or creates a new one if it does not exist -// the order of nodes does not matter, i.e., Conn(i,j) == Conn(j, i) -// it checks if the connection is already up, and if the nodes are running -// NOTE: -// it also checks whether there has been recent attempt to connect the peers -// this is cheating as the simulation is used as an oracle and know about -// remote peers attempt to connect to a node which will then not initiate the connection -func (net *Network) InitConn(oneID, otherID enode.ID) (*Conn, error) { - net.lock.Lock() - defer net.lock.Unlock() - return net.initConn(oneID, otherID) -} - -func (net *Network) initConn(oneID, otherID enode.ID) (*Conn, error) { - if oneID == otherID { - return nil, fmt.Errorf("refusing to connect to self %v", oneID) - } - conn, err := net.getOrCreateConn(oneID, otherID) - if err != nil { - return nil, err - } - if conn.Up { - return nil, fmt.Errorf("%v and %v already connected", oneID, otherID) - } - if time.Since(conn.initiated) < DialBanTimeout { - return nil, fmt.Errorf("connection between %v and %v recently attempted", oneID, otherID) - } - - err = conn.nodesUp() - if err != nil { - log.Trace("Nodes not up", "err", err) - return nil, fmt.Errorf("nodes not up: %v", err) - } - log.Debug("Connection initiated", "id", oneID, "other", otherID) - conn.initiated = time.Now() - return conn, nil -} - -// Shutdown stops all nodes in the network and closes the quit channel -func (net *Network) Shutdown() { - for _, node := range net.Nodes { - log.Debug("Stopping node", "id", node.ID()) - if err := node.Stop(); err != nil { - log.Warn("Can't stop node", "id", node.ID(), "err", err) - } - } - close(net.quitc) -} - -// Reset resets all network properties: -// empties the nodes and the connection list -func (net *Network) Reset() { - net.lock.Lock() - defer net.lock.Unlock() - - //re-initialize the maps - net.connMap = make(map[string]int) - net.nodeMap = make(map[enode.ID]int) - net.propertyMap = make(map[string][]int) - - net.Nodes = nil - net.Conns = nil -} - -// Node is a wrapper around adapters.Node which is used to track the status -// of a node in the network -type Node struct { - adapters.Node `json:"-"` - - // Config if the config used to created the node - Config *adapters.NodeConfig `json:"config"` - - // up tracks whether or not the node is running - up bool - upMu *sync.RWMutex -} - -func newNode(an adapters.Node, ac *adapters.NodeConfig, up bool) *Node { - return &Node{Node: an, Config: ac, up: up, upMu: new(sync.RWMutex)} -} - -func (n *Node) copy() *Node { - configCpy := *n.Config - return newNode(n.Node, &configCpy, n.Up()) -} - -// Up returns whether the node is currently up (online) -func (n *Node) Up() bool { - n.upMu.RLock() - defer n.upMu.RUnlock() - return n.up -} - -// SetUp sets the up (online) status of the nodes with the given value -func (n *Node) SetUp(up bool) { - n.upMu.Lock() - defer n.upMu.Unlock() - n.up = up -} - -// ID returns the ID of the node -func (n *Node) ID() enode.ID { - return n.Config.ID -} - -// String returns a log-friendly string -func (n *Node) String() string { - return fmt.Sprintf("Node %v", n.ID().TerminalString()) -} - -// NodeInfo returns information about the node -func (n *Node) NodeInfo() *p2p.NodeInfo { - // avoid a panic if the node is not started yet - if n.Node == nil { - return nil - } - info := n.Node.NodeInfo() - info.Name = n.Config.Name - return info -} - -// MarshalJSON implements the json.Marshaler interface so that the encoded -// JSON includes the NodeInfo -func (n *Node) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - Info *p2p.NodeInfo `json:"info,omitempty"` - Config *adapters.NodeConfig `json:"config,omitempty"` - Up bool `json:"up"` - }{ - Info: n.NodeInfo(), - Config: n.Config, - Up: n.Up(), - }) -} - -// UnmarshalJSON implements json.Unmarshaler interface so that we don't lose Node.up -// status. IMPORTANT: The implementation is incomplete; we lose p2p.NodeInfo. -func (n *Node) UnmarshalJSON(raw []byte) error { - // TODO: How should we turn back NodeInfo into n.Node? - // Ticket: https://github.com/ethersphere/go-ethereum/issues/1177 - var node struct { - Config *adapters.NodeConfig `json:"config,omitempty"` - Up bool `json:"up"` - } - if err := json.Unmarshal(raw, &node); err != nil { - return err - } - *n = *newNode(nil, node.Config, node.Up) - return nil -} - -// Conn represents a connection between two nodes in the network -type Conn struct { - // One is the node which initiated the connection - One enode.ID `json:"one"` - - // Other is the node which the connection was made to - Other enode.ID `json:"other"` - - // Up tracks whether or not the connection is active - Up bool `json:"up"` - // Registers when the connection was grabbed to dial - initiated time.Time - - one *Node - other *Node -} - -// nodesUp returns whether both nodes are currently up -func (c *Conn) nodesUp() error { - if !c.one.Up() { - return fmt.Errorf("one %v is not up", c.One) - } - if !c.other.Up() { - return fmt.Errorf("other %v is not up", c.Other) - } - return nil -} - -// String returns a log-friendly string -func (c *Conn) String() string { - return fmt.Sprintf("Conn %v->%v", c.One.TerminalString(), c.Other.TerminalString()) -} - -// Msg represents a p2p message sent between two nodes in the network -type Msg struct { - One enode.ID `json:"one"` - Other enode.ID `json:"other"` - Protocol string `json:"protocol"` - Code uint64 `json:"code"` - Received bool `json:"received"` -} - -// String returns a log-friendly string -func (m *Msg) String() string { - return fmt.Sprintf("Msg(%d) %v->%v", m.Code, m.One.TerminalString(), m.Other.TerminalString()) -} - -// ConnLabel generates a deterministic string which represents a connection -// between two nodes, used to compare if two connections are between the same -// nodes -func ConnLabel(source, target enode.ID) string { - var first, second enode.ID - if bytes.Compare(source.Bytes(), target.Bytes()) > 0 { - first = target - second = source - } else { - first = source - second = target - } - return fmt.Sprintf("%v-%v", first, second) -} - -// Snapshot represents the state of a network at a single point in time and can -// be used to restore the state of a network -type Snapshot struct { - Nodes []NodeSnapshot `json:"nodes,omitempty"` - Conns []Conn `json:"conns,omitempty"` -} - -// NodeSnapshot represents the state of a node in the network -type NodeSnapshot struct { - Node Node `json:"node,omitempty"` - - // Snapshots is arbitrary data gathered from calling node.Snapshots() - Snapshots map[string][]byte `json:"snapshots,omitempty"` -} - -// Snapshot creates a network snapshot -func (net *Network) Snapshot() (*Snapshot, error) { - return net.snapshot(nil, nil) -} - -func (net *Network) SnapshotWithServices(addServices []string, removeServices []string) (*Snapshot, error) { - return net.snapshot(addServices, removeServices) -} - -func (net *Network) snapshot(addServices []string, removeServices []string) (*Snapshot, error) { - net.lock.Lock() - defer net.lock.Unlock() - snap := &Snapshot{ - Nodes: make([]NodeSnapshot, len(net.Nodes)), - } - for i, node := range net.Nodes { - snap.Nodes[i] = NodeSnapshot{Node: *node.copy()} - if !node.Up() { - continue - } - snapshots, err := node.Snapshots() - if err != nil { - return nil, err - } - snap.Nodes[i].Snapshots = snapshots - for _, addSvc := range addServices { - haveSvc := false - for _, svc := range snap.Nodes[i].Node.Config.Lifecycles { - if svc == addSvc { - haveSvc = true - break - } - } - if !haveSvc { - snap.Nodes[i].Node.Config.Lifecycles = append(snap.Nodes[i].Node.Config.Lifecycles, addSvc) - } - } - if len(removeServices) > 0 { - var cleanedServices []string - for _, svc := range snap.Nodes[i].Node.Config.Lifecycles { - haveSvc := false - for _, rmSvc := range removeServices { - if rmSvc == svc { - haveSvc = true - break - } - } - if !haveSvc { - cleanedServices = append(cleanedServices, svc) - } - } - snap.Nodes[i].Node.Config.Lifecycles = cleanedServices - } - } - for _, conn := range net.Conns { - if conn.Up { - snap.Conns = append(snap.Conns, *conn) - } - } - return snap, nil -} - -// longrunning tests may need a longer timeout -var snapshotLoadTimeout = 900 * time.Second - -// Load loads a network snapshot -func (net *Network) Load(snap *Snapshot) error { - // Start nodes. - for _, n := range snap.Nodes { - if _, err := net.NewNodeWithConfig(n.Node.Config); err != nil { - return err - } - if !n.Node.Up() { - continue - } - if err := net.startWithSnapshots(n.Node.Config.ID, n.Snapshots); err != nil { - return err - } - } - - // Prepare connection events counter. - allConnected := make(chan struct{}) // closed when all connections are established - done := make(chan struct{}) // ensures that the event loop goroutine is terminated - defer close(done) - - // Subscribe to event channel. - // It needs to be done outside of the event loop goroutine (created below) - // to ensure that the event channel is blocking before connect calls are made. - events := make(chan *Event) - sub := net.Events().Subscribe(events) - defer sub.Unsubscribe() - - go func() { - // Expected number of connections. - total := len(snap.Conns) - // Set of all established connections from the snapshot, not other connections. - // Key array element 0 is the connection One field value, and element 1 connection Other field. - connections := make(map[[2]enode.ID]struct{}, total) - - for { - select { - case e := <-events: - // Ignore control events as they do not represent - // connect or disconnect (Up) state change. - if e.Control { - continue - } - // Detect only connection events. - if e.Type != EventTypeConn { - continue - } - connection := [2]enode.ID{e.Conn.One, e.Conn.Other} - // Nodes are still not connected or have been disconnected. - if !e.Conn.Up { - // Delete the connection from the set of established connections. - // This will prevent false positive in case disconnections happen. - delete(connections, connection) - log.Warn("load snapshot: unexpected disconnection", "one", e.Conn.One, "other", e.Conn.Other) - continue - } - // Check that the connection is from the snapshot. - for _, conn := range snap.Conns { - if conn.One == e.Conn.One && conn.Other == e.Conn.Other { - // Add the connection to the set of established connections. - connections[connection] = struct{}{} - if len(connections) == total { - // Signal that all nodes are connected. - close(allConnected) - return - } - - break - } - } - case <-done: - // Load function returned, terminate this goroutine. - return - } - } - }() - - // Start connecting. - for _, conn := range snap.Conns { - if !net.GetNode(conn.One).Up() || !net.GetNode(conn.Other).Up() { - //in this case, at least one of the nodes of a connection is not up, - //so it would result in the snapshot `Load` to fail - continue - } - if err := net.Connect(conn.One, conn.Other); err != nil { - return err - } - } - - timeout := time.NewTimer(snapshotLoadTimeout) - defer timeout.Stop() - - select { - // Wait until all connections from the snapshot are established. - case <-allConnected: - // Make sure that we do not wait forever. - case <-timeout.C: - return errors.New("snapshot connections not established") - } - return nil -} - -// Subscribe reads control events from a channel and executes them -func (net *Network) Subscribe(events chan *Event) { - for { - select { - case event, ok := <-events: - if !ok { - return - } - if event.Control { - net.executeControlEvent(event) - } - case <-net.quitc: - return - } - } -} - -func (net *Network) executeControlEvent(event *Event) { - log.Trace("Executing control event", "type", event.Type, "event", event) - switch event.Type { - case EventTypeNode: - if err := net.executeNodeEvent(event); err != nil { - log.Error("Error executing node event", "event", event, "err", err) - } - case EventTypeConn: - if err := net.executeConnEvent(event); err != nil { - log.Error("Error executing conn event", "event", event, "err", err) - } - case EventTypeMsg: - log.Warn("Ignoring control msg event") - } -} - -func (net *Network) executeNodeEvent(e *Event) error { - if !e.Node.Up() { - return net.Stop(e.Node.ID()) - } - - if _, err := net.NewNodeWithConfig(e.Node.Config); err != nil { - return err - } - return net.Start(e.Node.ID()) -} - -func (net *Network) executeConnEvent(e *Event) error { - if e.Conn.Up { - return net.Connect(e.Conn.One, e.Conn.Other) - } - return net.Disconnect(e.Conn.One, e.Conn.Other) -} diff --git a/p2p/simulations/network_test.go b/p2p/simulations/network_test.go deleted file mode 100644 index 4ed1e4e6c3..0000000000 --- a/p2p/simulations/network_test.go +++ /dev/null @@ -1,872 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package simulations - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "reflect" - "strconv" - "strings" - "testing" - "time" - - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" -) - -// Tests that a created snapshot with a minimal service only contains the expected connections -// and that a network when loaded with this snapshot only contains those same connections -func TestSnapshot(t *testing.T) { - // PART I - // create snapshot from ring network - - // this is a minimal service, whose protocol will take exactly one message OR close of connection before quitting - adapter := adapters.NewSimAdapter(adapters.LifecycleConstructors{ - "noopwoop": func(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { - return NewNoopService(nil), nil - }, - }) - - // create network - network := NewNetwork(adapter, &NetworkConfig{ - DefaultService: "noopwoop", - }) - // \todo consider making a member of network, set to true threadsafe when shutdown - runningOne := true - defer func() { - if runningOne { - network.Shutdown() - } - }() - - // create and start nodes - nodeCount := 20 - ids := make([]enode.ID, nodeCount) - for i := 0; i < nodeCount; i++ { - conf := adapters.RandomNodeConfig() - node, err := network.NewNodeWithConfig(conf) - if err != nil { - t.Fatalf("error creating node: %s", err) - } - if err := network.Start(node.ID()); err != nil { - t.Fatalf("error starting node: %s", err) - } - ids[i] = node.ID() - } - - // subscribe to peer events - evC := make(chan *Event) - sub := network.Events().Subscribe(evC) - defer sub.Unsubscribe() - - // connect nodes in a ring - // spawn separate thread to avoid deadlock in the event listeners - connectErr := make(chan error, 1) - go func() { - for i, id := range ids { - peerID := ids[(i+1)%len(ids)] - if err := network.Connect(id, peerID); err != nil { - connectErr <- err - return - } - } - }() - - // collect connection events up to expected number - ctx, cancel := context.WithTimeout(context.TODO(), time.Second) - defer cancel() - checkIds := make(map[enode.ID][]enode.ID) - connEventCount := nodeCount -OUTER: - for { - select { - case <-ctx.Done(): - t.Fatal(ctx.Err()) - case err := <-connectErr: - t.Fatal(err) - case ev := <-evC: - if ev.Type == EventTypeConn && !ev.Control { - // fail on any disconnect - if !ev.Conn.Up { - t.Fatalf("unexpected disconnect: %v -> %v", ev.Conn.One, ev.Conn.Other) - } - checkIds[ev.Conn.One] = append(checkIds[ev.Conn.One], ev.Conn.Other) - checkIds[ev.Conn.Other] = append(checkIds[ev.Conn.Other], ev.Conn.One) - connEventCount-- - log.Debug("ev", "count", connEventCount) - if connEventCount == 0 { - break OUTER - } - } - } - } - - // create snapshot of current network - snap, err := network.Snapshot() - if err != nil { - t.Fatal(err) - } - j, err := json.Marshal(snap) - if err != nil { - t.Fatal(err) - } - log.Debug("snapshot taken", "nodes", len(snap.Nodes), "conns", len(snap.Conns), "json", string(j)) - - // verify that the snap element numbers check out - if len(checkIds) != len(snap.Conns) || len(checkIds) != len(snap.Nodes) { - t.Fatalf("snapshot wrong node,conn counts %d,%d != %d", len(snap.Nodes), len(snap.Conns), len(checkIds)) - } - - // shut down sim network - runningOne = false - sub.Unsubscribe() - network.Shutdown() - - // check that we have all the expected connections in the snapshot - for nodid, nodConns := range checkIds { - for _, nodConn := range nodConns { - var match bool - for _, snapConn := range snap.Conns { - if snapConn.One == nodid && snapConn.Other == nodConn { - match = true - break - } else if snapConn.Other == nodid && snapConn.One == nodConn { - match = true - break - } - } - if !match { - t.Fatalf("snapshot missing conn %v -> %v", nodid, nodConn) - } - } - } - log.Info("snapshot checked") - - // PART II - // load snapshot and verify that exactly same connections are formed - - adapter = adapters.NewSimAdapter(adapters.LifecycleConstructors{ - "noopwoop": func(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { - return NewNoopService(nil), nil - }, - }) - network = NewNetwork(adapter, &NetworkConfig{ - DefaultService: "noopwoop", - }) - defer func() { - network.Shutdown() - }() - - // subscribe to peer events - // every node up and conn up event will generate one additional control event - // therefore multiply the count by two - evC = make(chan *Event, (len(snap.Conns)*2)+(len(snap.Nodes)*2)) - sub = network.Events().Subscribe(evC) - defer sub.Unsubscribe() - - // load the snapshot - // spawn separate thread to avoid deadlock in the event listeners - err = network.Load(snap) - if err != nil { - t.Fatal(err) - } - - // collect connection events up to expected number - ctx, cancel = context.WithTimeout(context.TODO(), time.Second*3) - defer cancel() - - connEventCount = nodeCount - -OuterTwo: - for { - select { - case <-ctx.Done(): - t.Fatal(ctx.Err()) - case ev := <-evC: - if ev.Type == EventTypeConn && !ev.Control { - // fail on any disconnect - if !ev.Conn.Up { - t.Fatalf("unexpected disconnect: %v -> %v", ev.Conn.One, ev.Conn.Other) - } - log.Debug("conn", "on", ev.Conn.One, "other", ev.Conn.Other) - checkIds[ev.Conn.One] = append(checkIds[ev.Conn.One], ev.Conn.Other) - checkIds[ev.Conn.Other] = append(checkIds[ev.Conn.Other], ev.Conn.One) - connEventCount-- - log.Debug("ev", "count", connEventCount) - if connEventCount == 0 { - break OuterTwo - } - } - } - } - - // check that we have all expected connections in the network - for _, snapConn := range snap.Conns { - var match bool - for nodid, nodConns := range checkIds { - for _, nodConn := range nodConns { - if snapConn.One == nodid && snapConn.Other == nodConn { - match = true - break - } else if snapConn.Other == nodid && snapConn.One == nodConn { - match = true - break - } - } - } - if !match { - t.Fatalf("network missing conn %v -> %v", snapConn.One, snapConn.Other) - } - } - - // verify that network didn't generate any other additional connection events after the ones we have collected within a reasonable period of time - ctx, cancel = context.WithTimeout(context.TODO(), time.Second) - defer cancel() - select { - case <-ctx.Done(): - case ev := <-evC: - if ev.Type == EventTypeConn { - t.Fatalf("Superfluous conn found %v -> %v", ev.Conn.One, ev.Conn.Other) - } - } - - // This test validates if all connections from the snapshot - // are created in the network. - t.Run("conns after load", func(t *testing.T) { - // Create new network. - n := NewNetwork( - adapters.NewSimAdapter(adapters.LifecycleConstructors{ - "noopwoop": func(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { - return NewNoopService(nil), nil - }, - }), - &NetworkConfig{ - DefaultService: "noopwoop", - }, - ) - defer n.Shutdown() - - // Load the same snapshot. - err := n.Load(snap) - if err != nil { - t.Fatal(err) - } - - // Check every connection from the snapshot - // if it is in the network, too. - for _, c := range snap.Conns { - if n.GetConn(c.One, c.Other) == nil { - t.Errorf("missing connection: %s -> %s", c.One, c.Other) - } - } - }) -} - -// TestNetworkSimulation creates a multi-node simulation network with each node -// connected in a ring topology, checks that all nodes successfully handshake -// with each other and that a snapshot fully represents the desired topology -func TestNetworkSimulation(t *testing.T) { - // create simulation network with 20 testService nodes - adapter := adapters.NewSimAdapter(adapters.LifecycleConstructors{ - "test": newTestService, - }) - network := NewNetwork(adapter, &NetworkConfig{ - DefaultService: "test", - }) - defer network.Shutdown() - nodeCount := 20 - ids := make([]enode.ID, nodeCount) - for i := 0; i < nodeCount; i++ { - conf := adapters.RandomNodeConfig() - node, err := network.NewNodeWithConfig(conf) - if err != nil { - t.Fatalf("error creating node: %s", err) - } - if err := network.Start(node.ID()); err != nil { - t.Fatalf("error starting node: %s", err) - } - ids[i] = node.ID() - } - - // perform a check which connects the nodes in a ring (so each node is - // connected to exactly two peers) and then checks that all nodes - // performed two handshakes by checking their peerCount - action := func(_ context.Context) error { - for i, id := range ids { - peerID := ids[(i+1)%len(ids)] - if err := network.Connect(id, peerID); err != nil { - return err - } - } - return nil - } - check := func(ctx context.Context, id enode.ID) (bool, error) { - // check we haven't run out of time - select { - case <-ctx.Done(): - return false, ctx.Err() - default: - } - - // get the node - node := network.GetNode(id) - if node == nil { - return false, fmt.Errorf("unknown node: %s", id) - } - - // check it has exactly two peers - client, err := node.Client() - if err != nil { - return false, err - } - var peerCount int64 - if err := client.CallContext(ctx, &peerCount, "test_peerCount"); err != nil { - return false, err - } - switch { - case peerCount < 2: - return false, nil - case peerCount == 2: - return true, nil - default: - return false, fmt.Errorf("unexpected peerCount: %d", peerCount) - } - } - - timeout := 30 * time.Second - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - // trigger a check every 100ms - trigger := make(chan enode.ID) - go triggerChecks(ctx, ids, trigger, 100*time.Millisecond) - - result := NewSimulation(network).Run(ctx, &Step{ - Action: action, - Trigger: trigger, - Expect: &Expectation{ - Nodes: ids, - Check: check, - }, - }) - if result.Error != nil { - t.Fatalf("simulation failed: %s", result.Error) - } - - // take a network snapshot and check it contains the correct topology - snap, err := network.Snapshot() - if err != nil { - t.Fatal(err) - } - if len(snap.Nodes) != nodeCount { - t.Fatalf("expected snapshot to contain %d nodes, got %d", nodeCount, len(snap.Nodes)) - } - if len(snap.Conns) != nodeCount { - t.Fatalf("expected snapshot to contain %d connections, got %d", nodeCount, len(snap.Conns)) - } - for i, id := range ids { - conn := snap.Conns[i] - if conn.One != id { - t.Fatalf("expected conn[%d].One to be %s, got %s", i, id, conn.One) - } - peerID := ids[(i+1)%len(ids)] - if conn.Other != peerID { - t.Fatalf("expected conn[%d].Other to be %s, got %s", i, peerID, conn.Other) - } - } -} - -func createTestNodes(count int, network *Network) (nodes []*Node, err error) { - for i := 0; i < count; i++ { - nodeConf := adapters.RandomNodeConfig() - node, err := network.NewNodeWithConfig(nodeConf) - if err != nil { - return nil, err - } - if err := network.Start(node.ID()); err != nil { - return nil, err - } - - nodes = append(nodes, node) - } - - return nodes, nil -} - -func createTestNodesWithProperty(property string, count int, network *Network) (propertyNodes []*Node, err error) { - for i := 0; i < count; i++ { - nodeConf := adapters.RandomNodeConfig() - nodeConf.Properties = append(nodeConf.Properties, property) - - node, err := network.NewNodeWithConfig(nodeConf) - if err != nil { - return nil, err - } - if err := network.Start(node.ID()); err != nil { - return nil, err - } - - propertyNodes = append(propertyNodes, node) - } - - return propertyNodes, nil -} - -// TestGetNodeIDs creates a set of nodes and attempts to retrieve their IDs,. -// It then tests again whilst excluding a node ID from being returned. -// If a node ID is not returned, or more node IDs than expected are returned, the test fails. -func TestGetNodeIDs(t *testing.T) { - adapter := adapters.NewSimAdapter(adapters.LifecycleConstructors{ - "test": newTestService, - }) - network := NewNetwork(adapter, &NetworkConfig{ - DefaultService: "test", - }) - defer network.Shutdown() - - numNodes := 5 - nodes, err := createTestNodes(numNodes, network) - if err != nil { - t.Fatalf("Could not create test nodes %v", err) - } - - gotNodeIDs := network.GetNodeIDs() - if len(gotNodeIDs) != numNodes { - t.Fatalf("Expected %d nodes, got %d", numNodes, len(gotNodeIDs)) - } - - for _, node1 := range nodes { - match := false - for _, node2ID := range gotNodeIDs { - if bytes.Equal(node1.ID().Bytes(), node2ID.Bytes()) { - match = true - break - } - } - - if !match { - t.Fatalf("A created node was not returned by GetNodes(), ID: %s", node1.ID().String()) - } - } - - excludeNodeID := nodes[3].ID() - gotNodeIDsExcl := network.GetNodeIDs(excludeNodeID) - if len(gotNodeIDsExcl) != numNodes-1 { - t.Fatalf("Expected one less node ID to be returned") - } - for _, nodeID := range gotNodeIDsExcl { - if bytes.Equal(excludeNodeID.Bytes(), nodeID.Bytes()) { - t.Fatalf("GetNodeIDs returned the node ID we excluded, ID: %s", nodeID.String()) - } - } -} - -// TestGetNodes creates a set of nodes and attempts to retrieve them again. -// It then tests again whilst excluding a node from being returned. -// If a node is not returned, or more nodes than expected are returned, the test fails. -func TestGetNodes(t *testing.T) { - adapter := adapters.NewSimAdapter(adapters.LifecycleConstructors{ - "test": newTestService, - }) - network := NewNetwork(adapter, &NetworkConfig{ - DefaultService: "test", - }) - defer network.Shutdown() - - numNodes := 5 - nodes, err := createTestNodes(numNodes, network) - if err != nil { - t.Fatalf("Could not create test nodes %v", err) - } - - gotNodes := network.GetNodes() - if len(gotNodes) != numNodes { - t.Fatalf("Expected %d nodes, got %d", numNodes, len(gotNodes)) - } - - for _, node1 := range nodes { - match := false - for _, node2 := range gotNodes { - if bytes.Equal(node1.ID().Bytes(), node2.ID().Bytes()) { - match = true - break - } - } - - if !match { - t.Fatalf("A created node was not returned by GetNodes(), ID: %s", node1.ID().String()) - } - } - - excludeNodeID := nodes[3].ID() - gotNodesExcl := network.GetNodes(excludeNodeID) - if len(gotNodesExcl) != numNodes-1 { - t.Fatalf("Expected one less node to be returned") - } - for _, node := range gotNodesExcl { - if bytes.Equal(excludeNodeID.Bytes(), node.ID().Bytes()) { - t.Fatalf("GetNodes returned the node we excluded, ID: %s", node.ID().String()) - } - } -} - -// TestGetNodesByID creates a set of nodes and attempts to retrieve a subset of them by ID -// If a node is not returned, or more nodes than expected are returned, the test fails. -func TestGetNodesByID(t *testing.T) { - adapter := adapters.NewSimAdapter(adapters.LifecycleConstructors{ - "test": newTestService, - }) - network := NewNetwork(adapter, &NetworkConfig{ - DefaultService: "test", - }) - defer network.Shutdown() - - numNodes := 5 - nodes, err := createTestNodes(numNodes, network) - if err != nil { - t.Fatalf("Could not create test nodes: %v", err) - } - - numSubsetNodes := 2 - subsetNodes := nodes[0:numSubsetNodes] - var subsetNodeIDs []enode.ID - for _, node := range subsetNodes { - subsetNodeIDs = append(subsetNodeIDs, node.ID()) - } - - gotNodesByID := network.GetNodesByID(subsetNodeIDs) - if len(gotNodesByID) != numSubsetNodes { - t.Fatalf("Expected %d nodes, got %d", numSubsetNodes, len(gotNodesByID)) - } - - for _, node1 := range subsetNodes { - match := false - for _, node2 := range gotNodesByID { - if bytes.Equal(node1.ID().Bytes(), node2.ID().Bytes()) { - match = true - break - } - } - - if !match { - t.Fatalf("A created node was not returned by GetNodesByID(), ID: %s", node1.ID().String()) - } - } -} - -// TestGetNodesByProperty creates a subset of nodes with a property assigned. -// GetNodesByProperty is then checked for correctness by comparing the nodes returned to those initially created. -// If a node with a property is not found, or more nodes than expected are returned, the test fails. -func TestGetNodesByProperty(t *testing.T) { - adapter := adapters.NewSimAdapter(adapters.LifecycleConstructors{ - "test": newTestService, - }) - network := NewNetwork(adapter, &NetworkConfig{ - DefaultService: "test", - }) - defer network.Shutdown() - - numNodes := 3 - _, err := createTestNodes(numNodes, network) - if err != nil { - t.Fatalf("Failed to create nodes: %v", err) - } - - numPropertyNodes := 3 - propertyTest := "test" - propertyNodes, err := createTestNodesWithProperty(propertyTest, numPropertyNodes, network) - if err != nil { - t.Fatalf("Failed to create nodes with property: %v", err) - } - - gotNodesByProperty := network.GetNodesByProperty(propertyTest) - if len(gotNodesByProperty) != numPropertyNodes { - t.Fatalf("Expected %d nodes with a property, got %d", numPropertyNodes, len(gotNodesByProperty)) - } - - for _, node1 := range propertyNodes { - match := false - for _, node2 := range gotNodesByProperty { - if bytes.Equal(node1.ID().Bytes(), node2.ID().Bytes()) { - match = true - break - } - } - - if !match { - t.Fatalf("A created node with property was not returned by GetNodesByProperty(), ID: %s", node1.ID().String()) - } - } -} - -// TestGetNodeIDsByProperty creates a subset of nodes with a property assigned. -// GetNodeIDsByProperty is then checked for correctness by comparing the node IDs returned to those initially created. -// If a node ID with a property is not found, or more nodes IDs than expected are returned, the test fails. -func TestGetNodeIDsByProperty(t *testing.T) { - adapter := adapters.NewSimAdapter(adapters.LifecycleConstructors{ - "test": newTestService, - }) - network := NewNetwork(adapter, &NetworkConfig{ - DefaultService: "test", - }) - defer network.Shutdown() - - numNodes := 3 - _, err := createTestNodes(numNodes, network) - if err != nil { - t.Fatalf("Failed to create nodes: %v", err) - } - - numPropertyNodes := 3 - propertyTest := "test" - propertyNodes, err := createTestNodesWithProperty(propertyTest, numPropertyNodes, network) - if err != nil { - t.Fatalf("Failed to created nodes with property: %v", err) - } - - gotNodeIDsByProperty := network.GetNodeIDsByProperty(propertyTest) - if len(gotNodeIDsByProperty) != numPropertyNodes { - t.Fatalf("Expected %d nodes with a property, got %d", numPropertyNodes, len(gotNodeIDsByProperty)) - } - - for _, node1 := range propertyNodes { - match := false - id1 := node1.ID() - for _, id2 := range gotNodeIDsByProperty { - if bytes.Equal(id1.Bytes(), id2.Bytes()) { - match = true - break - } - } - - if !match { - t.Fatalf("Not all nodes IDs were returned by GetNodeIDsByProperty(), ID: %s", id1.String()) - } - } -} - -func triggerChecks(ctx context.Context, ids []enode.ID, trigger chan enode.ID, interval time.Duration) { - tick := time.NewTicker(interval) - defer tick.Stop() - for { - select { - case <-tick.C: - for _, id := range ids { - select { - case trigger <- id: - case <-ctx.Done(): - return - } - } - case <-ctx.Done(): - return - } - } -} - -// \todo: refactor to implement snapshots -// and connect configuration methods once these are moved from -// swarm/network/simulations/connect.go -func BenchmarkMinimalService(b *testing.B) { - b.Run("ring/32", benchmarkMinimalServiceTmp) -} - -func benchmarkMinimalServiceTmp(b *testing.B) { - // stop timer to discard setup time pollution - args := strings.Split(b.Name(), "/") - nodeCount, err := strconv.ParseInt(args[2], 10, 16) - if err != nil { - b.Fatal(err) - } - - for i := 0; i < b.N; i++ { - // this is a minimal service, whose protocol will close a channel upon run of protocol - // making it possible to bench the time it takes for the service to start and protocol actually to be run - protoCMap := make(map[enode.ID]map[enode.ID]chan struct{}) - adapter := adapters.NewSimAdapter(adapters.LifecycleConstructors{ - "noopwoop": func(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { - protoCMap[ctx.Config.ID] = make(map[enode.ID]chan struct{}) - svc := NewNoopService(protoCMap[ctx.Config.ID]) - return svc, nil - }, - }) - - // create network - network := NewNetwork(adapter, &NetworkConfig{ - DefaultService: "noopwoop", - }) - defer network.Shutdown() - - // create and start nodes - ids := make([]enode.ID, nodeCount) - for i := 0; i < int(nodeCount); i++ { - conf := adapters.RandomNodeConfig() - node, err := network.NewNodeWithConfig(conf) - if err != nil { - b.Fatalf("error creating node: %s", err) - } - if err := network.Start(node.ID()); err != nil { - b.Fatalf("error starting node: %s", err) - } - ids[i] = node.ID() - } - - // ready, set, go - b.ResetTimer() - - // connect nodes in a ring - for i, id := range ids { - peerID := ids[(i+1)%len(ids)] - if err := network.Connect(id, peerID); err != nil { - b.Fatal(err) - } - } - - // wait for all protocols to signal to close down - ctx, cancel := context.WithTimeout(context.TODO(), time.Second) - defer cancel() - for nodid, peers := range protoCMap { - for peerid, peerC := range peers { - log.Debug("getting ", "node", nodid, "peer", peerid) - select { - case <-ctx.Done(): - b.Fatal(ctx.Err()) - case <-peerC: - } - } - } - } -} - -func TestNode_UnmarshalJSON(t *testing.T) { - t.Run("up_field", func(t *testing.T) { - runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONUpField()) - }) - t.Run("config_field", func(t *testing.T) { - runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONConfigField()) - }) -} - -func runNodeUnmarshalJSON(t *testing.T, tests []nodeUnmarshalTestCase) { - t.Helper() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var got *Node - if err := json.Unmarshal([]byte(tt.marshaled), &got); err != nil { - expectErrorMessageToContain(t, err, tt.wantErr) - got = nil - } - expectNodeEquality(t, got, tt.want) - }) - } -} - -type nodeUnmarshalTestCase struct { - name string - marshaled string - want *Node - wantErr string -} - -func expectErrorMessageToContain(t *testing.T, got error, want string) { - t.Helper() - if got == nil && want == "" { - return - } - - if got == nil && want != "" { - t.Errorf("error was expected, got: nil, want: %v", want) - return - } - - if !strings.Contains(got.Error(), want) { - t.Errorf( - "unexpected error message, got %v, want: %v", - want, - got, - ) - } -} - -func expectNodeEquality(t *testing.T, got, want *Node) { - t.Helper() - if !reflect.DeepEqual(got, want) { - t.Errorf("Node.UnmarshalJSON() = %v, want %v", got, want) - } -} - -func casesNodeUnmarshalJSONUpField() []nodeUnmarshalTestCase { - return []nodeUnmarshalTestCase{ - { - name: "empty json", - marshaled: "{}", - want: newNode(nil, nil, false), - }, - { - name: "a stopped node", - marshaled: "{\"up\": false}", - want: newNode(nil, nil, false), - }, - { - name: "a running node", - marshaled: "{\"up\": true}", - want: newNode(nil, nil, true), - }, - { - name: "invalid JSON value on valid key", - marshaled: "{\"up\": foo}", - wantErr: "invalid character", - }, - { - name: "invalid JSON key and value", - marshaled: "{foo: bar}", - wantErr: "invalid character", - }, - { - name: "bool value expected but got something else (string)", - marshaled: "{\"up\": \"true\"}", - wantErr: "cannot unmarshal string into Go struct", - }, - } -} - -func casesNodeUnmarshalJSONConfigField() []nodeUnmarshalTestCase { - // Don't do a big fuss around testing, as adapters.NodeConfig should - // handle it's own serialization. Just do a sanity check. - return []nodeUnmarshalTestCase{ - { - name: "Config field is omitted", - marshaled: "{}", - want: newNode(nil, nil, false), - }, - { - name: "Config field is nil", - marshaled: "{\"config\": null}", - want: newNode(nil, nil, false), - }, - { - name: "a non default Config field", - marshaled: "{\"config\":{\"name\":\"node_ecdd0\",\"port\":44665}}", - want: newNode(nil, &adapters.NodeConfig{Name: "node_ecdd0", Port: 44665}, false), - }, - } -} diff --git a/p2p/simulations/simulation.go b/p2p/simulations/simulation.go deleted file mode 100644 index ae62c42b9c..0000000000 --- a/p2p/simulations/simulation.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package simulations - -import ( - "context" - "time" - - "github.com/ethereum/go-ethereum/p2p/enode" -) - -// Simulation provides a framework for running actions in a simulated network -// and then waiting for expectations to be met -type Simulation struct { - network *Network -} - -// NewSimulation returns a new simulation which runs in the given network -func NewSimulation(network *Network) *Simulation { - return &Simulation{ - network: network, - } -} - -// Run performs a step of the simulation by performing the step's action and -// then waiting for the step's expectation to be met -func (s *Simulation) Run(ctx context.Context, step *Step) (result *StepResult) { - result = newStepResult() - - result.StartedAt = time.Now() - defer func() { result.FinishedAt = time.Now() }() - - // watch network events for the duration of the step - stop := s.watchNetwork(result) - defer stop() - - // perform the action - if err := step.Action(ctx); err != nil { - result.Error = err - return - } - - // wait for all node expectations to either pass, error or timeout - nodes := make(map[enode.ID]struct{}, len(step.Expect.Nodes)) - for _, id := range step.Expect.Nodes { - nodes[id] = struct{}{} - } - for len(result.Passes) < len(nodes) { - select { - case id := <-step.Trigger: - // skip if we aren't checking the node - if _, ok := nodes[id]; !ok { - continue - } - - // skip if the node has already passed - if _, ok := result.Passes[id]; ok { - continue - } - - // run the node expectation check - pass, err := step.Expect.Check(ctx, id) - if err != nil { - result.Error = err - return - } - if pass { - result.Passes[id] = time.Now() - } - case <-ctx.Done(): - result.Error = ctx.Err() - return - } - } - - return -} - -func (s *Simulation) watchNetwork(result *StepResult) func() { - stop := make(chan struct{}) - done := make(chan struct{}) - events := make(chan *Event) - sub := s.network.Events().Subscribe(events) - go func() { - defer close(done) - defer sub.Unsubscribe() - for { - select { - case event := <-events: - result.NetworkEvents = append(result.NetworkEvents, event) - case <-stop: - return - } - } - }() - return func() { - close(stop) - <-done - } -} - -type Step struct { - // Action is the action to perform for this step - Action func(context.Context) error - - // Trigger is a channel which receives node ids and triggers an - // expectation check for that node - Trigger chan enode.ID - - // Expect is the expectation to wait for when performing this step - Expect *Expectation -} - -type Expectation struct { - // Nodes is a list of nodes to check - Nodes []enode.ID - - // Check checks whether a given node meets the expectation - Check func(context.Context, enode.ID) (bool, error) -} - -func newStepResult() *StepResult { - return &StepResult{ - Passes: make(map[enode.ID]time.Time), - } -} - -type StepResult struct { - // Error is the error encountered whilst running the step - Error error - - // StartedAt is the time the step started - StartedAt time.Time - - // FinishedAt is the time the step finished - FinishedAt time.Time - - // Passes are the timestamps of the successful node expectations - Passes map[enode.ID]time.Time - - // NetworkEvents are the network events which occurred during the step - NetworkEvents []*Event -} diff --git a/p2p/simulations/test.go b/p2p/simulations/test.go deleted file mode 100644 index 0edb07b127..0000000000 --- a/p2p/simulations/test.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package simulations - -import ( - "testing" - - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/enr" - "github.com/ethereum/go-ethereum/rpc" -) - -// NoopService is the service that does not do anything -// but implements node.Service interface. -type NoopService struct { - c map[enode.ID]chan struct{} -} - -func NewNoopService(ackC map[enode.ID]chan struct{}) *NoopService { - return &NoopService{ - c: ackC, - } -} - -func (t *NoopService) Protocols() []p2p.Protocol { - return []p2p.Protocol{ - { - Name: "noop", - Version: 666, - Length: 0, - Run: func(peer *p2p.Peer, rw p2p.MsgReadWriter) error { - if t.c != nil { - t.c[peer.ID()] = make(chan struct{}) - close(t.c[peer.ID()]) - } - rw.ReadMsg() - return nil - }, - NodeInfo: func() interface{} { - return struct{}{} - }, - PeerInfo: func(id enode.ID) interface{} { - return struct{}{} - }, - Attributes: []enr.Entry{}, - }, - } -} - -func (t *NoopService) APIs() []rpc.API { - return []rpc.API{} -} - -func (t *NoopService) Start() error { - return nil -} - -func (t *NoopService) Stop() error { - return nil -} - -func VerifyRing(t *testing.T, net *Network, ids []enode.ID) { - t.Helper() - n := len(ids) - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - c := net.GetConn(ids[i], ids[j]) - if i == j-1 || (i == 0 && j == n-1) { - if c == nil { - t.Errorf("nodes %v and %v are not connected, but they should be", i, j) - } - } else { - if c != nil { - t.Errorf("nodes %v and %v are connected, but they should not be", i, j) - } - } - } - } -} - -func VerifyChain(t *testing.T, net *Network, ids []enode.ID) { - t.Helper() - n := len(ids) - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - c := net.GetConn(ids[i], ids[j]) - if i == j-1 { - if c == nil { - t.Errorf("nodes %v and %v are not connected, but they should be", i, j) - } - } else { - if c != nil { - t.Errorf("nodes %v and %v are connected, but they should not be", i, j) - } - } - } - } -} - -func VerifyFull(t *testing.T, net *Network, ids []enode.ID) { - t.Helper() - n := len(ids) - var connections int - for i, lid := range ids { - for _, rid := range ids[i+1:] { - if net.GetConn(lid, rid) != nil { - connections++ - } - } - } - - want := n * (n - 1) / 2 - if connections != want { - t.Errorf("wrong number of connections, got: %v, want: %v", connections, want) - } -} - -func VerifyStar(t *testing.T, net *Network, ids []enode.ID, centerIndex int) { - t.Helper() - n := len(ids) - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - c := net.GetConn(ids[i], ids[j]) - if i == centerIndex || j == centerIndex { - if c == nil { - t.Errorf("nodes %v and %v are not connected, but they should be", i, j) - } - } else { - if c != nil { - t.Errorf("nodes %v and %v are connected, but they should not be", i, j) - } - } - } - } -} diff --git a/p2p/transport_test.go b/p2p/transport_test.go index 24e06c5a06..01695cd3af 100644 --- a/p2p/transport_test.go +++ b/p2p/transport_test.go @@ -24,7 +24,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/p2p/simulations/pipes" + "github.com/ethereum/go-ethereum/p2p/pipes" ) func TestProtocolHandshake(t *testing.T) { From 5adf4adc8ec2c497eddd3b1ff20d2d35d65ec5fc Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 12 Aug 2024 16:43:54 +0800 Subject: [PATCH 24/39] eth/protocols/snap: cleanup dangling account trie nodes due to incomplete storage (#30258) This pull request fixes #30229. During snap sync, large storage will be split into several pieces and synchronized concurrently. Unfortunately, the tradeoff is that the respective merkle trie of each storage chunk will be incomplete due to the incomplete boundaries. The trie nodes on these boundaries will be discarded, and any dangling nodes on disk will also be removed if they fall on these paths, ensuring the state healer won't be blocked. However, the dangling account trie nodes on the path from the root to the associated account are left untouched. This means the dangling account trie nodes could potentially stop the state healing and break the assumption that the entire subtrie should exist if the subtrie root exists. We should consider the account trie node as the ancestor of the corresponding storage trie node. In the scenarios described in the above ticket, the state corruption could occur if there is a dangling account trie node while some storage trie nodes are removed due to synchronization redo. The fixing idea is pretty straightforward, the trie nodes on the path from root to account should all be explicitly removed if an incomplete storage trie occurs. Therefore, a `delete` operation has been added into `gentrie` to explicitly clear the account along with all nodes on this path. The special thing is that it's a cross-trie clearing. In theory, there may be a dangling node at any position on this account key and we have to clear all of them. --- eth/protocols/snap/gentrie.go | 44 ++++++++- eth/protocols/snap/gentrie_test.go | 142 +++++++++++++++++++++++++++++ eth/protocols/snap/sync.go | 11 ++- trie/stacktrie.go | 10 +- 4 files changed, 198 insertions(+), 9 deletions(-) diff --git a/eth/protocols/snap/gentrie.go b/eth/protocols/snap/gentrie.go index 6255fb221d..5126d26777 100644 --- a/eth/protocols/snap/gentrie.go +++ b/eth/protocols/snap/gentrie.go @@ -31,6 +31,9 @@ type genTrie interface { // update inserts the state item into generator trie. update(key, value []byte) error + // delete removes the state item from the generator trie. + delete(key []byte) error + // commit flushes the right boundary nodes if complete flag is true. This // function must be called before flushing the associated database batch. commit(complete bool) common.Hash @@ -113,7 +116,7 @@ func (t *pathTrie) onTrieNode(path []byte, hash common.Hash, blob []byte) { // removed because it's a sibling of the nodes we want to commit, not // the parent or ancestor. for i := 0; i < len(path); i++ { - t.delete(path[:i], false) + t.deleteNode(path[:i], false) } } return @@ -136,7 +139,7 @@ func (t *pathTrie) onTrieNode(path []byte, hash common.Hash, blob []byte) { // byte key. In either case, no gaps will be left in the path. if t.last != nil && bytes.HasPrefix(t.last, path) && len(t.last)-len(path) > 1 { for i := len(path) + 1; i < len(t.last); i++ { - t.delete(t.last[:i], true) + t.deleteNode(t.last[:i], true) } } t.write(path, blob) @@ -192,8 +195,8 @@ func (t *pathTrie) deleteStorageNode(path []byte, inner bool) { rawdb.DeleteStorageTrieNode(t.batch, t.owner, path) } -// delete commits the node deletion to provided database batch in path mode. -func (t *pathTrie) delete(path []byte, inner bool) { +// deleteNode commits the node deletion to provided database batch in path mode. +func (t *pathTrie) deleteNode(path []byte, inner bool) { if t.owner == (common.Hash{}) { t.deleteAccountNode(path, inner) } else { @@ -207,6 +210,34 @@ func (t *pathTrie) update(key, value []byte) error { return t.tr.Update(key, value) } +// delete implements genTrie interface, deleting the item from the stack trie. +func (t *pathTrie) delete(key []byte) error { + // Commit the trie since the right boundary is incomplete because + // of the deleted item. This will implicitly discard the last inserted + // item and clean some ancestor trie nodes of the last committed + // item in the database. + t.commit(false) + + // Reset the trie and all the internal trackers + t.first = nil + t.last = nil + t.tr.Reset() + + // Explicitly mark the left boundary as incomplete, as the left-side + // item of the next one has been deleted. Be aware that the next item + // to be inserted will be ignored from committing as well as it's on + // the left boundary. + t.skipLeftBoundary = true + + // Explicitly delete the potential leftover nodes on the specific + // path from the database. + tkey := t.tr.TrieKey(key) + for i := 0; i <= len(tkey); i++ { + t.deleteNode(tkey[:i], false) + } + return nil +} + // commit implements genTrie interface, flushing the right boundary if it's // considered as complete. Otherwise, the nodes on the right boundary are // discarded and cleaned up. @@ -255,7 +286,7 @@ func (t *pathTrie) commit(complete bool) common.Hash { // with no issues as they are actually complete. Also, from a database // perspective, first deleting and then rewriting is a valid data update. for i := 0; i < len(t.last); i++ { - t.delete(t.last[:i], false) + t.deleteNode(t.last[:i], false) } return common.Hash{} // the hash is meaningless for incomplete commit } @@ -278,6 +309,9 @@ func (t *hashTrie) update(key, value []byte) error { return t.tr.Update(key, value) } +// delete implements genTrie interface, ignoring the state item for deleting. +func (t *hashTrie) delete(key []byte) error { return nil } + // commit implements genTrie interface, committing the nodes on right boundary. func (t *hashTrie) commit(complete bool) common.Hash { if !complete { diff --git a/eth/protocols/snap/gentrie_test.go b/eth/protocols/snap/gentrie_test.go index 1fb2dbce75..2da4f3c866 100644 --- a/eth/protocols/snap/gentrie_test.go +++ b/eth/protocols/snap/gentrie_test.go @@ -551,3 +551,145 @@ func TestTinyPartialTree(t *testing.T) { } } } + +func TestTrieDelete(t *testing.T) { + var entries []*kv + for i := 0; i < 1024; i++ { + entries = append(entries, &kv{ + k: testrand.Bytes(32), + v: testrand.Bytes(32), + }) + } + slices.SortFunc(entries, (*kv).cmp) + + nodes := make(map[string]common.Hash) + tr := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { + nodes[string(path)] = hash + }) + for i := 0; i < len(entries); i++ { + tr.Update(entries[i].k, entries[i].v) + } + tr.Hash() + + check := func(index []int) { + var ( + db = rawdb.NewMemoryDatabase() + batch = db.NewBatch() + marks = map[int]struct{}{} + neighbors = map[int]struct{}{} + ) + for _, n := range index { + marks[n] = struct{}{} + } + for _, n := range index { + if n != 0 { + if _, ok := marks[n-1]; !ok { + neighbors[n-1] = struct{}{} + } + } + if n != len(entries)-1 { + if _, ok := neighbors[n+1]; !ok { + neighbors[n+1] = struct{}{} + } + } + } + // Write the junk nodes as the dangling + var injects []string + for _, n := range index { + nibbles := byteToHex(entries[n].k) + for i := 0; i <= len(nibbles); i++ { + injects = append(injects, string(nibbles[:i])) + } + } + for _, path := range injects { + rawdb.WriteAccountTrieNode(db, []byte(path), testrand.Bytes(32)) + } + tr := newPathTrie(common.Hash{}, false, db, batch) + for i := 0; i < len(entries); i++ { + if _, ok := marks[i]; ok { + tr.delete(entries[i].k) + } else { + tr.update(entries[i].k, entries[i].v) + } + } + tr.commit(true) + + r := newBatchReplay() + batch.Replay(r) + batch.Write() + + for _, path := range injects { + if rawdb.HasAccountTrieNode(db, []byte(path)) { + t.Fatalf("Unexpected leftover node %v", []byte(path)) + } + } + + // ensure all the written nodes match with the complete tree + set := make(map[string]common.Hash) + for path, hash := range r.modifies() { + if hash == (common.Hash{}) { + continue + } + n, ok := nodes[path] + if !ok { + t.Fatalf("Unexpected trie node: %v", []byte(path)) + } + if n != hash { + t.Fatalf("Unexpected trie node content: %v, want: %x, got: %x", []byte(path), n, hash) + } + set[path] = hash + } + + // ensure all the missing nodes either on the deleted path, or + // on the neighbor paths. + isMissing := func(path []byte) bool { + for n := range marks { + key := byteToHex(entries[n].k) + if bytes.HasPrefix(key, path) { + return true + } + } + for n := range neighbors { + key := byteToHex(entries[n].k) + if bytes.HasPrefix(key, path) { + return true + } + } + return false + } + for path := range nodes { + if _, ok := set[path]; ok { + continue + } + if !isMissing([]byte(path)) { + t.Fatalf("Missing node %v", []byte(path)) + } + } + } + var cases = []struct { + index []int + }{ + // delete the first + {[]int{0}}, + + // delete the last + {[]int{len(entries) - 1}}, + + // delete the first two + {[]int{0, 1}}, + + // delete the last two + {[]int{len(entries) - 2, len(entries) - 1}}, + + {[]int{ + 0, 2, 4, 6, + len(entries) - 1, + len(entries) - 3, + len(entries) - 5, + len(entries) - 7, + }}, + } + for _, c := range cases { + check(c.index) + } +} diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 88d7d34dcc..cdd03e6a0c 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -2424,14 +2424,21 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { slim := types.SlimAccountRLP(*res.accounts[i]) rawdb.WriteAccountSnapshot(batch, hash, slim) - // If the task is complete, drop it into the stack trie to generate - // account trie nodes for it if !task.needHeal[i] { + // If the storage task is complete, drop it into the stack trie + // to generate account trie nodes for it full, err := types.FullAccountRLP(slim) // TODO(karalabe): Slim parsing can be omitted if err != nil { panic(err) // Really shouldn't ever happen } task.genTrie.update(hash[:], full) + } else { + // If the storage task is incomplete, explicitly delete the corresponding + // account item from the account trie to ensure that all nodes along the + // path to the incomplete storage trie are cleaned up. + if err := task.genTrie.delete(hash[:]); err != nil { + panic(err) // Really shouldn't ever happen + } } } // Flush anything written just now and update the stats diff --git a/trie/stacktrie.go b/trie/stacktrie.go index 9c574db0bf..d194cbf0ae 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -64,8 +64,7 @@ func (t *StackTrie) Update(key, value []byte) error { if len(value) == 0 { return errors.New("trying to insert empty (deletion)") } - k := keybytesToHex(key) - k = k[:len(k)-1] // chop the termination flag + k := t.TrieKey(key) if bytes.Compare(t.last, k) >= 0 { return errors.New("non-ascending key order") } @@ -84,6 +83,13 @@ func (t *StackTrie) Reset() { t.last = nil } +// TrieKey returns the internal key representation for the given user key. +func (t *StackTrie) TrieKey(key []byte) []byte { + k := keybytesToHex(key) + k = k[:len(k)-1] // chop the termination flag + return k +} + // stNode represents a node within a StackTrie type stNode struct { typ uint8 // node type (as in branch, ext, leaf) From 880511dc39e5d4788a07313a1289c928c8b7c7b7 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 12 Aug 2024 14:15:30 +0200 Subject: [PATCH 25/39] params: release go-ethereum v1.14.8 stable --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 852f9fd1e1..050b2122f7 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 14 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 14 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 1cf3b5d38ac64921819b8e999d194047c84c4126 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 12 Aug 2024 14:19:35 +0200 Subject: [PATCH 26/39] params: begin v1.14.9 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 050b2122f7..22e59d6838 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 14 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 14 // Minor version component of the current release + VersionPatch = 9 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From ab03c5746c040be2336a0efa0e2dcc43871435f0 Mon Sep 17 00:00:00 2001 From: Shude Li Date: Tue, 13 Aug 2024 02:42:36 +0800 Subject: [PATCH 27/39] go.mod: remove github.com/julienschmidt/httprouter (#30290) --- go.mod | 1 - go.sum | 1 - 2 files changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 763210decd..ae6b12c0b8 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,6 @@ require ( github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 - github.com/julienschmidt/httprouter v1.3.0 github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 github.com/kilic/bls12-381 v0.1.0 github.com/kylelemons/godebug v1.1.0 diff --git a/go.sum b/go.sum index 562362dee6..4dadd40a75 100644 --- a/go.sum +++ b/go.sum @@ -339,7 +339,6 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 h1:msKODTL1m0wigztaqILOtla9HeW1ciscYG4xjLtvk5I= github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8= From bc95452e024b91c238ef3e43f3573eb179a762d2 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Mon, 12 Aug 2024 11:46:15 -0700 Subject: [PATCH 28/39] build: run 'go mod tidy' check as part of lint (#30291) --- build/ci.go | 68 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/build/ci.go b/build/ci.go index db57633704..009d69e6e4 100644 --- a/build/ci.go +++ b/build/ci.go @@ -349,10 +349,10 @@ func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string { return filepath.Join(cachedir, base) } -// hashSourceFiles iterates all files under the top-level project directory +// hashAllSourceFiles iterates all files under the top-level project directory // computing the hash of each file (excluding files within the tests // subrepo) -func hashSourceFiles() (map[string]common.Hash, error) { +func hashAllSourceFiles() (map[string]common.Hash, error) { res := make(map[string]common.Hash) err := filepath.WalkDir(".", func(path string, d os.DirEntry, err error) error { if strings.HasPrefix(path, filepath.FromSlash("tests/testdata")) { @@ -379,6 +379,56 @@ func hashSourceFiles() (map[string]common.Hash, error) { return res, nil } +// hashSourceFiles iterates the provided set of filepaths (relative to the top-level geth project directory) +// computing the hash of each file. +func hashSourceFiles(files []string) (map[string]common.Hash, error) { + res := make(map[string]common.Hash) + for _, filePath := range files { + f, err := os.OpenFile(filePath, os.O_RDONLY, 0666) + if err != nil { + return nil, err + } + hasher := sha256.New() + if _, err := io.Copy(hasher, f); err != nil { + return nil, err + } + res[filePath] = common.Hash(hasher.Sum(nil)) + } + return res, nil +} + +// compareHashedFilesets compares two maps (key is relative file path to top-level geth directory, value is its hash) +// and returns the list of file paths whose hashes differed. +func compareHashedFilesets(preHashes map[string]common.Hash, postHashes map[string]common.Hash) []string { + updates := []string{} + for path, postHash := range postHashes { + preHash, ok := preHashes[path] + if !ok || preHash != postHash { + updates = append(updates, path) + } + } + return updates +} + +func doGoModTidy() { + targetFiles := []string{"go.mod", "go.sum"} + preHashes, err := hashSourceFiles(targetFiles) + if err != nil { + log.Fatal("failed to hash go.mod/go.sum", "err", err) + } + tc := new(build.GoToolchain) + c := tc.Go("mod", "tidy") + build.MustRun(c) + postHashes, err := hashSourceFiles(targetFiles) + updates := compareHashedFilesets(preHashes, postHashes) + for _, updatedFile := range updates { + fmt.Fprintf(os.Stderr, "changed file %s\n", updatedFile) + } + if len(updates) != 0 { + log.Fatal("go.sum and/or go.mod were updated by running 'go mod tidy'") + } +} + // doGenerate ensures that re-generating generated files does not cause // any mutations in the source file tree: i.e. all generated files were // updated and committed. Any stale generated files are updated. @@ -395,7 +445,7 @@ func doGenerate() { var preHashes map[string]common.Hash if *verify { var err error - preHashes, err = hashSourceFiles() + preHashes, err = hashAllSourceFiles() if err != nil { log.Fatal("failed to compute map of source hashes", "err", err) } @@ -410,17 +460,11 @@ func doGenerate() { return } // Check if files were changed. - postHashes, err := hashSourceFiles() + postHashes, err := hashAllSourceFiles() if err != nil { log.Fatal("error computing source tree file hashes", "err", err) } - updates := []string{} - for path, postHash := range postHashes { - preHash, ok := preHashes[path] - if !ok || preHash != postHash { - updates = append(updates, path) - } - } + updates := compareHashedFilesets(preHashes, postHashes) for _, updatedFile := range updates { fmt.Fprintf(os.Stderr, "changed file %s\n", updatedFile) } @@ -443,6 +487,8 @@ func doLint(cmdline []string) { linter := downloadLinter(*cachedir) lflags := []string{"run", "--config", ".golangci.yml"} build.MustRunCommandWithOutput(linter, append(lflags, packages...)...) + + doGoModTidy() fmt.Println("You have achieved perfection.") } From bd57f35f8d1e920ef2b234ea6cd1aa18715b358f Mon Sep 17 00:00:00 2001 From: Zoo Date: Tue, 13 Aug 2024 19:49:31 +0800 Subject: [PATCH 29/39] core/txpool/blobpool: fix error message (#30247) the validation process only checks for 'less than', which is inconsistent with the error output --- core/txpool/blobpool/blobpool.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index cb266fd10a..d66a08aa17 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1155,11 +1155,11 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error { ) switch { case tx.GasFeeCapIntCmp(minGasFeeCap.ToBig()) < 0: - return fmt.Errorf("%w: new tx gas fee cap %v <= %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.GasFeeCap(), prev.execFeeCap, p.config.PriceBump) + return fmt.Errorf("%w: new tx gas fee cap %v < %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.GasFeeCap(), prev.execFeeCap, p.config.PriceBump) case tx.GasTipCapIntCmp(minGasTipCap.ToBig()) < 0: - return fmt.Errorf("%w: new tx gas tip cap %v <= %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.GasTipCap(), prev.execTipCap, p.config.PriceBump) + return fmt.Errorf("%w: new tx gas tip cap %v < %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.GasTipCap(), prev.execTipCap, p.config.PriceBump) case tx.BlobGasFeeCapIntCmp(minBlobGasFeeCap.ToBig()) < 0: - return fmt.Errorf("%w: new tx blob gas fee cap %v <= %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.BlobGasFeeCap(), prev.blobFeeCap, p.config.PriceBump) + return fmt.Errorf("%w: new tx blob gas fee cap %v < %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.BlobGasFeeCap(), prev.blobFeeCap, p.config.PriceBump) } } return nil From 2b9d19870646140c4dc90645cf7c828cc76860aa Mon Sep 17 00:00:00 2001 From: Adrian Sutton Date: Wed, 14 Aug 2024 19:45:51 +1000 Subject: [PATCH 30/39] go.mod: upgrade to pebble v1.1.2 (#30297) Includes a fix for MIPS32 support. Pebble release: https://github.com/cockroachdb/pebble/releases/tag/v1.1.2 Key fix for mips32: https://github.com/cockroachdb/pebble/commit/9f3904a705d60b9832febb6c6494183d92c8f556 (also the only change from v1.1.1. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ae6b12c0b8..909add8be6 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.3.4 github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.79.0 - github.com/cockroachdb/pebble v1.1.1 + github.com/cockroachdb/pebble v1.1.2 github.com/consensys/gnark-crypto v0.12.1 github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c github.com/crate-crypto/go-kzg-4844 v1.0.0 diff --git a/go.sum b/go.sum index 4dadd40a75..089c05c2ec 100644 --- a/go.sum +++ b/go.sum @@ -122,8 +122,8 @@ github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/e github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.1 h1:XnKU22oiCLy2Xn8vp1re67cXg4SAasg/WDt1NtcRFaw= -github.com/cockroachdb/pebble v1.1.1/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= From c686485a062d0addddf40c3808de7fe90793ed18 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 15 Aug 2024 09:16:23 +0200 Subject: [PATCH 31/39] core: only compute state root once (#30299) This PR refactors the genesis initialization a bit, s.th. we only compute the blockhash once instead of twice as before (during hashAlloc and flushAlloc) This will significantly reduce the amount of memory allocated during genesis init --------- Co-authored-by: Gary Rong --- core/genesis.go | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/core/genesis.go b/core/genesis.go index 4ca24807fc..9e213e5163 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -145,13 +145,12 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) { return statedb.Commit(0, false) } -// flushAlloc is very similar with hash, but the main difference is all the generated -// states will be persisted into the given database. Also, the genesis state -// specification will be flushed as well. -func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Database, blockhash common.Hash) error { +// flushAlloc is very similar with hash, but the main difference is all the +// generated states will be persisted into the given database. +func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Database) (common.Hash, error) { statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil) if err != nil { - return err + return common.Hash{}, err } for addr, account := range *ga { if account.Balance != nil { @@ -167,21 +166,15 @@ func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Databa } root, err := statedb.Commit(0, false) if err != nil { - return err + return common.Hash{}, err } // Commit newly generated states into disk if it's not empty. if root != types.EmptyRootHash { if err := triedb.Commit(root, true); err != nil { - return err + return common.Hash{}, err } } - // Marshal the genesis state specification and persist. - blob, err := json.Marshal(ga) - if err != nil { - return err - } - rawdb.WriteGenesisStateSpec(db, blockhash, blob) - return nil + return root, nil } func getGenesisState(db ethdb.Database, blockhash common.Hash) (alloc types.GenesisAlloc, err error) { @@ -426,6 +419,11 @@ func (g *Genesis) ToBlock() *types.Block { if err != nil { panic(err) } + return g.toBlockWithRoot(root) +} + +// toBlockWithRoot constructs the genesis block with the given genesis state root. +func (g *Genesis) toBlockWithRoot(root common.Hash) *types.Block { head := &types.Header{ Number: new(big.Int).SetUint64(g.Number), Nonce: types.EncodeNonce(g.Nonce), @@ -482,8 +480,7 @@ func (g *Genesis) ToBlock() *types.Block { // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Block, error) { - block := g.ToBlock() - if block.Number().Sign() != 0 { + if g.Number != 0 { return nil, errors.New("can't commit genesis block with number > 0") } config := g.Config @@ -493,15 +490,22 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo if err := config.CheckConfigForkOrder(); err != nil { return nil, err } - if config.Clique != nil && len(block.Extra()) < 32+crypto.SignatureLength { + if config.Clique != nil && len(g.ExtraData) < 32+crypto.SignatureLength { return nil, errors.New("can't start clique chain without signers") } - // All the checks has passed, flushAlloc the states derived from the genesis - // specification as well as the specification itself into the provided - // database. - if err := flushAlloc(&g.Alloc, db, triedb, block.Hash()); err != nil { + // flush the data to disk and compute the state root + root, err := flushAlloc(&g.Alloc, db, triedb) + if err != nil { return nil, err } + block := g.toBlockWithRoot(root) + + // Marshal the genesis state specification and persist. + blob, err := json.Marshal(g.Alloc) + if err != nil { + return nil, err + } + rawdb.WriteGenesisStateSpec(db, block.Hash(), blob) rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty()) rawdb.WriteBlock(db, block) rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), nil) From 2f2e5b088a08ff164fb20f704d04d67ddd6edbf5 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 15 Aug 2024 10:11:03 +0200 Subject: [PATCH 32/39] .golangci.yml: remove lint warning for TxLookupLimit --- .golangci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 75452472d0..d3b8535cfb 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -60,6 +60,8 @@ issues: - path: crypto/bn256/ linters: - revive + - path: cmd/utils/flags.go + text: "SA1019: cfg.TxLookupLimit is deprecated: use 'TransactionHistory' instead." - path: internal/build/pgp.go text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.' - path: core/vm/contracts.go From c4b01d80b94820abeea52504c661e2a2ca8739d7 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 15 Aug 2024 17:35:16 +0800 Subject: [PATCH 33/39] eth/fetcher: always expect transaction metadata in announcement (#30288) This pull request drops the legacy transaction retrieval support from before eth68, adding the restrictions that transaction metadata must be provided along with the transaction announment. --- eth/fetcher/tx_fetcher.go | 23 +- eth/fetcher/tx_fetcher_test.go | 981 ++++++++++++++++++--------------- 2 files changed, 547 insertions(+), 457 deletions(-) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 18c5ff007a..a113155009 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -114,7 +114,7 @@ var errTerminated = errors.New("terminated") type txAnnounce struct { origin string // Identifier of the peer originating the notification hashes []common.Hash // Batch of transaction hashes being announced - metas []*txMetadata // Batch of metadatas associated with the hashes (nil before eth/68) + metas []*txMetadata // Batch of metadata associated with the hashes } // txMetadata is a set of extra data transmitted along the announcement for better @@ -137,7 +137,7 @@ type txRequest struct { type txDelivery struct { origin string // Identifier of the peer originating the notification hashes []common.Hash // Batch of transaction hashes having been delivered - metas []txMetadata // Batch of metadatas associated with the delivered hashes + metas []txMetadata // Batch of metadata associated with the delivered hashes direct bool // Whether this is a direct reply or a broadcast } @@ -260,11 +260,11 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c underpriced++ default: unknownHashes = append(unknownHashes, hash) - if types == nil { - unknownMetas = append(unknownMetas, nil) - } else { - unknownMetas = append(unknownMetas, &txMetadata{kind: types[i], size: sizes[i]}) - } + + // Transaction metadata has been available since eth68, and all + // legacy eth protocols (prior to eth68) have been deprecated. + // Therefore, metadata is always expected in the announcement. + unknownMetas = append(unknownMetas, &txMetadata{kind: types[i], size: sizes[i]}) } } txAnnounceKnownMeter.Mark(duplicate) @@ -892,13 +892,8 @@ func (f *TxFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, if len(hashes) >= maxTxRetrievals { return false // break in the for-each } - if meta != nil { // Only set eth/68 and upwards - bytes += uint64(meta.size) - if bytes >= maxTxRetrievalSize { - return false - } - } - return true // scheduled, try to add more + bytes += uint64(meta.size) + return bytes < maxTxRetrievalSize }) // If any hashes were allocated, request them from the peer if len(hashes) > 0 { diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index 3d3ef81ede..0b47646669 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -45,13 +45,10 @@ var ( type announce struct { hash common.Hash - kind *byte - size *uint32 + kind byte + size uint32 } -func typeptr(t byte) *byte { return &t } -func sizeptr(n uint32) *uint32 { return &n } - type doTxNotify struct { peer string hashes []common.Hash @@ -70,16 +67,10 @@ type doWait struct { type doDrop string type doFunc func() -type isWaitingWithMeta map[string][]announce -type isWaiting map[string][]common.Hash +type isWaiting map[string][]announce -type isScheduledWithMeta struct { - tracking map[string][]announce - fetching map[string][]common.Hash - dangling map[string][]common.Hash -} type isScheduled struct { - tracking map[string][]common.Hash + tracking map[string][]announce fetching map[string][]common.Hash dangling map[string][]common.Hash } @@ -92,104 +83,13 @@ type txFetcherTest struct { steps []interface{} } -// Tests that transaction announcements are added to a waitlist, and none -// of them are scheduled for retrieval until the wait expires. +// Tests that transaction announcements with associated metadata are added to a +// waitlist, and none of them are scheduled for retrieval until the wait expires. +// +// This test is an extended version of TestTransactionFetcherWaiting. It's mostly +// to cover the metadata checks without bloating up the basic behavioral tests +// with all the useless extra fields. func TestTransactionFetcherWaiting(t *testing.T) { - testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, - steps: []interface{}{ - // Initial announcement to get something into the waitlist - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, - }), - // Announce from a new peer to check that no overwrite happens - doTxNotify{peer: "B", hashes: []common.Hash{{0x03}, {0x04}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, - "B": {{0x03}, {0x04}}, - }), - // Announce clashing hashes but unique new peer - doTxNotify{peer: "C", hashes: []common.Hash{{0x01}, {0x04}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, - "B": {{0x03}, {0x04}}, - "C": {{0x01}, {0x04}}, - }), - // Announce existing and clashing hashes from existing peer - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x03}, {0x05}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x05}}, - "B": {{0x03}, {0x04}}, - "C": {{0x01}, {0x04}}, - }), - isScheduled{tracking: nil, fetching: nil}, - - // Wait for the arrival timeout which should move all expired items - // from the wait list to the scheduler - doWait{time: txArriveTimeout, step: true}, - isWaiting(nil), - isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x05}}, - "B": {{0x03}, {0x04}}, - "C": {{0x01}, {0x04}}, - }, - fetching: map[string][]common.Hash{ // Depends on deterministic test randomizer - "A": {{0x02}, {0x03}, {0x05}}, - "C": {{0x01}, {0x04}}, - }, - }, - // Queue up a non-fetchable transaction and then trigger it with a new - // peer (weird case to test 1 line in the fetcher) - doTxNotify{peer: "C", hashes: []common.Hash{{0x06}, {0x07}}}, - isWaiting(map[string][]common.Hash{ - "C": {{0x06}, {0x07}}, - }), - doWait{time: txArriveTimeout, step: true}, - isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x05}}, - "B": {{0x03}, {0x04}}, - "C": {{0x01}, {0x04}, {0x06}, {0x07}}, - }, - fetching: map[string][]common.Hash{ - "A": {{0x02}, {0x03}, {0x05}}, - "C": {{0x01}, {0x04}}, - }, - }, - doTxNotify{peer: "D", hashes: []common.Hash{{0x06}, {0x07}}}, - isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x05}}, - "B": {{0x03}, {0x04}}, - "C": {{0x01}, {0x04}, {0x06}, {0x07}}, - "D": {{0x06}, {0x07}}, - }, - fetching: map[string][]common.Hash{ - "A": {{0x02}, {0x03}, {0x05}}, - "C": {{0x01}, {0x04}}, - "D": {{0x06}, {0x07}}, - }, - }, - }, - }) -} - -// Tests that transaction announcements with associated metadata are added to a -// waitlist, and none of them are scheduled for retrieval until the wait expires. -// -// This test is an extended version of TestTransactionFetcherWaiting. It's mostly -// to cover the metadata checks without bloating up the basic behavioral tests -// with all the useless extra fields. -func TestTransactionFetcherWaitingWithMeta(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { return NewTxFetcher( @@ -202,80 +102,80 @@ func TestTransactionFetcherWaitingWithMeta(t *testing.T) { steps: []interface{}{ // Initial announcement to get something into the waitlist doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, }, }), // Announce from a new peer to check that no overwrite happens doTxNotify{peer: "B", hashes: []common.Hash{{0x03}, {0x04}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{333, 444}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, }), // Announce clashing hashes but unique new peer doTxNotify{peer: "C", hashes: []common.Hash{{0x01}, {0x04}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 444}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, }), // Announce existing and clashing hashes from existing peer. Clashes // should not overwrite previous announcements. doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x03}, {0x05}}, types: []byte{types.LegacyTxType, types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{999, 333, 555}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x05}, types.LegacyTxType, 555}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, }), // Announce clashing hashes with conflicting metadata. Somebody will // be in the wrong, but we don't know yet who. doTxNotify{peer: "D", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.BlobTxType}, sizes: []uint32{999, 222}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x05}, types.LegacyTxType, 555}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "D": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, - {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 999}, + {common.Hash{0x02}, types.BlobTxType, 222}, }, }), isScheduled{tracking: nil, fetching: nil}, @@ -284,25 +184,25 @@ func TestTransactionFetcherWaitingWithMeta(t *testing.T) { // from the wait list to the scheduler doWait{time: txArriveTimeout, step: true}, isWaiting(nil), - isScheduledWithMeta{ + isScheduled{ tracking: map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x05}, types.LegacyTxType, 555}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "D": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, - {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 999}, + {common.Hash{0x02}, types.BlobTxType, 222}, }, }, fetching: map[string][]common.Hash{ // Depends on deterministic test randomizer @@ -314,34 +214,34 @@ func TestTransactionFetcherWaitingWithMeta(t *testing.T) { // Queue up a non-fetchable transaction and then trigger it with a new // peer (weird case to test 1 line in the fetcher) doTxNotify{peer: "C", hashes: []common.Hash{{0x06}, {0x07}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{666, 777}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "C": { - {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, - {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + {common.Hash{0x06}, types.LegacyTxType, 666}, + {common.Hash{0x07}, types.LegacyTxType, 777}, }, }), doWait{time: txArriveTimeout, step: true}, - isScheduledWithMeta{ + isScheduled{ tracking: map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x05}, types.LegacyTxType, 555}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, - {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, - {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + {common.Hash{0x06}, types.LegacyTxType, 666}, + {common.Hash{0x07}, types.LegacyTxType, 777}, }, "D": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, - {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 999}, + {common.Hash{0x02}, types.BlobTxType, 222}, }, }, fetching: map[string][]common.Hash{ @@ -351,31 +251,31 @@ func TestTransactionFetcherWaitingWithMeta(t *testing.T) { }, }, doTxNotify{peer: "E", hashes: []common.Hash{{0x06}, {0x07}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{666, 777}}, - isScheduledWithMeta{ + isScheduled{ tracking: map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x05}, types.LegacyTxType, 555}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, - {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, - {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + {common.Hash{0x06}, types.LegacyTxType, 666}, + {common.Hash{0x07}, types.LegacyTxType, 777}, }, "D": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, - {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 999}, + {common.Hash{0x02}, types.BlobTxType, 222}, }, "E": { - {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, - {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + {common.Hash{0x06}, types.LegacyTxType, 666}, + {common.Hash{0x07}, types.LegacyTxType, 777}, }, }, fetching: map[string][]common.Hash{ @@ -403,17 +303,28 @@ func TestTransactionFetcherSkipWaiting(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + doTxNotify{ + peer: "A", + hashes: []common.Hash{{0x01}, {0x02}}, + types: []byte{types.LegacyTxType, types.LegacyTxType}, + sizes: []uint32{111, 222}, + }, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -421,13 +332,18 @@ func TestTransactionFetcherSkipWaiting(t *testing.T) { }, // Announce overlaps from the same peer, ensure the new ones end up // in stage one, and clashing ones don't get double tracked - doTxNotify{peer: "A", hashes: []common.Hash{{0x02}, {0x03}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x03}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}, {0x03}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{222, 333}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x03}, types.LegacyTxType, 333}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -435,15 +351,25 @@ func TestTransactionFetcherSkipWaiting(t *testing.T) { }, // Announce overlaps from a new peer, ensure new transactions end up // in stage one and clashing ones get tracked for the new peer - doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x04}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x03}}, - "B": {{0x03}, {0x04}}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x04}}, types: []byte{types.LegacyTxType, types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{222, 333, 444}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x03}, types.LegacyTxType, 333}, + }, + "B": { + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, - "B": {{0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -467,17 +393,23 @@ func TestTransactionFetcherSingletonRequesting(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -485,13 +417,19 @@ func TestTransactionFetcherSingletonRequesting(t *testing.T) { }, // Announce a new set of transactions from the same peer and ensure // they do not start fetching since the peer is already busy - doTxNotify{peer: "A", hashes: []common.Hash{{0x03}, {0x04}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x03}, {0x04}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x03}, {0x04}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{333, 444}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -500,8 +438,13 @@ func TestTransactionFetcherSingletonRequesting(t *testing.T) { doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x04}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -509,14 +452,25 @@ func TestTransactionFetcherSingletonRequesting(t *testing.T) { }, // Announce a duplicate set of transactions from a new peer and ensure // uniquely new ones start downloading, even if clashing. - doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x05}, {0x06}}}, - isWaiting(map[string][]common.Hash{ - "B": {{0x05}, {0x06}}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x05}, {0x06}}, types: []byte{types.LegacyTxType, types.LegacyTxType, types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{222, 333, 555, 666}}, + isWaiting(map[string][]announce{ + "B": { + {common.Hash{0x05}, types.LegacyTxType, 555}, + {common.Hash{0x06}, types.LegacyTxType, 666}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x04}}, - "B": {{0x02}, {0x03}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + }, + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -547,17 +501,23 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -565,12 +525,17 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) { }, // While the original peer is stuck in the request, push in an second // data source. - doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}, types: []byte{types.LegacyTxType}, sizes: []uint32{222}}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, - "B": {{0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -584,8 +549,10 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) { doWait{time: 0, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "B": {{0x02}}, + tracking: map[string][]announce{ + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "B": {{0x02}}, @@ -617,17 +584,21 @@ func TestTransactionFetcherCleanup(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -657,17 +628,21 @@ func TestTransactionFetcherCleanupEmpty(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -696,17 +671,29 @@ func TestTransactionFetcherMissingRescheduling(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + doTxNotify{peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type(), testTxs[2].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size()), uint32(testTxs[2].Size())}, + }, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + {testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + {testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, @@ -716,8 +703,10 @@ func TestTransactionFetcherMissingRescheduling(t *testing.T) { // should be dropped and the one after re-requested. doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true}, // This depends on the deterministic random isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[2]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[2]}, @@ -743,17 +732,27 @@ func TestTransactionFetcherMissingCleanup(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1]}, + doTxNotify{peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size())}, + }, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0], testTxsHashes[1]}, @@ -782,18 +781,23 @@ func TestTransactionFetcherBroadcasts(t *testing.T) { }, steps: []interface{}{ // Set up three transactions to be in different stats, waiting, queued and fetching - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}, types: []byte{testTxs[1].Type()}, sizes: []uint32{uint32(testTxs[1].Size())}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}, types: []byte{testTxs[2].Type()}, sizes: []uint32{uint32(testTxs[2].Size())}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[2]}, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -830,29 +834,40 @@ func TestTransactionFetcherWaitTimerResets(t *testing.T) { ) }, steps: []interface{}{ - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + }, }), isScheduled{nil, nil, nil}, doWait{time: txArriveTimeout / 2, step: false}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + }, }), isScheduled{nil, nil, nil}, - doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}, types: []byte{types.LegacyTxType}, sizes: []uint32{222}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }), isScheduled{nil, nil, nil}, doWait{time: txArriveTimeout / 2, step: true}, - isWaiting(map[string][]common.Hash{ - "A": {{0x02}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}}, @@ -862,8 +877,11 @@ func TestTransactionFetcherWaitTimerResets(t *testing.T) { doWait{time: txArriveTimeout / 2, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}}, @@ -889,17 +907,22 @@ func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + doTxNotify{ + peer: "A", + hashes: []common.Hash{testTxsHashes[0]}, + types: []byte{testTxs[0].Type()}, + sizes: []uint32{uint32(testTxs[0].Size())}, + }, + isWaiting(map[string][]announce{ + "A": {{testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}}, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + tracking: map[string][]announce{ + "A": {{testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}}, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -916,11 +939,16 @@ func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { }, }, // Ensure that followup announcements don't get scheduled - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doTxNotify{ + peer: "A", + hashes: []common.Hash{testTxsHashes[1]}, + types: []byte{testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[1].Size())}, + }, doWait{time: txArriveTimeout, step: true}, isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[1]}, + tracking: map[string][]announce{ + "A": {{testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}}, }, fetching: nil, dangling: map[string][]common.Hash{ @@ -931,8 +959,8 @@ func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[1]}, + tracking: map[string][]announce{ + "A": {{testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}}, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[1]}, @@ -954,16 +982,20 @@ func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { ) }, steps: []interface{}{ - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}, types: []byte{types.LegacyTxType}, sizes: []uint32{222}}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}}, - "B": {{0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + }, + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}}, @@ -972,8 +1004,10 @@ func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { }, doWait{time: txFetchTimeout - txArriveTimeout, step: true}, isScheduled{ - tracking: map[string][]common.Hash{ - "B": {{0x02}}, + tracking: map[string][]announce{ + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "B": {{0x02}}, @@ -999,9 +1033,22 @@ func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { // number of them will be requested at a time. func TestTransactionFetcherRateLimiting(t *testing.T) { // Create a slew of transactions and announce them - var hashes []common.Hash + var ( + hashes []common.Hash + ts []byte + sizes []uint32 + announces []announce + ) for i := 0; i < maxTxAnnounces; i++ { - hashes = append(hashes, common.Hash{byte(i / 256), byte(i % 256)}) + hash := common.Hash{byte(i / 256), byte(i % 256)} + hashes = append(hashes, hash) + ts = append(ts, types.LegacyTxType) + sizes = append(sizes, 111) + announces = append(announces, announce{ + hash: hash, + kind: types.LegacyTxType, + size: 111, + }) } testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { @@ -1015,12 +1062,12 @@ func TestTransactionFetcherRateLimiting(t *testing.T) { steps: []interface{}{ // Announce all the transactions, wait a bit and ensure only a small // percentage gets requested - doTxNotify{peer: "A", hashes: hashes}, + doTxNotify{peer: "A", hashes: hashes, types: ts, sizes: sizes}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": hashes, + tracking: map[string][]announce{ + "A": announces, }, fetching: map[string][]common.Hash{ "A": hashes[1643 : 1643+maxTxRetrievals], @@ -1065,21 +1112,21 @@ func TestTransactionFetcherBandwidthLimiting(t *testing.T) { }, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), - isScheduledWithMeta{ + isScheduled{ tracking: map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, + {common.Hash{0x01}, types.LegacyTxType, 48 * 1024}, + {common.Hash{0x02}, types.LegacyTxType, 48 * 1024}, + {common.Hash{0x03}, types.LegacyTxType, 48 * 1024}, + {common.Hash{0x04}, types.LegacyTxType, 48 * 1024}, }, "B": { - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(maxTxRetrievalSize)}, - {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(maxTxRetrievalSize)}, + {common.Hash{0x05}, types.LegacyTxType, maxTxRetrievalSize}, + {common.Hash{0x06}, types.LegacyTxType, maxTxRetrievalSize}, }, "C": { - {common.Hash{0x07}, typeptr(types.BlobTxType), sizeptr(params.MaxBlobGasPerBlock)}, - {common.Hash{0x08}, typeptr(types.BlobTxType), sizeptr(params.MaxBlobGasPerBlock)}, + {common.Hash{0x07}, types.BlobTxType, params.MaxBlobGasPerBlock}, + {common.Hash{0x08}, types.BlobTxType, params.MaxBlobGasPerBlock}, }, }, fetching: map[string][]common.Hash{ @@ -1096,13 +1143,41 @@ func TestTransactionFetcherBandwidthLimiting(t *testing.T) { // request at the same time is hard capped. func TestTransactionFetcherDoSProtection(t *testing.T) { // Create a slew of transactions and to announce them - var hashesA []common.Hash + var ( + hashesA []common.Hash + typesA []byte + sizesA []uint32 + announceA []announce + ) for i := 0; i < maxTxAnnounces+1; i++ { - hashesA = append(hashesA, common.Hash{0x01, byte(i / 256), byte(i % 256)}) + hash := common.Hash{0x01, byte(i / 256), byte(i % 256)} + hashesA = append(hashesA, hash) + typesA = append(typesA, types.LegacyTxType) + sizesA = append(sizesA, 111) + + announceA = append(announceA, announce{ + hash: hash, + kind: types.LegacyTxType, + size: 111, + }) } - var hashesB []common.Hash + var ( + hashesB []common.Hash + typesB []byte + sizesB []uint32 + announceB []announce + ) for i := 0; i < maxTxAnnounces+1; i++ { - hashesB = append(hashesB, common.Hash{0x02, byte(i / 256), byte(i % 256)}) + hash := common.Hash{0x02, byte(i / 256), byte(i % 256)} + hashesB = append(hashesB, hash) + typesB = append(typesB, types.LegacyTxType) + sizesB = append(sizesB, 111) + + announceB = append(announceB, announce{ + hash: hash, + kind: types.LegacyTxType, + size: 111, + }) } testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { @@ -1115,23 +1190,23 @@ func TestTransactionFetcherDoSProtection(t *testing.T) { }, steps: []interface{}{ // Announce half of the transaction and wait for them to be scheduled - doTxNotify{peer: "A", hashes: hashesA[:maxTxAnnounces/2]}, - doTxNotify{peer: "B", hashes: hashesB[:maxTxAnnounces/2-1]}, + doTxNotify{peer: "A", hashes: hashesA[:maxTxAnnounces/2], types: typesA[:maxTxAnnounces/2], sizes: sizesA[:maxTxAnnounces/2]}, + doTxNotify{peer: "B", hashes: hashesB[:maxTxAnnounces/2-1], types: typesB[:maxTxAnnounces/2-1], sizes: sizesB[:maxTxAnnounces/2-1]}, doWait{time: txArriveTimeout, step: true}, // Announce the second half and keep them in the wait list - doTxNotify{peer: "A", hashes: hashesA[maxTxAnnounces/2 : maxTxAnnounces]}, - doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1]}, + doTxNotify{peer: "A", hashes: hashesA[maxTxAnnounces/2 : maxTxAnnounces], types: typesA[maxTxAnnounces/2 : maxTxAnnounces], sizes: sizesA[maxTxAnnounces/2 : maxTxAnnounces]}, + doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1], types: typesB[maxTxAnnounces/2-1 : maxTxAnnounces-1], sizes: sizesB[maxTxAnnounces/2-1 : maxTxAnnounces-1]}, // Ensure the hashes are split half and half - isWaiting(map[string][]common.Hash{ - "A": hashesA[maxTxAnnounces/2 : maxTxAnnounces], - "B": hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1], + isWaiting(map[string][]announce{ + "A": announceA[maxTxAnnounces/2 : maxTxAnnounces], + "B": announceB[maxTxAnnounces/2-1 : maxTxAnnounces-1], }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": hashesA[:maxTxAnnounces/2], - "B": hashesB[:maxTxAnnounces/2-1], + tracking: map[string][]announce{ + "A": announceA[:maxTxAnnounces/2], + "B": announceB[:maxTxAnnounces/2-1], }, fetching: map[string][]common.Hash{ "A": hashesA[1643 : 1643+maxTxRetrievals], @@ -1139,17 +1214,17 @@ func TestTransactionFetcherDoSProtection(t *testing.T) { }, }, // Ensure that adding even one more hash results in dropping the hash - doTxNotify{peer: "A", hashes: []common.Hash{hashesA[maxTxAnnounces]}}, - doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces-1 : maxTxAnnounces+1]}, + doTxNotify{peer: "A", hashes: []common.Hash{hashesA[maxTxAnnounces]}, types: []byte{typesA[maxTxAnnounces]}, sizes: []uint32{sizesA[maxTxAnnounces]}}, + doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces-1 : maxTxAnnounces+1], types: typesB[maxTxAnnounces-1 : maxTxAnnounces+1], sizes: sizesB[maxTxAnnounces-1 : maxTxAnnounces+1]}, - isWaiting(map[string][]common.Hash{ - "A": hashesA[maxTxAnnounces/2 : maxTxAnnounces], - "B": hashesB[maxTxAnnounces/2-1 : maxTxAnnounces], + isWaiting(map[string][]announce{ + "A": announceA[maxTxAnnounces/2 : maxTxAnnounces], + "B": announceB[maxTxAnnounces/2-1 : maxTxAnnounces], }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": hashesA[:maxTxAnnounces/2], - "B": hashesB[:maxTxAnnounces/2-1], + tracking: map[string][]announce{ + "A": announceA[:maxTxAnnounces/2], + "B": announceB[:maxTxAnnounces/2-1], }, fetching: map[string][]common.Hash{ "A": hashesA[1643 : 1643+maxTxRetrievals], @@ -1183,15 +1258,23 @@ func TestTransactionFetcherUnderpricedDedup(t *testing.T) { }, steps: []interface{}{ // Deliver a transaction through the fetcher, but reject as underpriced - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, + doTxNotify{peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size())}, + }, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1]}, direct: true}, isScheduled{nil, nil, nil}, // Try to announce the transaction again, ensure it's not scheduled back - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}}, // [2] is needed to force a step in the fetcher - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[2]}, + doTxNotify{peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type(), testTxs[2].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size()), uint32(testTxs[2].Size())}, + }, // [2] is needed to force a step in the fetcher + isWaiting(map[string][]announce{ + "A": {{testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}}, }), isScheduled{nil, nil, nil}, }, @@ -1211,21 +1294,38 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { for i := 0; i < maxTxUnderpricedSetSize+1; i++ { txs = append(txs, types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil)) } - hashes := make([]common.Hash, len(txs)) - for i, tx := range txs { - hashes[i] = tx.Hash() + var ( + hashes []common.Hash + ts []byte + sizes []uint32 + annos []announce + ) + for _, tx := range txs { + hashes = append(hashes, tx.Hash()) + ts = append(ts, tx.Type()) + sizes = append(sizes, uint32(tx.Size())) + annos = append(annos, announce{ + hash: tx.Hash(), + kind: tx.Type(), + size: uint32(tx.Size()), + }) } // Generate a set of steps to announce and deliver the entire set of transactions var steps []interface{} for i := 0; i < maxTxUnderpricedSetSize/maxTxRetrievals; i++ { - steps = append(steps, doTxNotify{peer: "A", hashes: hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals]}) - steps = append(steps, isWaiting(map[string][]common.Hash{ - "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + steps = append(steps, doTxNotify{ + peer: "A", + hashes: hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + types: ts[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + sizes: sizes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + }) + steps = append(steps, isWaiting(map[string][]announce{ + "A": annos[i*maxTxRetrievals : (i+1)*maxTxRetrievals], })) steps = append(steps, doWait{time: txArriveTimeout, step: true}) steps = append(steps, isScheduled{ - tracking: map[string][]common.Hash{ - "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + tracking: map[string][]announce{ + "A": annos[i*maxTxRetrievals : (i+1)*maxTxRetrievals], }, fetching: map[string][]common.Hash{ "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], @@ -1253,7 +1353,12 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { }, steps: append(steps, []interface{}{ // The preparation of the test has already been done in `steps`, add the last check - doTxNotify{peer: "A", hashes: []common.Hash{hashes[maxTxUnderpricedSetSize]}}, + doTxNotify{ + peer: "A", + hashes: []common.Hash{hashes[maxTxUnderpricedSetSize]}, + types: []byte{ts[maxTxUnderpricedSetSize]}, + sizes: []uint32{sizes[maxTxUnderpricedSetSize]}, + }, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{txs[maxTxUnderpricedSetSize]}, direct: true}, isUnderpriced(maxTxUnderpricedSetSize), @@ -1283,18 +1388,23 @@ func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { isScheduled{nil, nil, nil}, // Set up a few hashes into various stages - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}, types: []byte{testTxs[1].Type()}, sizes: []uint32{uint32(testTxs[1].Size())}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}, types: []byte{testTxs[2].Type()}, sizes: []uint32{uint32(testTxs[2].Size())}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[2]}, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -1330,18 +1440,23 @@ func TestTransactionFetcherDrop(t *testing.T) { }, steps: []interface{}{ // Set up a few hashes into various stages - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}, types: []byte{types.LegacyTxType}, sizes: []uint32{222}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{{0x03}}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x03}}, types: []byte{types.LegacyTxType}, sizes: []uint32{333}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x03}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x03}, types.LegacyTxType, 333}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}}, @@ -1353,12 +1468,14 @@ func TestTransactionFetcherDrop(t *testing.T) { isScheduled{nil, nil, nil}, // Push the node into a dangling (timeout) state - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -1397,15 +1514,15 @@ func TestTransactionFetcherDropRescheduling(t *testing.T) { }, steps: []interface{}{ // Set up a few hashes into various stages - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "B", hashes: []common.Hash{{0x01}}}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}}, - "B": {{0x01}}, + tracking: map[string][]announce{ + "A": {{common.Hash{0x01}, types.LegacyTxType, 111}}, + "B": {{common.Hash{0x01}, types.LegacyTxType, 111}}, }, fetching: map[string][]common.Hash{ "A": {{0x01}}, @@ -1415,8 +1532,8 @@ func TestTransactionFetcherDropRescheduling(t *testing.T) { doDrop("A"), isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "B": {{0x01}}, + tracking: map[string][]announce{ + "B": {{common.Hash{0x01}, types.LegacyTxType, 111}}, }, fetching: map[string][]common.Hash{ "B": {{0x01}}, @@ -1443,42 +1560,57 @@ func TestInvalidAnnounceMetadata(t *testing.T) { }, steps: []interface{}{ // Initial announcement to get something into the waitlist - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}, types: []byte{testTxs[0].Type(), testTxs[1].Type()}, sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size())}}, - isWaitingWithMeta(map[string][]announce{ + doTxNotify{ + peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size())}, + }, + isWaiting(map[string][]announce{ "A": { - {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(uint32(testTxs[0].Size()))}, - {testTxsHashes[1], typeptr(testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, }, }), // Announce from new peers conflicting transactions - doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{1024 + uint32(testTxs[0].Size())}}, - doTxNotify{peer: "C", hashes: []common.Hash{testTxsHashes[1]}, types: []byte{1 + testTxs[1].Type()}, sizes: []uint32{uint32(testTxs[1].Size())}}, - isWaitingWithMeta(map[string][]announce{ + doTxNotify{ + peer: "B", + hashes: []common.Hash{testTxsHashes[0]}, + types: []byte{testTxs[0].Type()}, + sizes: []uint32{1024 + uint32(testTxs[0].Size())}, + }, + doTxNotify{ + peer: "C", + hashes: []common.Hash{testTxsHashes[1]}, + types: []byte{1 + testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[1].Size())}, + }, + isWaiting(map[string][]announce{ "A": { - {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(uint32(testTxs[0].Size()))}, - {testTxsHashes[1], typeptr(testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, }, "B": { - {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(1024 + uint32(testTxs[0].Size()))}, + {testTxsHashes[0], testTxs[0].Type(), 1024 + uint32(testTxs[0].Size())}, }, "C": { - {testTxsHashes[1], typeptr(1 + testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + {testTxsHashes[1], 1 + testTxs[1].Type(), uint32(testTxs[1].Size())}, }, }), // Schedule all the transactions for retrieval doWait{time: txArriveTimeout, step: true}, - isWaitingWithMeta(nil), - isScheduledWithMeta{ + isWaiting(nil), + isScheduled{ tracking: map[string][]announce{ "A": { - {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(uint32(testTxs[0].Size()))}, - {testTxsHashes[1], typeptr(testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, }, "B": { - {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(1024 + uint32(testTxs[0].Size()))}, + {testTxsHashes[0], testTxs[0].Type(), 1024 + uint32(testTxs[0].Size())}, }, "C": { - {testTxsHashes[1], typeptr(1 + testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + {testTxsHashes[1], 1 + testTxs[1].Type(), uint32(testTxs[1].Size())}, }, }, fetching: map[string][]common.Hash{ @@ -1511,12 +1643,12 @@ func TestTransactionFetcherFuzzCrash01(t *testing.T) { }, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, // Notify the dangling transaction once more and crash via a timeout - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txFetchTimeout, step: true}, }, }) @@ -1539,12 +1671,12 @@ func TestTransactionFetcherFuzzCrash02(t *testing.T) { }, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, // Notify the dangling transaction once more, re-fetch, and crash via a drop and timeout - doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doDrop("A"), doWait{time: txFetchTimeout, step: true}, @@ -1569,12 +1701,17 @@ func TestTransactionFetcherFuzzCrash03(t *testing.T) { }, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, + doTxNotify{ + peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size())}, + }, doWait{time: txFetchTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1]}}, // Notify the dangling transaction once more, partially deliver, clash&crash with a timeout - doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[1]}, direct: true}, @@ -1606,12 +1743,12 @@ func TestTransactionFetcherFuzzCrash04(t *testing.T) { }, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, // Notify the dangling transaction once more, re-fetch, and crash via an in-flight disconnect - doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doFunc(func() { proceed <- struct{}{} // Allow peer A to return the failure @@ -1652,30 +1789,6 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { // Crunch through all the test steps and execute them for i, step := range tt.steps { - // Auto-expand certain steps to ones with metadata - switch old := step.(type) { - case isWaiting: - new := make(isWaitingWithMeta) - for peer, hashes := range old { - for _, hash := range hashes { - new[peer] = append(new[peer], announce{hash, nil, nil}) - } - } - step = new - - case isScheduled: - new := isScheduledWithMeta{ - tracking: make(map[string][]announce), - fetching: old.fetching, - dangling: old.dangling, - } - for peer, hashes := range old.tracking { - for _, hash := range hashes { - new.tracking[peer] = append(new.tracking[peer], announce{hash, nil, nil}) - } - } - step = new - } // Process the original or expanded steps switch step := step.(type) { case doTxNotify: @@ -1710,7 +1823,7 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { case doFunc: step() - case isWaitingWithMeta: + case isWaiting: // We need to check that the waiting list (stage 1) internals // match with the expected set. Check the peer->hash mappings // first. @@ -1724,18 +1837,13 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { if meta, ok := waiting[ann.hash]; !ok { t.Errorf("step %d, peer %s: hash %x missing from waitslots", i, peer, ann.hash) } else { - if (meta == nil && (ann.kind != nil || ann.size != nil)) || - (meta != nil && (ann.kind == nil || ann.size == nil)) || - (meta != nil && (meta.kind != *ann.kind || meta.size != *ann.size)) { - t.Errorf("step %d, peer %s, hash %x: waitslot metadata mismatch: want %v, have %v/%v", i, peer, ann.hash, meta, *ann.kind, *ann.size) + if meta.kind != ann.kind || meta.size != ann.size { + t.Errorf("step %d, peer %s, hash %x: waitslot metadata mismatch: want %v, have %v/%v", i, peer, ann.hash, meta, ann.kind, ann.size) } } } for hash, meta := range waiting { - ann := announce{hash: hash} - if meta != nil { - ann.kind, ann.size = &meta.kind, &meta.size - } + ann := announce{hash: hash, kind: meta.kind, size: meta.size} if !containsAnnounce(announces, ann) { t.Errorf("step %d, peer %s: announce %v extra in waitslots", i, peer, ann) } @@ -1780,7 +1888,7 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { } } - case isScheduledWithMeta: + case isScheduled: // Check that all scheduled announces are accounted for and no // extra ones are present. for peer, announces := range step.tracking { @@ -1793,18 +1901,13 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { if meta, ok := scheduled[ann.hash]; !ok { t.Errorf("step %d, peer %s: hash %x missing from announces", i, peer, ann.hash) } else { - if (meta == nil && (ann.kind != nil || ann.size != nil)) || - (meta != nil && (ann.kind == nil || ann.size == nil)) || - (meta != nil && (meta.kind != *ann.kind || meta.size != *ann.size)) { - t.Errorf("step %d, peer %s, hash %x: announce metadata mismatch: want %v, have %v/%v", i, peer, ann.hash, meta, *ann.kind, *ann.size) + if meta.kind != ann.kind || meta.size != ann.size { + t.Errorf("step %d, peer %s, hash %x: announce metadata mismatch: want %v, have %v/%v", i, peer, ann.hash, meta, ann.kind, ann.size) } } } for hash, meta := range scheduled { - ann := announce{hash: hash} - if meta != nil { - ann.kind, ann.size = &meta.kind, &meta.size - } + ann := announce{hash: hash, kind: meta.kind, size: meta.size} if !containsAnnounce(announces, ann) { t.Errorf("step %d, peer %s: announce %x extra in announces", i, peer, hash) } @@ -1954,18 +2057,10 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { func containsAnnounce(slice []announce, ann announce) bool { for _, have := range slice { if have.hash == ann.hash { - if have.kind == nil || ann.kind == nil { - if have.kind != ann.kind { - return false - } - } else if *have.kind != *ann.kind { + if have.kind != ann.kind { return false } - if have.size == nil || ann.size == nil { - if have.size != ann.size { - return false - } - } else if *have.size != *ann.size { + if have.size != ann.size { return false } return true From c35684709c8c71da78d5abbc6c38a501dd05b58a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 15 Aug 2024 11:42:39 +0200 Subject: [PATCH 34/39] eth/ethconfig: remove LES server config (#30298) --- .golangci.yml | 2 ++ cmd/geth/config.go | 32 +++++++++++++++----------------- cmd/utils/flags_legacy.go | 4 ---- eth/backend.go | 12 ++---------- eth/ethconfig/config.go | 15 ++++----------- eth/ethconfig/gen_config.go | 36 ------------------------------------ 6 files changed, 23 insertions(+), 78 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index d3b8535cfb..adb59f318f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -62,6 +62,8 @@ issues: - revive - path: cmd/utils/flags.go text: "SA1019: cfg.TxLookupLimit is deprecated: use 'TransactionHistory' instead." + - path: cmd/utils/flags.go + text: "SA1019: ethconfig.Defaults.TxLookupLimit is deprecated: use 'TransactionHistory' instead." - path: internal/build/pgp.go text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.' - path: core/vm/contracts.go diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 522e5e22f2..fd57ff40de 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -75,8 +75,8 @@ var tomlSettings = toml.Config{ }, MissingField: func(rt reflect.Type, field string) error { id := fmt.Sprintf("%s.%s", rt.String(), field) - if deprecated(id) { - log.Warn("Config field is deprecated and won't have an effect", "name", id) + if deprecatedConfigFields[id] { + log.Warn(fmt.Sprintf("Config field '%s' is deprecated and won't have any effect.", id)) return nil } var link string @@ -87,6 +87,19 @@ var tomlSettings = toml.Config{ }, } +var deprecatedConfigFields = map[string]bool{ + "ethconfig.Config.EVMInterpreter": true, + "ethconfig.Config.EWASMInterpreter": true, + "ethconfig.Config.TrieCleanCacheJournal": true, + "ethconfig.Config.TrieCleanCacheRejournal": true, + "ethconfig.Config.LightServ": true, + "ethconfig.Config.LightIngress": true, + "ethconfig.Config.LightEgress": true, + "ethconfig.Config.LightPeers": true, + "ethconfig.Config.LightNoPrune": true, + "ethconfig.Config.LightNoSyncServe": true, +} + type ethstatsConfig struct { URL string `toml:",omitempty"` } @@ -314,21 +327,6 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { } } -func deprecated(field string) bool { - switch field { - case "ethconfig.Config.EVMInterpreter": - return true - case "ethconfig.Config.EWASMInterpreter": - return true - case "ethconfig.Config.TrieCleanCacheJournal": - return true - case "ethconfig.Config.TrieCleanCacheRejournal": - return true - default: - return false - } -} - func setAccountManagerBackends(conf *node.Config, am *accounts.Manager, keydir string) error { scryptN := keystore.StandardScryptN scryptP := keystore.StandardScryptP diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index 1dfd1a5f89..f145f605d6 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -92,25 +92,21 @@ var ( LightServeFlag = &cli.IntFlag{ Name: "light.serve", Usage: "Maximum percentage of time allowed for serving LES requests (deprecated)", - Value: ethconfig.Defaults.LightServ, Category: flags.DeprecatedCategory, } LightIngressFlag = &cli.IntFlag{ Name: "light.ingress", Usage: "Incoming bandwidth limit for serving light clients (deprecated)", - Value: ethconfig.Defaults.LightIngress, Category: flags.DeprecatedCategory, } LightEgressFlag = &cli.IntFlag{ Name: "light.egress", Usage: "Outgoing bandwidth limit for serving light clients (deprecated)", - Value: ethconfig.Defaults.LightEgress, Category: flags.DeprecatedCategory, } LightMaxPeersFlag = &cli.IntFlag{ Name: "light.maxpeers", Usage: "Maximum number of light clients to serve, or light servers to attach to (deprecated)", - Value: ethconfig.Defaults.LightPeers, Category: flags.DeprecatedCategory, } LightNoPruneFlag = &cli.BoolFlag{ diff --git a/eth/backend.go b/eth/backend.go index 8679018dab..442d408bb0 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -377,16 +377,8 @@ func (s *Ethereum) Start() error { // Regularly update shutdown marker s.shutdownTracker.Start() - // Figure out a max peers count based on the server limits - maxPeers := s.p2pServer.MaxPeers - if s.config.LightServ > 0 { - if s.config.LightPeers >= s.p2pServer.MaxPeers { - return fmt.Errorf("invalid peer config: light peer count (%d) >= total peer count (%d)", s.config.LightPeers, s.p2pServer.MaxPeers) - } - maxPeers -= s.config.LightPeers - } - // Start the networking layer and the light server if requested - s.handler.Start(maxPeers) + // Start the networking layer + s.handler.Start(s.p2pServer.MaxPeers) return nil } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 7453fb1efd..5a955d9bbf 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -53,7 +53,6 @@ var Defaults = Config{ TxLookupLimit: 2350000, TransactionHistory: 2350000, StateHistory: params.FullImmutabilityThreshold, - LightPeers: 100, DatabaseCache: 512, TrieCleanCache: 154, TrieDirtyCache: 256, @@ -87,11 +86,13 @@ type Config struct { EthDiscoveryURLs []string SnapDiscoveryURLs []string + // State options. NoPruning bool // Whether to disable pruning and flush everything to disk NoPrefetch bool // Whether to disable prefetching and only load state on demand - // Deprecated, use 'TransactionHistory' instead. - TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + // Deprecated: use 'TransactionHistory' instead. + TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved. @@ -105,14 +106,6 @@ type Config struct { // presence of these blocks for every new peer connection. RequiredBlocks map[uint64]common.Hash `toml:"-"` - // Light client options - LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests - LightIngress int `toml:",omitempty"` // Incoming bandwidth limit for light servers - LightEgress int `toml:",omitempty"` // Outgoing bandwidth limit for light servers - LightPeers int `toml:",omitempty"` // Maximum number of LES client peers - LightNoPrune bool `toml:",omitempty"` // Whether to disable light chain pruning - LightNoSyncServe bool `toml:",omitempty"` // Whether to serve light clients before syncing - // Database options SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 147a559984..2854c87097 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -29,12 +29,6 @@ func (c Config) MarshalTOML() (interface{}, error) { StateHistory uint64 `toml:",omitempty"` StateScheme string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ int `toml:",omitempty"` - LightIngress int `toml:",omitempty"` - LightEgress int `toml:",omitempty"` - LightPeers int `toml:",omitempty"` - LightNoPrune bool `toml:",omitempty"` - LightNoSyncServe bool `toml:",omitempty"` SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int @@ -73,12 +67,6 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.StateHistory = c.StateHistory enc.StateScheme = c.StateScheme enc.RequiredBlocks = c.RequiredBlocks - enc.LightServ = c.LightServ - enc.LightIngress = c.LightIngress - enc.LightEgress = c.LightEgress - enc.LightPeers = c.LightPeers - enc.LightNoPrune = c.LightNoPrune - enc.LightNoSyncServe = c.LightNoSyncServe enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache @@ -121,12 +109,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { StateHistory *uint64 `toml:",omitempty"` StateScheme *string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ *int `toml:",omitempty"` - LightIngress *int `toml:",omitempty"` - LightEgress *int `toml:",omitempty"` - LightPeers *int `toml:",omitempty"` - LightNoPrune *bool `toml:",omitempty"` - LightNoSyncServe *bool `toml:",omitempty"` SkipBcVersionCheck *bool `toml:"-"` DatabaseHandles *int `toml:"-"` DatabaseCache *int @@ -192,24 +174,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.RequiredBlocks != nil { c.RequiredBlocks = dec.RequiredBlocks } - if dec.LightServ != nil { - c.LightServ = *dec.LightServ - } - if dec.LightIngress != nil { - c.LightIngress = *dec.LightIngress - } - if dec.LightEgress != nil { - c.LightEgress = *dec.LightEgress - } - if dec.LightPeers != nil { - c.LightPeers = *dec.LightPeers - } - if dec.LightNoPrune != nil { - c.LightNoPrune = *dec.LightNoPrune - } - if dec.LightNoSyncServe != nil { - c.LightNoSyncServe = *dec.LightNoSyncServe - } if dec.SkipBcVersionCheck != nil { c.SkipBcVersionCheck = *dec.SkipBcVersionCheck } From 7a149a159a609f5c0a1eb0cc76dc27a944a1007b Mon Sep 17 00:00:00 2001 From: Dylan Vassallo Date: Thu, 15 Aug 2024 16:36:35 +0200 Subject: [PATCH 35/39] eth/tracers/js: add coinbase addr to ctx (#30231) Add coinbase address to javascript tracer context. This PR adds the `coinbase` address to `jsTracer.ctx`, allowing access to the coinbase address (fee receipient) in custom JavaScript tracers. Example usage: ```javascript result: function(ctx) { return toAddress(ctx.coinbase); } ``` This change enables custom tracers to access coinbase address, previously unavailable, enhancing their capabilities to match built-in tracers. --- eth/tracers/js/goja.go | 6 ++++++ eth/tracers/js/tracer_test.go | 3 +++ 2 files changed, 9 insertions(+) diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index 5290d4f709..fd8f74fc54 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -254,6 +254,12 @@ func (t *jsTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from return } t.ctx["gasPrice"] = gasPriceBig + coinbase, err := t.toBuf(t.vm, env.Coinbase.Bytes()) + if err != nil { + t.err = err + return + } + t.ctx["coinbase"] = t.vm.ToValue(coinbase) } // OnTxEnd implements the Tracer interface and is invoked at the end of diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 05adedf265..7122b3c90e 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -154,6 +154,9 @@ func TestTracer(t *testing.T) { want: "", fail: "reached limit for padding memory slice: 1049568 at step (:1:83(20)) in server-side tracer function 'step'", contract: []byte{byte(vm.PUSH1), byte(0xff), byte(vm.PUSH1), byte(0x00), byte(vm.MSTORE8), byte(vm.STOP)}, + }, { // tests ctx.coinbase + code: "{lengths: [], step: function(log) { }, fault: function() {}, result: function(ctx) { var coinbase = ctx.coinbase; return toAddress(coinbase); }}", + want: `{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0,"8":0,"9":0,"10":0,"11":0,"12":0,"13":0,"14":0,"15":0,"16":0,"17":0,"18":0,"19":0}`, }, } { if have, err := execTracer(tt.code, tt.contract); tt.want != string(have) || tt.fail != err { From 6eb42a6b4f9a07a4a11a9ac706b6a738212de210 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 15 Aug 2024 22:14:42 +0200 Subject: [PATCH 36/39] eth: dial nodes from discv5 (#30302) Here I am adding a discv5 nodes source into the p2p dial iterator. It's an improved version of #29533. Unlike discv4, the discv5 random nodes iterator will always provide full ENRs. This means we can apply filtering to the results and will only try dialing nodes which explictly opt into the eth protocol with a matching chain. I have also removed the dial iterator from snap. We don't have an official DNS list for snap anymore, and I doubt anyone else is running one. While we could potentially filter for snap on discv5, there will be very few nodes announcing it, and the extra iterator would just stall the dialer. --------- Co-authored-by: lightclient --- eth/backend.go | 67 ++++++++++++++++++++++------------ eth/protocols/eth/discovery.go | 14 +++++++ eth/protocols/eth/handler.go | 3 +- eth/protocols/snap/handler.go | 11 +----- 4 files changed, 60 insertions(+), 35 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index 442d408bb0..51011ed0b2 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -64,15 +64,13 @@ type Config = ethconfig.Config // Ethereum implements the Ethereum full node service. type Ethereum struct { - config *ethconfig.Config + // core protocol objects + config *ethconfig.Config + txPool *txpool.TxPool + blockchain *core.BlockChain - // Handlers - txPool *txpool.TxPool - - blockchain *core.BlockChain - handler *handler - ethDialCandidates enode.Iterator - snapDialCandidates enode.Iterator + handler *handler + discmix *enode.FairMix // DB interfaces chainDb ethdb.Database // Block chain database @@ -162,6 +160,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms), p2pServer: stack.Server(), + discmix: enode.NewFairMix(0), shutdownTracker: shutdowncheck.NewShutdownTracker(chainDb), } bcVersion := rawdb.ReadDatabaseVersion(chainDb) @@ -266,17 +265,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, config.GPO, config.Miner.GasPrice) - // Setup DNS discovery iterators. - dnsclient := dnsdisc.NewClient(dnsdisc.Config{}) - eth.ethDialCandidates, err = dnsclient.NewIterator(eth.config.EthDiscoveryURLs...) - if err != nil { - return nil, err - } - eth.snapDialCandidates, err = dnsclient.NewIterator(eth.config.SnapDiscoveryURLs...) - if err != nil { - return nil, err - } - // Start the RPC service eth.netRPCService = ethapi.NewNetAPI(eth.p2pServer, networkID) @@ -359,9 +347,9 @@ func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } // Protocols returns all the currently configured // network protocols to start. func (s *Ethereum) Protocols() []p2p.Protocol { - protos := eth.MakeProtocols((*ethHandler)(s.handler), s.networkID, s.ethDialCandidates) + protos := eth.MakeProtocols((*ethHandler)(s.handler), s.networkID, s.discmix) if s.config.SnapshotCache > 0 { - protos = append(protos, snap.MakeProtocols((*snapHandler)(s.handler), s.snapDialCandidates)...) + protos = append(protos, snap.MakeProtocols((*snapHandler)(s.handler))...) } return protos } @@ -369,7 +357,7 @@ func (s *Ethereum) Protocols() []p2p.Protocol { // Start implements node.Lifecycle, starting all internal goroutines needed by the // Ethereum protocol implementation. func (s *Ethereum) Start() error { - eth.StartENRUpdater(s.blockchain, s.p2pServer.LocalNode()) + s.setupDiscovery() // Start the bloom bits servicing goroutines s.startBloomHandlers(params.BloomBitsBlocks) @@ -382,12 +370,43 @@ func (s *Ethereum) Start() error { return nil } +func (s *Ethereum) setupDiscovery() error { + eth.StartENRUpdater(s.blockchain, s.p2pServer.LocalNode()) + + // Add eth nodes from DNS. + dnsclient := dnsdisc.NewClient(dnsdisc.Config{}) + if len(s.config.EthDiscoveryURLs) > 0 { + iter, err := dnsclient.NewIterator(s.config.EthDiscoveryURLs...) + if err != nil { + return err + } + s.discmix.AddSource(iter) + } + + // Add snap nodes from DNS. + if len(s.config.SnapDiscoveryURLs) > 0 { + iter, err := dnsclient.NewIterator(s.config.SnapDiscoveryURLs...) + if err != nil { + return err + } + s.discmix.AddSource(iter) + } + + // Add DHT nodes from discv5. + if s.p2pServer.DiscoveryV5() != nil { + filter := eth.NewNodeFilter(s.blockchain) + iter := enode.Filter(s.p2pServer.DiscoveryV5().RandomNodes(), filter) + s.discmix.AddSource(iter) + } + + return nil +} + // Stop implements node.Lifecycle, terminating all internal goroutines used by the // Ethereum protocol. func (s *Ethereum) Stop() error { // Stop all the peer-related stuff first. - s.ethDialCandidates.Close() - s.snapDialCandidates.Close() + s.discmix.Close() s.handler.Stop() // Then stop everything else. diff --git a/eth/protocols/eth/discovery.go b/eth/protocols/eth/discovery.go index a7bdd47daf..075a85b69c 100644 --- a/eth/protocols/eth/discovery.go +++ b/eth/protocols/eth/discovery.go @@ -64,3 +64,17 @@ func currentENREntry(chain *core.BlockChain) *enrEntry { ForkID: forkid.NewID(chain.Config(), chain.Genesis(), head.Number.Uint64(), head.Time), } } + +// NewNodeFilter returns a filtering function that returns whether the provided +// enode advertises a forkid compatible with the current chain. +func NewNodeFilter(chain *core.BlockChain) func(*enode.Node) bool { + filter := forkid.NewFilter(chain) + return func(n *enode.Node) bool { + var entry enrEntry + if err := n.Load(entry); err != nil { + return false + } + err := filter(entry.ForkID) + return err == nil + } +} diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 2d69ecdc83..ecf07b6a2c 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -113,8 +113,7 @@ func MakeProtocols(backend Backend, network uint64, dnsdisc enode.Iterator) []p2 PeerInfo: func(id enode.ID) interface{} { return backend.PeerInfo(id) }, - Attributes: []enr.Entry{currentENREntry(backend.Chain())}, - DialCandidates: dnsdisc, + Attributes: []enr.Entry{currentENREntry(backend.Chain())}, }) } return protocols diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 5cbe9d1270..a6c60bc075 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -82,13 +82,7 @@ type Backend interface { } // MakeProtocols constructs the P2P protocol definitions for `snap`. -func MakeProtocols(backend Backend, dnsdisc enode.Iterator) []p2p.Protocol { - // Filter the discovery iterator for nodes advertising snap support. - dnsdisc = enode.Filter(dnsdisc, func(n *enode.Node) bool { - var snap enrEntry - return n.Load(&snap) == nil - }) - +func MakeProtocols(backend Backend) []p2p.Protocol { protocols := make([]p2p.Protocol, len(ProtocolVersions)) for i, version := range ProtocolVersions { version := version // Closure @@ -108,8 +102,7 @@ func MakeProtocols(backend Backend, dnsdisc enode.Iterator) []p2p.Protocol { PeerInfo: func(id enode.ID) interface{} { return backend.PeerInfo(id) }, - Attributes: []enr.Entry{&enrEntry{}}, - DialCandidates: dnsdisc, + Attributes: []enr.Entry{&enrEntry{}}, } } return protocols From 43640f12d8576cf5e27a99f34280b3e20fb900d3 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Fri, 16 Aug 2024 14:39:57 +0200 Subject: [PATCH 37/39] beacon/light: handle endpoint URL more gracefully (#30306) blsync was failing if the light endpoint it was provided ended with a `/`. This change should handle the joining more gracefully. --- beacon/light/api/light_api.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/beacon/light/api/light_api.go b/beacon/light/api/light_api.go index 6f60fc0cc6..45b425164f 100755 --- a/beacon/light/api/light_api.go +++ b/beacon/light/api/light_api.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "net/http" + "net/url" "sync" "time" @@ -121,7 +122,11 @@ func NewBeaconLightApi(url string, customHeaders map[string]string) *BeaconLight } func (api *BeaconLightApi) httpGet(path string) ([]byte, error) { - req, err := http.NewRequest("GET", api.url+path, nil) + uri, err := api.buildURL(path, nil) + if err != nil { + return nil, err + } + req, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } @@ -545,9 +550,13 @@ func (api *BeaconLightApi) StartHeadListener(listener HeadEventListener) func() // established. It can only return nil when the context is canceled. func (api *BeaconLightApi) startEventStream(ctx context.Context, listener *HeadEventListener) *eventsource.Stream { for retry := true; retry; retry = ctxSleep(ctx, 5*time.Second) { - path := "/eth/v1/events?topics=head&topics=light_client_finality_update&topics=light_client_optimistic_update" log.Trace("Sending event subscription request") - req, err := http.NewRequestWithContext(ctx, "GET", api.url+path, nil) + uri, err := api.buildURL("/eth/v1/events", map[string][]string{"topics": {"head", "light_client_finality_update", "light_client_optimistic_update"}}) + if err != nil { + listener.OnError(fmt.Errorf("error creating event subscription URL: %v", err)) + continue + } + req, err := http.NewRequestWithContext(ctx, "GET", uri, nil) if err != nil { listener.OnError(fmt.Errorf("error creating event subscription request: %v", err)) continue @@ -576,3 +585,15 @@ func ctxSleep(ctx context.Context, timeout time.Duration) (ok bool) { return false } } + +func (api *BeaconLightApi) buildURL(path string, params url.Values) (string, error) { + uri, err := url.Parse(api.url) + if err != nil { + return "", err + } + uri = uri.JoinPath(path) + if params != nil { + uri.RawQuery = params.Encode() + } + return uri.String(), nil +} From 09d889d2e3405f3c15d0b92fc0f027bed232a6e1 Mon Sep 17 00:00:00 2001 From: Shude Li Date: Fri, 16 Aug 2024 22:33:41 +0800 Subject: [PATCH 38/39] core: remove withdrawal length check for state processor (#30286) The withdrawal length is already verified by the beacon consensus package, so the check in the state processor is a duplicate. --- core/state_processor.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/state_processor.go b/core/state_processor.go index c21f644f98..cc41583165 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -17,7 +17,6 @@ package core import ( - "errors" "fmt" "math/big" @@ -93,11 +92,6 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg receipts = append(receipts, receipt) allLogs = append(allLogs, receipt.Logs...) } - // Fail if Shanghai not enabled and len(withdrawals) is non-zero. - withdrawals := block.Withdrawals() - if len(withdrawals) > 0 && !p.config.IsShanghai(block.Number(), block.Time()) { - return nil, nil, 0, errors.New("withdrawals before shanghai") - } // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) p.chain.engine.Finalize(p.chain, header, statedb, block.Body()) From 710c3f32ac8e4e5829a6a631dcfb1e0e13a49220 Mon Sep 17 00:00:00 2001 From: Arran Schlosberg <519948+ARR4N@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:41:44 +0100 Subject: [PATCH 39/39] vm: simplify error handling in `vm.EVM.create()` (#30292) To allow all error paths in `vm.EVM.create()` to consume the necessary gas, there is currently a pattern of gating code on `if err == nil` instead of returning as soon as the error occurs. The same behaviour can be achieved by abstracting the gated code into a method that returns immediately on error, improving readability and thus making it easier to understand and maintain. --- core/vm/evm.go | 97 ++++++++++++++++++++++++-------------------------- 1 file changed, 46 insertions(+), 51 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 1944189b5d..7617d843c7 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -510,66 +510,61 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, contract.SetCodeOptionalHash(&address, codeAndHash) contract.IsDeployment = true - // Charge the contract creation init gas in verkle mode - if evm.chainRules.IsEIP4762 { - if !contract.UseGas(evm.AccessEvents.ContractCreateInitGas(address, value.Sign() != 0), evm.Config.Tracer, tracing.GasChangeWitnessContractInit) { - err = ErrOutOfGas - } - } - - if err == nil { - ret, err = evm.interpreter.Run(contract, nil, false) - } - - // Check whether the max code size has been exceeded, assign err if the case. - if err == nil && evm.chainRules.IsEIP158 && len(ret) > params.MaxCodeSize { - err = ErrMaxCodeSizeExceeded - } - - // Reject code starting with 0xEF if EIP-3541 is enabled. - if err == nil && len(ret) >= 1 && ret[0] == 0xEF && evm.chainRules.IsLondon { - err = ErrInvalidCode - } - - // if the contract creation ran successfully and no errors were returned - // calculate the gas required to store the code. If the code could not - // be stored due to not enough gas set an error and let it be handled - // by the error checking condition below. - if err == nil { - if !evm.chainRules.IsEIP4762 { - createDataGas := uint64(len(ret)) * params.CreateDataGas - if !contract.UseGas(createDataGas, evm.Config.Tracer, tracing.GasChangeCallCodeStorage) { - err = ErrCodeStoreOutOfGas - } - } else { - // Contract creation completed, touch the missing fields in the contract - if !contract.UseGas(evm.AccessEvents.AddAccount(address, true), evm.Config.Tracer, tracing.GasChangeWitnessContractCreation) { - err = ErrCodeStoreOutOfGas - } - - if err == nil && len(ret) > 0 && !contract.UseGas(evm.AccessEvents.CodeChunksRangeGas(address, 0, uint64(len(ret)), uint64(len(ret)), true), evm.Config.Tracer, tracing.GasChangeWitnessCodeChunk) { - err = ErrCodeStoreOutOfGas - } - } - - if err == nil { - evm.StateDB.SetCode(address, ret) - } - } - - // When an error was returned by the EVM or when setting the creation code - // above we revert to the snapshot and consume any gas remaining. Additionally, - // when we're in homestead this also counts for code storage gas errors. + ret, err = evm.initNewContract(contract, address, value) if err != nil && (evm.chainRules.IsHomestead || err != ErrCodeStoreOutOfGas) { evm.StateDB.RevertToSnapshot(snapshot) if err != ErrExecutionReverted { contract.UseGas(contract.Gas, evm.Config.Tracer, tracing.GasChangeCallFailedExecution) } } - return ret, address, contract.Gas, err } +// initNewContract runs a new contract's creation code, performs checks on the +// resulting code that is to be deployed, and consumes necessary gas. +func (evm *EVM) initNewContract(contract *Contract, address common.Address, value *uint256.Int) ([]byte, error) { + // Charge the contract creation init gas in verkle mode + if evm.chainRules.IsEIP4762 { + if !contract.UseGas(evm.AccessEvents.ContractCreateInitGas(address, value.Sign() != 0), evm.Config.Tracer, tracing.GasChangeWitnessContractInit) { + return nil, ErrOutOfGas + } + } + + ret, err := evm.interpreter.Run(contract, nil, false) + if err != nil { + return ret, err + } + + // Check whether the max code size has been exceeded, assign err if the case. + if evm.chainRules.IsEIP158 && len(ret) > params.MaxCodeSize { + return ret, ErrMaxCodeSizeExceeded + } + + // Reject code starting with 0xEF if EIP-3541 is enabled. + if len(ret) >= 1 && ret[0] == 0xEF && evm.chainRules.IsLondon { + return ret, ErrInvalidCode + } + + if !evm.chainRules.IsEIP4762 { + createDataGas := uint64(len(ret)) * params.CreateDataGas + if !contract.UseGas(createDataGas, evm.Config.Tracer, tracing.GasChangeCallCodeStorage) { + return ret, ErrCodeStoreOutOfGas + } + } else { + // Contract creation completed, touch the missing fields in the contract + if !contract.UseGas(evm.AccessEvents.AddAccount(address, true), evm.Config.Tracer, tracing.GasChangeWitnessContractCreation) { + return ret, ErrCodeStoreOutOfGas + } + + if len(ret) > 0 && !contract.UseGas(evm.AccessEvents.CodeChunksRangeGas(address, 0, uint64(len(ret)), uint64(len(ret)), true), evm.Config.Tracer, tracing.GasChangeWitnessCodeChunk) { + return ret, ErrCodeStoreOutOfGas + } + } + + evm.StateDB.SetCode(address, ret) + return ret, nil +} + // Create creates a new contract using code as deployment code. func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))