diff --git a/.mailmap b/.mailmap index aa074b76d6..312e51d854 100644 --- a/.mailmap +++ b/.mailmap @@ -56,7 +56,6 @@ Diederik Loerakker Dimitry Khokhlov Domino Valdano -Domino Valdano Edgar Aroutiounian diff --git a/Makefile b/Makefile index d736ef61c0..99b8ba54b4 100644 --- a/Makefile +++ b/Makefile @@ -8,20 +8,25 @@ GOBIN = ./build/bin GO ?= latest GORUN = go run +#? geth: Build geth geth: $(GORUN) build/ci.go install ./cmd/geth @echo "Done building." @echo "Run \"$(GOBIN)/geth\" to launch geth." +#? all: Build all packages and executables all: $(GORUN) build/ci.go install +#? test: Run the tests test: all $(GORUN) build/ci.go test +#? lint: Run certain pre-selected linters lint: ## Run linters. $(GORUN) build/ci.go lint +#? clean: Clean go cache, built executables, and the auto generated folder clean: go clean -cache rm -fr build/_workspace/pkg/ $(GOBIN)/* @@ -29,6 +34,7 @@ clean: # The devtools target installs tools required for 'go generate'. # You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'. +#? devtools: Install recommended developer tools devtools: env GOBIN= go install golang.org/x/tools/cmd/stringer@latest env GOBIN= go install github.com/fjl/gencodec@latest @@ -36,3 +42,9 @@ devtools: env GOBIN= go install ./cmd/abigen @type "solc" 2> /dev/null || echo 'Please install solc' @type "protoc" 2> /dev/null || echo 'Please install protoc' + +#? help: Get more info on make commands. +help: Makefile + @echo " Choose a command run in go-ethereum:" + @sed -n 's/^#?//p' $< | column -t -s ':' | sort | sed -e 's/^/ /' +.PHONY: help diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index 4abf298068..c7bc2b4541 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -29,7 +29,7 @@ import ( ) // The ABI holds information about a contract's context and available -// invokable methods. It will allow you to type check function calls and +// invocable methods. It will allow you to type check function calls and // packs data accordingly. type ABI struct { Constructor Method diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 756a9d3552..dfd9296952 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -20,7 +20,7 @@ import ( "context" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient/simulated" ) @@ -43,7 +43,7 @@ func (b *SimulatedBackend) Fork(ctx context.Context, parentHash common.Hash) err // // Deprecated: please use simulated.Backend from package // github.com/ethereum/go-ethereum/ethclient/simulated instead. -func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend { +func NewSimulatedBackend(alloc types.GenesisAlloc, gasLimit uint64) *SimulatedBackend { b := simulated.NewBackend(alloc, simulated.WithBlockGasLimit(gasLimit)) return &SimulatedBackend{ Backend: b, diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index a6ffe7609d..a390a3c47c 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -289,7 +289,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -297,7 +297,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy an interaction tester contract and call a transaction on it @@ -345,7 +345,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -353,7 +353,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a tuple tester contract and execute a structured call on it @@ -391,7 +391,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -399,7 +399,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a tuple tester contract and execute a structured call on it @@ -449,7 +449,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -457,7 +457,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a slice tester contract and execute a n array call on it @@ -497,7 +497,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -505,7 +505,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a default method invoker contract and execute its default method @@ -564,7 +564,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -572,7 +572,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a structs method invoker contract and execute its default method @@ -610,12 +610,12 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" `, ` // Create a simulator and wrap a non-deployed contract - sim := backends.NewSimulatedBackend(core.GenesisAlloc{}, uint64(10000000000)) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{}, uint64(10000000000)) defer sim.Close() nonexistent, err := NewNonExistent(common.Address{}, sim) @@ -649,12 +649,12 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" `, ` // Create a simulator and wrap a non-deployed contract - sim := backends.NewSimulatedBackend(core.GenesisAlloc{}, uint64(10000000000)) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{}, uint64(10000000000)) defer sim.Close() nonexistent, err := NewNonExistentStruct(common.Address{}, sim) @@ -696,7 +696,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -704,7 +704,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a funky gas pattern contract @@ -746,7 +746,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -754,7 +754,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a sender tester contract and execute a structured call on it @@ -821,7 +821,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -829,7 +829,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a underscorer tester contract and execute a structured call on it @@ -915,7 +915,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -923,7 +923,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy an eventer contract @@ -1105,7 +1105,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -1113,7 +1113,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() //deploy the test contract @@ -1240,7 +1240,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, @@ -1248,7 +1248,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() _, _, contract, err := DeployTuple(auth, sim) @@ -1382,7 +1382,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -1390,7 +1390,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() //deploy the test contract @@ -1448,14 +1448,14 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` // Initialize test accounts key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // deploy the test contract @@ -1537,7 +1537,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" `, ` // Initialize test accounts @@ -1545,7 +1545,7 @@ var bindTests = []struct { addr := crypto.PubkeyToAddress(key.PublicKey) // Deploy registrar contract - sim := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() transactOpts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) @@ -1600,14 +1600,14 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" `, ` key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) // Deploy registrar contract - sim := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() transactOpts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) @@ -1661,7 +1661,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -1669,7 +1669,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a tester contract and execute a structured call on it @@ -1722,14 +1722,14 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 1000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 1000000) defer sim.Close() opts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) @@ -1810,7 +1810,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" `, @@ -1818,7 +1818,7 @@ var bindTests = []struct { var ( key, _ = crypto.GenerateKey() user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) defer sim.Close() @@ -1881,7 +1881,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" `, @@ -1889,7 +1889,7 @@ var bindTests = []struct { var ( key, _ = crypto.GenerateKey() user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) defer sim.Close() @@ -1934,7 +1934,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" `, @@ -1942,7 +1942,7 @@ var bindTests = []struct { var ( key, _ = crypto.GenerateKey() user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) defer sim.Close() @@ -1983,7 +1983,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" `, @@ -1991,7 +1991,7 @@ var bindTests = []struct { var ( key, _ = crypto.GenerateKey() user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) defer sim.Close() @@ -2024,7 +2024,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" `, @@ -2032,7 +2032,7 @@ var bindTests = []struct { var ( key, _ = crypto.GenerateKey() user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) _, tx, _, err := DeployRangeKeyword(user, sim) if err != nil { diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index 9fd919a295..592465f2ac 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient/simulated" @@ -57,7 +56,7 @@ func TestWaitDeployed(t *testing.T) { t.Parallel() for name, test := range waitDeployedTests { backend := simulated.NewBackend( - core.GenesisAlloc{ + types.GenesisAlloc{ crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)}, }, ) @@ -65,7 +64,7 @@ func TestWaitDeployed(t *testing.T) { // Create the transaction head, _ := backend.Client().HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(params.GWei)) tx := types.NewContractCreation(0, big.NewInt(0), test.gas, gasPrice, common.FromHex(test.code)) tx, _ = types.SignTx(tx, types.LatestSignerForChainID(big.NewInt(1337)), testKey) @@ -102,7 +101,7 @@ func TestWaitDeployed(t *testing.T) { func TestWaitDeployedCornerCases(t *testing.T) { backend := simulated.NewBackend( - core.GenesisAlloc{ + types.GenesisAlloc{ crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)}, }, ) diff --git a/accounts/abi/type.go b/accounts/abi/type.go index 2eee11787f..3839826633 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -179,9 +179,6 @@ func NewType(t string, internalType string, components []ArgumentMarshaling) (ty return Type{}, errors.New("abi: purely anonymous or underscored field is not supported") } fieldName := ResolveNameConflict(name, func(s string) bool { return used[s] }) - if err != nil { - return Type{}, err - } used[fieldName] = true if !isValidFieldName(fieldName) { return Type{}, fmt.Errorf("field %d has invalid name", idx) diff --git a/accounts/accounts.go b/accounts/accounts.go index 6c351a9649..b995498a6d 100644 --- a/accounts/accounts.go +++ b/accounts/accounts.go @@ -195,7 +195,7 @@ func TextHash(data []byte) []byte { // // This gives context to the signed message and prevents signing of transactions. func TextAndHash(data []byte) ([]byte, string) { - msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), string(data)) + msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data) hasher := sha3.NewLegacyKeccak256() hasher.Write([]byte(msg)) return hasher.Sum(nil), msg diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index 48a238048f..bb92cc2adc 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -86,7 +86,7 @@ func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error { func TestWatchNewFile(t *testing.T) { t.Parallel() - dir, ks := tmpKeyStore(t, false) + dir, ks := tmpKeyStore(t) // Ensure the watcher is started before adding any files. ks.Accounts() diff --git a/accounts/keystore/keystore.go b/accounts/keystore/keystore.go index 0ffcf376a5..e62a8eb257 100644 --- a/accounts/keystore/keystore.go +++ b/accounts/keystore/keystore.go @@ -87,15 +87,6 @@ func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore { return ks } -// NewPlaintextKeyStore creates a keystore for the given directory. -// Deprecated: Use NewKeyStore. -func NewPlaintextKeyStore(keydir string) *KeyStore { - keydir, _ = filepath.Abs(keydir) - ks := &KeyStore{storage: &keyStorePlain{keydir}} - ks.init(keydir) - return ks -} - func (ks *KeyStore) init(keydir string) { // Lock the mutex since the account cache might call back with events ks.mu.Lock() diff --git a/accounts/keystore/keystore_test.go b/accounts/keystore/keystore_test.go index c9a23eddd6..c871392b82 100644 --- a/accounts/keystore/keystore_test.go +++ b/accounts/keystore/keystore_test.go @@ -37,7 +37,7 @@ var testSigData = make([]byte, 32) func TestKeyStore(t *testing.T) { t.Parallel() - dir, ks := tmpKeyStore(t, true) + dir, ks := tmpKeyStore(t) a, err := ks.NewAccount("foo") if err != nil { @@ -72,7 +72,7 @@ func TestKeyStore(t *testing.T) { func TestSign(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) pass := "" // not used but required by API a1, err := ks.NewAccount(pass) @@ -89,7 +89,7 @@ func TestSign(t *testing.T) { func TestSignWithPassphrase(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) pass := "passwd" acc, err := ks.NewAccount(pass) @@ -117,7 +117,7 @@ func TestSignWithPassphrase(t *testing.T) { func TestTimedUnlock(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) pass := "foo" a1, err := ks.NewAccount(pass) @@ -152,7 +152,7 @@ func TestTimedUnlock(t *testing.T) { func TestOverrideUnlock(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) pass := "foo" a1, err := ks.NewAccount(pass) @@ -193,7 +193,7 @@ func TestOverrideUnlock(t *testing.T) { // This test should fail under -race if signing races the expiration goroutine. func TestSignRace(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) // Create a test account. a1, err := ks.NewAccount("") @@ -238,7 +238,7 @@ func waitForKsUpdating(t *testing.T, ks *KeyStore, wantStatus bool, maxTime time func TestWalletNotifierLifecycle(t *testing.T) { t.Parallel() // Create a temporary keystore to test with - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) // Ensure that the notification updater is not running yet time.Sleep(250 * time.Millisecond) @@ -284,7 +284,7 @@ type walletEvent struct { // or deleted from the keystore. func TestWalletNotifications(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) // Subscribe to the wallet feed and collect events. var ( @@ -346,7 +346,7 @@ func TestWalletNotifications(t *testing.T) { // TestImportExport tests the import functionality of a keystore. func TestImportECDSA(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) key, err := crypto.GenerateKey() if err != nil { t.Fatalf("failed to generate key: %v", key) @@ -365,7 +365,7 @@ func TestImportECDSA(t *testing.T) { // TestImportECDSA tests the import and export functionality of a keystore. func TestImportExport(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) acc, err := ks.NewAccount("old") if err != nil { t.Fatalf("failed to create account: %v", acc) @@ -374,7 +374,7 @@ func TestImportExport(t *testing.T) { if err != nil { t.Fatalf("failed to export account: %v", acc) } - _, ks2 := tmpKeyStore(t, true) + _, ks2 := tmpKeyStore(t) if _, err = ks2.Import(json, "old", "old"); err == nil { t.Errorf("importing with invalid password succeeded") } @@ -394,7 +394,7 @@ func TestImportExport(t *testing.T) { // This test should fail under -race if importing races. func TestImportRace(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) acc, err := ks.NewAccount("old") if err != nil { t.Fatalf("failed to create account: %v", acc) @@ -403,7 +403,7 @@ func TestImportRace(t *testing.T) { if err != nil { t.Fatalf("failed to export account: %v", acc) } - _, ks2 := tmpKeyStore(t, true) + _, ks2 := tmpKeyStore(t) var atom atomic.Uint32 var wg sync.WaitGroup wg.Add(2) @@ -457,11 +457,7 @@ func checkEvents(t *testing.T, want []walletEvent, have []walletEvent) { } } -func tmpKeyStore(t *testing.T, encrypted bool) (string, *KeyStore) { +func tmpKeyStore(t *testing.T) (string, *KeyStore) { d := t.TempDir() - newKs := NewPlaintextKeyStore - if encrypted { - newKs = func(kd string) *KeyStore { return NewKeyStore(kd, veryLightScryptN, veryLightScryptP) } - } - return d, newKs(d) + return d, NewKeyStore(d, veryLightScryptN, veryLightScryptP) } diff --git a/accounts/scwallet/hub.go b/accounts/scwallet/hub.go index f9dcf58e19..5f1f369ca2 100644 --- a/accounts/scwallet/hub.go +++ b/accounts/scwallet/hub.go @@ -241,7 +241,7 @@ func (hub *Hub) refreshWallets() { card.Disconnect(pcsc.LeaveCard) continue } - // Card connected, start tracking in amongs the wallets + // Card connected, start tracking among the wallets hub.wallets[reader] = wallet events = append(events, accounts.WalletEvent{Wallet: wallet, Kind: accounts.WalletArrived}) } diff --git a/accounts/usbwallet/hub.go b/accounts/usbwallet/hub.go index e67942dbc1..e22dffe971 100644 --- a/accounts/usbwallet/hub.go +++ b/accounts/usbwallet/hub.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" - "github.com/karalabe/usb" + "github.com/karalabe/hid" ) // LedgerScheme is the protocol scheme prefixing account and wallet URLs. @@ -109,7 +109,7 @@ func NewTrezorHubWithWebUSB() (*Hub, error) { // newHub creates a new hardware wallet manager for generic USB devices. func newHub(scheme string, vendorID uint16, productIDs []uint16, usageID uint16, endpointID int, makeDriver func(log.Logger) driver) (*Hub, error) { - if !usb.Supported() { + if !hid.Supported() { return nil, errors.New("unsupported platform") } hub := &Hub{ @@ -155,7 +155,7 @@ func (hub *Hub) refreshWallets() { return } // Retrieve the current list of USB wallet devices - var devices []usb.DeviceInfo + var devices []hid.DeviceInfo if runtime.GOOS == "linux" { // hidapi on Linux opens the device during enumeration to retrieve some infos, @@ -170,7 +170,7 @@ func (hub *Hub) refreshWallets() { return } } - infos, err := usb.Enumerate(hub.vendorID, 0) + infos, err := hid.Enumerate(hub.vendorID, 0) if err != nil { failcount := hub.enumFails.Add(1) if runtime.GOOS == "linux" { diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index d0cb93e74e..81836b3717 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -16,7 +16,7 @@ // This file contains the implementation for interacting with the Ledger hardware // wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo: -// https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc +// https://github.com/LedgerHQ/app-ethereum/blob/develop/doc/ethapp.adoc package usbwallet diff --git a/accounts/usbwallet/trezor/trezor.go b/accounts/usbwallet/trezor/trezor.go index 7e756e609b..cdca6b4e0b 100644 --- a/accounts/usbwallet/trezor/trezor.go +++ b/accounts/usbwallet/trezor/trezor.go @@ -16,7 +16,7 @@ // This file contains the implementation for interacting with the Trezor hardware // wallets. The wire protocol spec can be found on the SatoshiLabs website: -// https://wiki.trezor.io/Developers_guide-Message_Workflows +// https://docs.trezor.io/trezor-firmware/common/message-workflows.html // !!! STAHP !!! // diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 69083dc893..0fd0415a9e 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/karalabe/usb" + "github.com/karalabe/hid" ) // Maximum time between wallet health checks to detect USB unplugs. @@ -79,8 +79,8 @@ type wallet struct { driver driver // Hardware implementation of the low level device operations url *accounts.URL // Textual URL uniquely identifying this wallet - info usb.DeviceInfo // Known USB device infos about the wallet - device usb.Device // USB device advertising itself as a hardware wallet + info hid.DeviceInfo // Known USB device infos about the wallet + device hid.Device // USB device advertising itself as a hardware wallet accounts []accounts.Account // List of derive accounts pinned on the hardware wallet paths map[common.Address]accounts.DerivationPath // Known derivation paths for signing operations diff --git a/beacon/blsync/block_sync.go b/beacon/blsync/block_sync.go new file mode 100755 index 0000000000..91b21163e6 --- /dev/null +++ b/beacon/blsync/block_sync.go @@ -0,0 +1,203 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + ctypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/zrnt/eth2/configs" + "github.com/protolambda/ztyp/tree" +) + +// beaconBlockSync implements request.Module; it fetches the beacon blocks belonging +// to the validated and prefetch heads. +type beaconBlockSync struct { + recentBlocks *lru.Cache[common.Hash, *capella.BeaconBlock] + locked map[common.Hash]request.ServerAndID + serverHeads map[request.Server]common.Hash + headTracker headTracker + + lastHeadInfo types.HeadInfo + chainHeadFeed *event.Feed +} + +type headTracker interface { + PrefetchHead() types.HeadInfo + ValidatedHead() (types.SignedHeader, bool) + ValidatedFinality() (types.FinalityUpdate, bool) +} + +// newBeaconBlockSync returns a new beaconBlockSync. +func newBeaconBlockSync(headTracker headTracker, chainHeadFeed *event.Feed) *beaconBlockSync { + return &beaconBlockSync{ + headTracker: headTracker, + chainHeadFeed: chainHeadFeed, + recentBlocks: lru.NewCache[common.Hash, *capella.BeaconBlock](10), + locked: make(map[common.Hash]request.ServerAndID), + serverHeads: make(map[request.Server]common.Hash), + } +} + +// Process implements request.Module. +func (s *beaconBlockSync) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + switch event.Type { + case request.EvResponse, request.EvFail, request.EvTimeout: + sid, req, resp := event.RequestInfo() + blockRoot := common.Hash(req.(sync.ReqBeaconBlock)) + if resp != nil { + s.recentBlocks.Add(blockRoot, resp.(*capella.BeaconBlock)) + } + if s.locked[blockRoot] == sid { + delete(s.locked, blockRoot) + } + case sync.EvNewHead: + s.serverHeads[event.Server] = event.Data.(types.HeadInfo).BlockRoot + case request.EvUnregistered: + delete(s.serverHeads, event.Server) + } + } + s.updateEventFeed() + // request validated head block if unavailable and not yet requested + if vh, ok := s.headTracker.ValidatedHead(); ok { + s.tryRequestBlock(requester, vh.Header.Hash(), false) + } + // request prefetch head if the given server has announced it + if prefetchHead := s.headTracker.PrefetchHead().BlockRoot; prefetchHead != (common.Hash{}) { + s.tryRequestBlock(requester, prefetchHead, true) + } +} + +func (s *beaconBlockSync) tryRequestBlock(requester request.Requester, blockRoot common.Hash, needSameHead bool) { + if _, ok := s.recentBlocks.Get(blockRoot); ok { + return + } + if _, ok := s.locked[blockRoot]; ok { + return + } + for _, server := range requester.CanSendTo() { + if needSameHead && (s.serverHeads[server] != blockRoot) { + continue + } + id := requester.Send(server, sync.ReqBeaconBlock(blockRoot)) + s.locked[blockRoot] = request.ServerAndID{Server: server, ID: id} + return + } +} + +func blockHeadInfo(block *capella.BeaconBlock) types.HeadInfo { + if block == nil { + return types.HeadInfo{} + } + return types.HeadInfo{Slot: uint64(block.Slot), BlockRoot: beaconBlockHash(block)} +} + +// beaconBlockHash calculates the hash of a beacon block. +func beaconBlockHash(beaconBlock *capella.BeaconBlock) common.Hash { + return common.Hash(beaconBlock.HashTreeRoot(configs.Mainnet, tree.GetHashFn())) +} + +// getExecBlock extracts the execution block from the beacon block's payload. +func getExecBlock(beaconBlock *capella.BeaconBlock) (*ctypes.Block, error) { + payload := &beaconBlock.Body.ExecutionPayload + txs := make([]*ctypes.Transaction, len(payload.Transactions)) + for i, opaqueTx := range payload.Transactions { + var tx ctypes.Transaction + if err := tx.UnmarshalBinary(opaqueTx); err != nil { + return nil, fmt.Errorf("failed to parse tx %d: %v", i, err) + } + txs[i] = &tx + } + withdrawals := make([]*ctypes.Withdrawal, len(payload.Withdrawals)) + for i, w := range payload.Withdrawals { + withdrawals[i] = &ctypes.Withdrawal{ + Index: uint64(w.Index), + Validator: uint64(w.ValidatorIndex), + Address: common.Address(w.Address), + Amount: uint64(w.Amount), + } + } + wroot := ctypes.DeriveSha(ctypes.Withdrawals(withdrawals), trie.NewStackTrie(nil)) + execHeader := &ctypes.Header{ + ParentHash: common.Hash(payload.ParentHash), + UncleHash: ctypes.EmptyUncleHash, + Coinbase: common.Address(payload.FeeRecipient), + Root: common.Hash(payload.StateRoot), + TxHash: ctypes.DeriveSha(ctypes.Transactions(txs), trie.NewStackTrie(nil)), + ReceiptHash: common.Hash(payload.ReceiptsRoot), + Bloom: ctypes.Bloom(payload.LogsBloom), + Difficulty: common.Big0, + Number: new(big.Int).SetUint64(uint64(payload.BlockNumber)), + GasLimit: uint64(payload.GasLimit), + GasUsed: uint64(payload.GasUsed), + Time: uint64(payload.Timestamp), + Extra: []byte(payload.ExtraData), + MixDigest: common.Hash(payload.PrevRandao), // reused in merge + Nonce: ctypes.BlockNonce{}, // zero + BaseFee: (*uint256.Int)(&payload.BaseFeePerGas).ToBig(), + WithdrawalsHash: &wroot, + } + execBlock := ctypes.NewBlockWithHeader(execHeader).WithBody(txs, nil).WithWithdrawals(withdrawals) + if execBlockHash := execBlock.Hash(); execBlockHash != common.Hash(payload.BlockHash) { + return execBlock, fmt.Errorf("Sanity check failed, payload hash does not match (expected %x, got %x)", common.Hash(payload.BlockHash), execBlockHash) + } + return execBlock, nil +} + +func (s *beaconBlockSync) updateEventFeed() { + head, ok := s.headTracker.ValidatedHead() + if !ok { + return + } + finality, ok := s.headTracker.ValidatedFinality() //TODO fetch directly if subscription does not deliver + if !ok || head.Header.Epoch() != finality.Attested.Header.Epoch() { + return + } + validatedHead := head.Header.Hash() + headBlock, ok := s.recentBlocks.Get(validatedHead) + if !ok { + return + } + headInfo := blockHeadInfo(headBlock) + if headInfo == s.lastHeadInfo { + return + } + s.lastHeadInfo = headInfo + // new head block and finality info available; extract executable data and send event to feed + execBlock, err := getExecBlock(headBlock) + if err != nil { + log.Error("Error extracting execution block from validated beacon block", "error", err) + return + } + s.chainHeadFeed.Send(types.ChainHeadEvent{ + HeadBlock: engine.BlockToExecutableData(execBlock, nil, nil).ExecutionPayload, + Finalized: common.Hash(finality.Finalized.PayloadHeader.BlockHash), + }) +} diff --git a/beacon/blsync/block_sync_test.go b/beacon/blsync/block_sync_test.go new file mode 100644 index 0000000000..9ce434d862 --- /dev/null +++ b/beacon/blsync/block_sync_test.go @@ -0,0 +1,160 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "testing" + + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/event" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/zrnt/eth2/configs" + "github.com/protolambda/ztyp/tree" +) + +var ( + testServer1 = "testServer1" + testServer2 = "testServer2" + + testBlock1 = &capella.BeaconBlock{ + Slot: 123, + Body: capella.BeaconBlockBody{ + ExecutionPayload: capella.ExecutionPayload{BlockNumber: 456}, + }, + } + testBlock2 = &capella.BeaconBlock{ + Slot: 124, + Body: capella.BeaconBlockBody{ + ExecutionPayload: capella.ExecutionPayload{BlockNumber: 457}, + }, + } +) + +func init() { + eb1, _ := getExecBlock(testBlock1) + testBlock1.Body.ExecutionPayload.BlockHash = tree.Root(eb1.Hash()) + eb2, _ := getExecBlock(testBlock2) + testBlock2.Body.ExecutionPayload.BlockHash = tree.Root(eb2.Hash()) +} + +func TestBlockSync(t *testing.T) { + ht := &testHeadTracker{} + eventFeed := new(event.Feed) + blockSync := newBeaconBlockSync(ht, eventFeed) + headCh := make(chan types.ChainHeadEvent, 16) + eventFeed.Subscribe(headCh) + ts := sync.NewTestScheduler(t, blockSync) + ts.AddServer(testServer1, 1) + ts.AddServer(testServer2, 1) + + expHeadBlock := func(tci int, expHead *capella.BeaconBlock) { + var expNumber, headNumber uint64 + if expHead != nil { + expNumber = uint64(expHead.Body.ExecutionPayload.BlockNumber) + } + select { + case event := <-headCh: + headNumber = event.HeadBlock.Number + default: + } + if headNumber != expNumber { + t.Errorf("Wrong head block in test case #%d (expected block number %d, got %d)", tci, expNumber, headNumber) + } + } + + // no block requests expected until head tracker knows about a head + ts.Run(1) + expHeadBlock(1, nil) + + // set block 1 as prefetch head, announced by server 2 + head1 := blockHeadInfo(testBlock1) + ht.prefetch = head1 + ts.ServerEvent(sync.EvNewHead, testServer2, head1) + // expect request to server 2 which has announced the head + ts.Run(2, testServer2, sync.ReqBeaconBlock(head1.BlockRoot)) + + // valid response + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testBlock1) + ts.AddAllowance(testServer2, 1) + ts.Run(3) + // head block still not expected as the fetched block is not the validated head yet + expHeadBlock(3, nil) + + // set as validated head, expect no further requests but block 1 set as head block + ht.validated.Header = blockHeader(testBlock1) + ts.Run(4) + expHeadBlock(4, testBlock1) + + // set block 2 as prefetch head, announced by server 1 + head2 := blockHeadInfo(testBlock2) + ht.prefetch = head2 + ts.ServerEvent(sync.EvNewHead, testServer1, head2) + // expect request to server 1 + ts.Run(5, testServer1, sync.ReqBeaconBlock(head2.BlockRoot)) + + // req2 fails, no further requests expected because server 2 has not announced it + ts.RequestEvent(request.EvFail, ts.Request(5, 1), nil) + ts.Run(6) + + // set as validated head before retrieving block; now it's assumed to be available from server 2 too + ht.validated.Header = blockHeader(testBlock2) + // expect req2 retry to server 2 + ts.Run(7, testServer2, sync.ReqBeaconBlock(head2.BlockRoot)) + // now head block should be unavailable again + expHeadBlock(4, nil) + + // valid response, now head block should be block 2 immediately as it is already validated + ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testBlock2) + ts.Run(8) + expHeadBlock(5, testBlock2) +} + +func blockHeader(block *capella.BeaconBlock) types.Header { + return types.Header{ + Slot: uint64(block.Slot), + ProposerIndex: uint64(block.ProposerIndex), + ParentRoot: common.Hash(block.ParentRoot), + StateRoot: common.Hash(block.StateRoot), + BodyRoot: common.Hash(block.Body.HashTreeRoot(configs.Mainnet, tree.GetHashFn())), + } +} + +type testHeadTracker struct { + prefetch types.HeadInfo + validated types.SignedHeader +} + +func (h *testHeadTracker) PrefetchHead() types.HeadInfo { + return h.prefetch +} + +func (h *testHeadTracker) ValidatedHead() (types.SignedHeader, bool) { + return h.validated, h.validated.Header != (types.Header{}) +} + +// TODO add test case for finality +func (h *testHeadTracker) ValidatedFinality() (types.FinalityUpdate, bool) { + return types.FinalityUpdate{ + Attested: types.HeaderWithExecProof{Header: h.validated.Header}, + Finalized: types.HeaderWithExecProof{PayloadHeader: &capella.ExecutionPayloadHeader{}}, + Signature: h.validated.Signature, + SignatureSlot: h.validated.SignatureSlot, + }, h.validated.Header != (types.Header{}) +} diff --git a/beacon/blsync/client.go b/beacon/blsync/client.go new file mode 100644 index 0000000000..1bfbb13160 --- /dev/null +++ b/beacon/blsync/client.go @@ -0,0 +1,103 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "strings" + + "github.com/ethereum/go-ethereum/beacon/light" + "github.com/ethereum/go-ethereum/beacon/light/api" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/event" + "github.com/urfave/cli/v2" +) + +type Client struct { + scheduler *request.Scheduler + chainHeadFeed *event.Feed + urls []string + customHeader map[string]string +} + +func NewClient(ctx *cli.Context) *Client { + if !ctx.IsSet(utils.BeaconApiFlag.Name) { + utils.Fatalf("Beacon node light client API URL not specified") + } + var ( + chainConfig = makeChainConfig(ctx) + customHeader = make(map[string]string) + ) + for _, s := range ctx.StringSlice(utils.BeaconApiHeaderFlag.Name) { + kv := strings.Split(s, ":") + if len(kv) != 2 { + utils.Fatalf("Invalid custom API header entry: %s", s) + } + customHeader[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) + } + // create data structures + var ( + db = memorydb.New() + threshold = ctx.Int(utils.BeaconThresholdFlag.Name) + committeeChain = light.NewCommitteeChain(db, chainConfig.ChainConfig, threshold, !ctx.Bool(utils.BeaconNoFilterFlag.Name)) + headTracker = light.NewHeadTracker(committeeChain, threshold) + ) + headSync := sync.NewHeadSync(headTracker, committeeChain) + + // set up scheduler and sync modules + chainHeadFeed := new(event.Feed) + scheduler := request.NewScheduler() + checkpointInit := sync.NewCheckpointInit(committeeChain, chainConfig.Checkpoint) + forwardSync := sync.NewForwardUpdateSync(committeeChain) + beaconBlockSync := newBeaconBlockSync(headTracker, chainHeadFeed) + scheduler.RegisterTarget(headTracker) + scheduler.RegisterTarget(committeeChain) + scheduler.RegisterModule(checkpointInit, "checkpointInit") + scheduler.RegisterModule(forwardSync, "forwardSync") + scheduler.RegisterModule(headSync, "headSync") + scheduler.RegisterModule(beaconBlockSync, "beaconBlockSync") + + return &Client{ + scheduler: scheduler, + urls: ctx.StringSlice(utils.BeaconApiFlag.Name), + customHeader: customHeader, + chainHeadFeed: chainHeadFeed, + } +} + +// SubscribeChainHeadEvent allows callers to subscribe a provided channel to new +// head updates. +func (c *Client) SubscribeChainHeadEvent(ch chan<- types.ChainHeadEvent) event.Subscription { + return c.chainHeadFeed.Subscribe(ch) +} + +func (c *Client) Start() { + c.scheduler.Start() + // register server(s) + for _, url := range c.urls { + beaconApi := api.NewBeaconLightApi(url, c.customHeader) + c.scheduler.RegisterServer(request.NewServer(api.NewApiServer(beaconApi), &mclock.System{})) + } +} + +func (c *Client) Stop() { + c.scheduler.Stop() +} diff --git a/beacon/blsync/config.go b/beacon/blsync/config.go new file mode 100644 index 0000000000..b51d3e2b05 --- /dev/null +++ b/beacon/blsync/config.go @@ -0,0 +1,113 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/urfave/cli/v2" +) + +// lightClientConfig contains beacon light client configuration +type lightClientConfig struct { + *types.ChainConfig + Checkpoint common.Hash +} + +var ( + MainnetConfig = lightClientConfig{ + ChainConfig: (&types.ChainConfig{ + GenesisValidatorsRoot: common.HexToHash("0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95"), + GenesisTime: 1606824023, + }). + AddFork("GENESIS", 0, []byte{0, 0, 0, 0}). + AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}). + AddFork("BELLATRIX", 144896, []byte{2, 0, 0, 0}). + AddFork("CAPELLA", 194048, []byte{3, 0, 0, 0}), + Checkpoint: common.HexToHash("0x388be41594ec7d6a6894f18c73f3469f07e2c19a803de4755d335817ed8e2e5a"), + } + + SepoliaConfig = lightClientConfig{ + ChainConfig: (&types.ChainConfig{ + GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"), + GenesisTime: 1655733600, + }). + AddFork("GENESIS", 0, []byte{144, 0, 0, 105}). + AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}). + AddFork("BELLATRIX", 100, []byte{144, 0, 0, 113}). + AddFork("CAPELLA", 56832, []byte{144, 0, 0, 114}), + Checkpoint: common.HexToHash("0x1005a6d9175e96bfbce4d35b80f468e9bff0b674e1e861d16e09e10005a58e81"), + } + + GoerliConfig = lightClientConfig{ + ChainConfig: (&types.ChainConfig{ + GenesisValidatorsRoot: common.HexToHash("0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb"), + GenesisTime: 1614588812, + }). + AddFork("GENESIS", 0, []byte{0, 0, 16, 32}). + AddFork("ALTAIR", 36660, []byte{1, 0, 16, 32}). + AddFork("BELLATRIX", 112260, []byte{2, 0, 16, 32}). + AddFork("CAPELLA", 162304, []byte{3, 0, 16, 32}), + Checkpoint: common.HexToHash("0x53a0f4f0a378e2c4ae0a9ee97407eb69d0d737d8d8cd0a5fb1093f42f7b81c49"), + } +) + +func makeChainConfig(ctx *cli.Context) lightClientConfig { + utils.CheckExclusive(ctx, utils.MainnetFlag, utils.GoerliFlag, utils.SepoliaFlag) + customConfig := ctx.IsSet(utils.BeaconConfigFlag.Name) || ctx.IsSet(utils.BeaconGenesisRootFlag.Name) || ctx.IsSet(utils.BeaconGenesisTimeFlag.Name) + var config lightClientConfig + switch { + case ctx.Bool(utils.MainnetFlag.Name): + config = MainnetConfig + case ctx.Bool(utils.SepoliaFlag.Name): + config = SepoliaConfig + case ctx.Bool(utils.GoerliFlag.Name): + config = GoerliConfig + default: + if !customConfig { + config = MainnetConfig + } + } + if customConfig && config.Forks != nil { + utils.Fatalf("Cannot use custom beacon chain config flags in combination with pre-defined network config") + } + if ctx.IsSet(utils.BeaconGenesisRootFlag.Name) { + if c, err := hexutil.Decode(ctx.String(utils.BeaconGenesisRootFlag.Name)); err == nil && len(c) <= 32 { + copy(config.GenesisValidatorsRoot[:len(c)], c) + } else { + utils.Fatalf("Invalid hex string", "beacon.genesis.gvroot", ctx.String(utils.BeaconGenesisRootFlag.Name), "error", err) + } + } + if ctx.IsSet(utils.BeaconGenesisTimeFlag.Name) { + config.GenesisTime = ctx.Uint64(utils.BeaconGenesisTimeFlag.Name) + } + if ctx.IsSet(utils.BeaconConfigFlag.Name) { + if err := config.ChainConfig.LoadForks(ctx.String(utils.BeaconConfigFlag.Name)); err != nil { + utils.Fatalf("Could not load beacon chain config file", "file name", ctx.String(utils.BeaconConfigFlag.Name), "error", err) + } + } + if ctx.IsSet(utils.BeaconCheckpointFlag.Name) { + if c, err := hexutil.Decode(ctx.String(utils.BeaconCheckpointFlag.Name)); err == nil && len(c) <= 32 { + copy(config.Checkpoint[:len(c)], c) + } else { + utils.Fatalf("Invalid hex string", "beacon.checkpoint", ctx.String(utils.BeaconCheckpointFlag.Name), "error", err) + } + } + return config +} diff --git a/beacon/engine/types.go b/beacon/engine/types.go index f72319ad50..60accc3c79 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -303,3 +303,21 @@ type ExecutionPayloadBodyV1 struct { TransactionData []hexutil.Bytes `json:"transactions"` Withdrawals []*types.Withdrawal `json:"withdrawals"` } + +// Client identifiers to support ClientVersionV1. +const ( + ClientCode = "GE" + ClientName = "go-ethereum" +) + +// ClientVersionV1 contains information which identifies a client implementation. +type ClientVersionV1 struct { + Code string `json:"code"` + Name string `json:"clientName"` + Version string `json:"version"` + Commit string `json:"commit"` +} + +func (v *ClientVersionV1) String() string { + return fmt.Sprintf("%s-%s-%s-%s", v.Code, v.Name, v.Version, v.Commit) +} diff --git a/beacon/light/api/api_server.go b/beacon/light/api/api_server.go new file mode 100755 index 0000000000..da044f4b2d --- /dev/null +++ b/beacon/light/api/api_server.go @@ -0,0 +1,103 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "reflect" + + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +// ApiServer is a wrapper around BeaconLightApi that implements request.requestServer. +type ApiServer struct { + api *BeaconLightApi + eventCallback func(event request.Event) + unsubscribe func() +} + +// NewApiServer creates a new ApiServer. +func NewApiServer(api *BeaconLightApi) *ApiServer { + return &ApiServer{api: api} +} + +// Subscribe implements request.requestServer. +func (s *ApiServer) Subscribe(eventCallback func(event request.Event)) { + s.eventCallback = eventCallback + listener := HeadEventListener{ + OnNewHead: func(slot uint64, blockRoot common.Hash) { + log.Debug("New head received", "slot", slot, "blockRoot", blockRoot) + eventCallback(request.Event{Type: sync.EvNewHead, Data: types.HeadInfo{Slot: slot, BlockRoot: blockRoot}}) + }, + OnSignedHead: func(head types.SignedHeader) { + log.Debug("New signed head received", "slot", head.Header.Slot, "blockRoot", head.Header.Hash(), "signerCount", head.Signature.SignerCount()) + eventCallback(request.Event{Type: sync.EvNewSignedHead, Data: head}) + }, + OnFinality: func(head types.FinalityUpdate) { + log.Debug("New finality update received", "slot", head.Attested.Slot, "blockRoot", head.Attested.Hash(), "signerCount", head.Signature.SignerCount()) + eventCallback(request.Event{Type: sync.EvNewFinalityUpdate, Data: head}) + }, + OnError: func(err error) { + log.Warn("Head event stream error", "err", err) + }, + } + s.unsubscribe = s.api.StartHeadListener(listener) +} + +// SendRequest implements request.requestServer. +func (s *ApiServer) SendRequest(id request.ID, req request.Request) { + go func() { + var resp request.Response + var err error + switch data := req.(type) { + case sync.ReqUpdates: + log.Debug("Beacon API: requesting light client update", "reqid", id, "period", data.FirstPeriod, "count", data.Count) + var r sync.RespUpdates + r.Updates, r.Committees, err = s.api.GetBestUpdatesAndCommittees(data.FirstPeriod, data.Count) + resp = r + case sync.ReqHeader: + log.Debug("Beacon API: requesting header", "reqid", id, "hash", common.Hash(data)) + resp, err = s.api.GetHeader(common.Hash(data)) + case sync.ReqCheckpointData: + log.Debug("Beacon API: requesting checkpoint data", "reqid", id, "hash", common.Hash(data)) + resp, err = s.api.GetCheckpointData(common.Hash(data)) + case sync.ReqBeaconBlock: + log.Debug("Beacon API: requesting block", "reqid", id, "hash", common.Hash(data)) + resp, err = s.api.GetBeaconBlock(common.Hash(data)) + default: + } + + if err != nil { + log.Warn("Beacon API request failed", "type", reflect.TypeOf(req), "reqid", id, "err", err) + s.eventCallback(request.Event{Type: request.EvFail, Data: request.RequestResponse{ID: id, Request: req}}) + } else { + s.eventCallback(request.Event{Type: request.EvResponse, Data: request.RequestResponse{ID: id, Request: req, Response: resp}}) + } + }() +} + +// Unsubscribe implements request.requestServer. +// Note: Unsubscribe should not be called concurrently with Subscribe. +func (s *ApiServer) Unsubscribe() { + if s.unsubscribe != nil { + s.unsubscribe() + s.unsubscribe = nil + } +} diff --git a/beacon/light/api/light_api.go b/beacon/light/api/light_api.go new file mode 100755 index 0000000000..fd701dc0a8 --- /dev/null +++ b/beacon/light/api/light_api.go @@ -0,0 +1,496 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more detaiapi. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" + + "github.com/donovanhide/eventsource" + "github.com/ethereum/go-ethereum/beacon/merkle" + "github.com/ethereum/go-ethereum/beacon/params" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/zrnt/eth2/configs" + "github.com/protolambda/ztyp/tree" +) + +var ( + ErrNotFound = errors.New("404 Not Found") + ErrInternal = errors.New("500 Internal Server Error") +) + +type CommitteeUpdate struct { + Version string + Update types.LightClientUpdate + NextSyncCommittee types.SerializedSyncCommittee +} + +// See data structure definition here: +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientupdate +type committeeUpdateJson struct { + Version string `json:"version"` + Data committeeUpdateData `json:"data"` +} + +type committeeUpdateData struct { + Header jsonBeaconHeader `json:"attested_header"` + NextSyncCommittee types.SerializedSyncCommittee `json:"next_sync_committee"` + NextSyncCommitteeBranch merkle.Values `json:"next_sync_committee_branch"` + FinalizedHeader *jsonBeaconHeader `json:"finalized_header,omitempty"` + FinalityBranch merkle.Values `json:"finality_branch,omitempty"` + SyncAggregate types.SyncAggregate `json:"sync_aggregate"` + SignatureSlot common.Decimal `json:"signature_slot"` +} + +type jsonBeaconHeader struct { + Beacon types.Header `json:"beacon"` +} + +type jsonHeaderWithExecProof struct { + Beacon types.Header `json:"beacon"` + Execution *capella.ExecutionPayloadHeader `json:"execution"` + ExecutionBranch merkle.Values `json:"execution_branch"` +} + +// UnmarshalJSON unmarshals from JSON. +func (u *CommitteeUpdate) UnmarshalJSON(input []byte) error { + var dec committeeUpdateJson + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + u.Version = dec.Version + u.NextSyncCommittee = dec.Data.NextSyncCommittee + u.Update = types.LightClientUpdate{ + AttestedHeader: types.SignedHeader{ + Header: dec.Data.Header.Beacon, + Signature: dec.Data.SyncAggregate, + SignatureSlot: uint64(dec.Data.SignatureSlot), + }, + NextSyncCommitteeRoot: u.NextSyncCommittee.Root(), + NextSyncCommitteeBranch: dec.Data.NextSyncCommitteeBranch, + FinalityBranch: dec.Data.FinalityBranch, + } + if dec.Data.FinalizedHeader != nil { + u.Update.FinalizedHeader = &dec.Data.FinalizedHeader.Beacon + } + return nil +} + +// fetcher is an interface useful for debug-harnessing the http api. +type fetcher interface { + Do(req *http.Request) (*http.Response, error) +} + +// BeaconLightApi requests light client information from a beacon node REST API. +// Note: all required API endpoints are currently only implemented by Lodestar. +type BeaconLightApi struct { + url string + client fetcher + customHeaders map[string]string +} + +func NewBeaconLightApi(url string, customHeaders map[string]string) *BeaconLightApi { + return &BeaconLightApi{ + url: url, + client: &http.Client{ + Timeout: time.Second * 10, + }, + customHeaders: customHeaders, + } +} + +func (api *BeaconLightApi) httpGet(path string) ([]byte, error) { + req, err := http.NewRequest("GET", api.url+path, nil) + if err != nil { + return nil, err + } + for k, v := range api.customHeaders { + req.Header.Set(k, v) + } + resp, err := api.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + switch resp.StatusCode { + case 200: + return io.ReadAll(resp.Body) + case 404: + return nil, ErrNotFound + case 500: + return nil, ErrInternal + default: + return nil, fmt.Errorf("unexpected error from API endpoint \"%s\": status code %d", path, resp.StatusCode) + } +} + +func (api *BeaconLightApi) httpGetf(format string, params ...any) ([]byte, error) { + return api.httpGet(fmt.Sprintf(format, params...)) +} + +// GetBestUpdateAndCommittee fetches and validates LightClientUpdate for given +// period and full serialized committee for the next period (committee root hash +// equals update.NextSyncCommitteeRoot). +// Note that the results are validated but the update signature should be verified +// by the caller as its validity depends on the update chain. +func (api *BeaconLightApi) GetBestUpdatesAndCommittees(firstPeriod, count uint64) ([]*types.LightClientUpdate, []*types.SerializedSyncCommittee, error) { + resp, err := api.httpGetf("/eth/v1/beacon/light_client/updates?start_period=%d&count=%d", firstPeriod, count) + if err != nil { + return nil, nil, err + } + + var data []CommitteeUpdate + if err := json.Unmarshal(resp, &data); err != nil { + return nil, nil, err + } + if len(data) != int(count) { + return nil, nil, errors.New("invalid number of committee updates") + } + updates := make([]*types.LightClientUpdate, int(count)) + committees := make([]*types.SerializedSyncCommittee, int(count)) + for i, d := range data { + if d.Update.AttestedHeader.Header.SyncPeriod() != firstPeriod+uint64(i) { + return nil, nil, errors.New("wrong committee update header period") + } + if err := d.Update.Validate(); err != nil { + return nil, nil, err + } + if d.NextSyncCommittee.Root() != d.Update.NextSyncCommitteeRoot { + return nil, nil, errors.New("wrong sync committee root") + } + updates[i], committees[i] = new(types.LightClientUpdate), new(types.SerializedSyncCommittee) + *updates[i], *committees[i] = d.Update, d.NextSyncCommittee + } + return updates, committees, nil +} + +// GetOptimisticHeadUpdate fetches a signed header based on the latest available +// optimistic update. Note that the signature should be verified by the caller +// as its validity depends on the update chain. +// +// See data structure definition here: +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientoptimisticupdate +func (api *BeaconLightApi) GetOptimisticHeadUpdate() (types.SignedHeader, error) { + resp, err := api.httpGet("/eth/v1/beacon/light_client/optimistic_update") + if err != nil { + return types.SignedHeader{}, err + } + return decodeOptimisticHeadUpdate(resp) +} + +func decodeOptimisticHeadUpdate(enc []byte) (types.SignedHeader, error) { + var data struct { + Data struct { + Header jsonBeaconHeader `json:"attested_header"` + Aggregate types.SyncAggregate `json:"sync_aggregate"` + SignatureSlot common.Decimal `json:"signature_slot"` + } `json:"data"` + } + if err := json.Unmarshal(enc, &data); err != nil { + return types.SignedHeader{}, err + } + if data.Data.Header.Beacon.StateRoot == (common.Hash{}) { + // workaround for different event encoding format in Lodestar + if err := json.Unmarshal(enc, &data.Data); err != nil { + return types.SignedHeader{}, err + } + } + + if len(data.Data.Aggregate.Signers) != params.SyncCommitteeBitmaskSize { + return types.SignedHeader{}, errors.New("invalid sync_committee_bits length") + } + if len(data.Data.Aggregate.Signature) != params.BLSSignatureSize { + return types.SignedHeader{}, errors.New("invalid sync_committee_signature length") + } + return types.SignedHeader{ + Header: data.Data.Header.Beacon, + Signature: data.Data.Aggregate, + SignatureSlot: uint64(data.Data.SignatureSlot), + }, nil +} + +// GetFinalityUpdate fetches the latest available finality update. +// +// See data structure definition here: +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientfinalityupdate +func (api *BeaconLightApi) GetFinalityUpdate() (types.FinalityUpdate, error) { + resp, err := api.httpGet("/eth/v1/beacon/light_client/finality_update") + if err != nil { + return types.FinalityUpdate{}, err + } + return decodeFinalityUpdate(resp) +} + +func decodeFinalityUpdate(enc []byte) (types.FinalityUpdate, error) { + var data struct { + Data struct { + Attested jsonHeaderWithExecProof `json:"attested_header"` + Finalized jsonHeaderWithExecProof `json:"finalized_header"` + FinalityBranch merkle.Values `json:"finality_branch"` + Aggregate types.SyncAggregate `json:"sync_aggregate"` + SignatureSlot common.Decimal `json:"signature_slot"` + } `json:"data"` + } + if err := json.Unmarshal(enc, &data); err != nil { + return types.FinalityUpdate{}, err + } + + if len(data.Data.Aggregate.Signers) != params.SyncCommitteeBitmaskSize { + return types.FinalityUpdate{}, errors.New("invalid sync_committee_bits length") + } + if len(data.Data.Aggregate.Signature) != params.BLSSignatureSize { + return types.FinalityUpdate{}, errors.New("invalid sync_committee_signature length") + } + return types.FinalityUpdate{ + Attested: types.HeaderWithExecProof{ + Header: data.Data.Attested.Beacon, + PayloadHeader: data.Data.Attested.Execution, + PayloadBranch: data.Data.Attested.ExecutionBranch, + }, + Finalized: types.HeaderWithExecProof{ + Header: data.Data.Finalized.Beacon, + PayloadHeader: data.Data.Finalized.Execution, + PayloadBranch: data.Data.Finalized.ExecutionBranch, + }, + FinalityBranch: data.Data.FinalityBranch, + Signature: data.Data.Aggregate, + SignatureSlot: uint64(data.Data.SignatureSlot), + }, nil +} + +// GetHead fetches and validates the beacon header with the given blockRoot. +// If blockRoot is null hash then the latest head header is fetched. +func (api *BeaconLightApi) GetHeader(blockRoot common.Hash) (types.Header, error) { + var blockId string + if blockRoot == (common.Hash{}) { + blockId = "head" + } else { + blockId = blockRoot.Hex() + } + resp, err := api.httpGetf("/eth/v1/beacon/headers/%s", blockId) + if err != nil { + return types.Header{}, err + } + + var data struct { + Data struct { + Root common.Hash `json:"root"` + Canonical bool `json:"canonical"` + Header struct { + Message types.Header `json:"message"` + Signature hexutil.Bytes `json:"signature"` + } `json:"header"` + } `json:"data"` + } + if err := json.Unmarshal(resp, &data); err != nil { + return types.Header{}, err + } + header := data.Data.Header.Message + if blockRoot == (common.Hash{}) { + blockRoot = data.Data.Root + } + if header.Hash() != blockRoot { + return types.Header{}, errors.New("retrieved beacon header root does not match") + } + return header, nil +} + +// GetCheckpointData fetches and validates bootstrap data belonging to the given checkpoint. +func (api *BeaconLightApi) GetCheckpointData(checkpointHash common.Hash) (*types.BootstrapData, error) { + resp, err := api.httpGetf("/eth/v1/beacon/light_client/bootstrap/0x%x", checkpointHash[:]) + if err != nil { + return nil, err + } + + // See data structure definition here: + // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientbootstrap + type bootstrapData struct { + Data struct { + Header jsonBeaconHeader `json:"header"` + Committee *types.SerializedSyncCommittee `json:"current_sync_committee"` + CommitteeBranch merkle.Values `json:"current_sync_committee_branch"` + } `json:"data"` + } + + var data bootstrapData + if err := json.Unmarshal(resp, &data); err != nil { + return nil, err + } + if data.Data.Committee == nil { + return nil, errors.New("sync committee is missing") + } + header := data.Data.Header.Beacon + if header.Hash() != checkpointHash { + return nil, fmt.Errorf("invalid checkpoint block header, have %v want %v", header.Hash(), checkpointHash) + } + checkpoint := &types.BootstrapData{ + Header: header, + CommitteeBranch: data.Data.CommitteeBranch, + CommitteeRoot: data.Data.Committee.Root(), + Committee: data.Data.Committee, + } + if err := checkpoint.Validate(); err != nil { + return nil, fmt.Errorf("invalid checkpoint: %w", err) + } + if checkpoint.Header.Hash() != checkpointHash { + return nil, errors.New("wrong checkpoint hash") + } + return checkpoint, nil +} + +func (api *BeaconLightApi) GetBeaconBlock(blockRoot common.Hash) (*capella.BeaconBlock, error) { + resp, err := api.httpGetf("/eth/v2/beacon/blocks/0x%x", blockRoot) + if err != nil { + return nil, err + } + + var beaconBlockMessage struct { + Data struct { + Message capella.BeaconBlock `json:"message"` + } `json:"data"` + } + if err := json.Unmarshal(resp, &beaconBlockMessage); err != nil { + return nil, fmt.Errorf("invalid block json data: %v", err) + } + beaconBlock := new(capella.BeaconBlock) + *beaconBlock = beaconBlockMessage.Data.Message + root := common.Hash(beaconBlock.HashTreeRoot(configs.Mainnet, tree.GetHashFn())) + if root != blockRoot { + return nil, fmt.Errorf("Beacon block root hash mismatch (expected: %x, got: %x)", blockRoot, root) + } + return beaconBlock, nil +} + +func decodeHeadEvent(enc []byte) (uint64, common.Hash, error) { + var data struct { + Slot common.Decimal `json:"slot"` + Block common.Hash `json:"block"` + } + if err := json.Unmarshal(enc, &data); err != nil { + return 0, common.Hash{}, err + } + return uint64(data.Slot), data.Block, nil +} + +type HeadEventListener struct { + OnNewHead func(slot uint64, blockRoot common.Hash) + OnSignedHead func(head types.SignedHeader) + OnFinality func(head types.FinalityUpdate) + OnError func(err error) +} + +// StartHeadListener creates an event subscription for heads and signed (optimistic) +// head updates and calls the specified callback functions when they are received. +// The callbacks are also called for the current head and optimistic head at startup. +// They are never called concurrently. +func (api *BeaconLightApi) StartHeadListener(listener HeadEventListener) func() { + closeCh := make(chan struct{}) // initiate closing the stream + closedCh := make(chan struct{}) // stream closed (or failed to create) + stoppedCh := make(chan struct{}) // sync loop stopped + streamCh := make(chan *eventsource.Stream, 1) + go func() { + defer close(closedCh) + // when connected to a Lodestar node the subscription blocks until the + // first actual event arrives; therefore we create the subscription in + // a separate goroutine while letting the main goroutine sync up to the + // current head + req, err := http.NewRequest("GET", api.url+ + "/eth/v1/events?topics=head&topics=light_client_optimistic_update&topics=light_client_finality_update", nil) + if err != nil { + listener.OnError(fmt.Errorf("error creating event subscription request: %v", err)) + return + } + for k, v := range api.customHeaders { + req.Header.Set(k, v) + } + stream, err := eventsource.SubscribeWithRequest("", req) + if err != nil { + listener.OnError(fmt.Errorf("error creating event subscription: %v", err)) + close(streamCh) + return + } + streamCh <- stream + <-closeCh + stream.Close() + }() + + go func() { + defer close(stoppedCh) + + if head, err := api.GetHeader(common.Hash{}); err == nil { + listener.OnNewHead(head.Slot, head.Hash()) + } + if signedHead, err := api.GetOptimisticHeadUpdate(); err == nil { + listener.OnSignedHead(signedHead) + } + if finalityUpdate, err := api.GetFinalityUpdate(); err == nil { + listener.OnFinality(finalityUpdate) + } + stream := <-streamCh + if stream == nil { + return + } + + for { + select { + case event, ok := <-stream.Events: + if !ok { + break + } + switch event.Event() { + case "head": + if slot, blockRoot, err := decodeHeadEvent([]byte(event.Data())); err == nil { + listener.OnNewHead(slot, blockRoot) + } else { + listener.OnError(fmt.Errorf("error decoding head event: %v", err)) + } + case "light_client_optimistic_update": + if signedHead, err := decodeOptimisticHeadUpdate([]byte(event.Data())); err == nil { + listener.OnSignedHead(signedHead) + } else { + listener.OnError(fmt.Errorf("error decoding optimistic update event: %v", err)) + } + case "light_client_finality_update": + if finalityUpdate, err := decodeFinalityUpdate([]byte(event.Data())); err == nil { + listener.OnFinality(finalityUpdate) + } else { + listener.OnError(fmt.Errorf("error decoding finality update event: %v", err)) + } + default: + listener.OnError(fmt.Errorf("unexpected event: %s", event.Event())) + } + case err, ok := <-stream.Errors: + if !ok { + break + } + listener.OnError(err) + } + } + }() + return func() { + close(closeCh) + <-closedCh + <-stoppedCh + } +} diff --git a/beacon/light/committee_chain.go b/beacon/light/committee_chain.go index d707f8cc34..a8d032bb65 100644 --- a/beacon/light/committee_chain.go +++ b/beacon/light/committee_chain.go @@ -70,6 +70,7 @@ type CommitteeChain struct { committees *canonicalStore[*types.SerializedSyncCommittee] fixedCommitteeRoots *canonicalStore[common.Hash] committeeCache *lru.Cache[uint64, syncCommittee] // cache deserialized committees + changeCounter uint64 clock mclock.Clock // monotonic clock (simulated clock in tests) unixNano func() int64 // system clock (simulated clock in tests) @@ -86,6 +87,11 @@ func NewCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signer return newCommitteeChain(db, config, signerThreshold, enforceTime, blsVerifier{}, &mclock.System{}, func() int64 { return time.Now().UnixNano() }) } +// NewTestCommitteeChain creates a new CommitteeChain for testing. +func NewTestCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool, clock *mclock.Simulated) *CommitteeChain { + return newCommitteeChain(db, config, signerThreshold, enforceTime, dummyVerifier{}, clock, func() int64 { return int64(clock.Now()) }) +} + // newCommitteeChain creates a new CommitteeChain with the option of replacing the // clock source and signature verification for testing purposes. func newCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool, sigVerifier committeeSigVerifier, clock mclock.Clock, unixNano func() int64) *CommitteeChain { @@ -181,20 +187,20 @@ func (s *CommitteeChain) Reset() { if err := s.rollback(0); err != nil { log.Error("Error writing batch into chain database", "error", err) } + s.changeCounter++ } -// CheckpointInit initializes a CommitteeChain based on the checkpoint. +// CheckpointInit initializes a CommitteeChain based on a checkpoint. // Note: if the chain is already initialized and the committees proven by the // checkpoint do match the existing chain then the chain is retained and the // new checkpoint becomes fixed. -func (s *CommitteeChain) CheckpointInit(bootstrap *types.BootstrapData) error { +func (s *CommitteeChain) CheckpointInit(bootstrap types.BootstrapData) error { s.chainmu.Lock() defer s.chainmu.Unlock() if err := bootstrap.Validate(); err != nil { return err } - period := bootstrap.Header.SyncPeriod() if err := s.deleteFixedCommitteeRootsFrom(period + 2); err != nil { s.Reset() @@ -215,6 +221,7 @@ func (s *CommitteeChain) CheckpointInit(bootstrap *types.BootstrapData) error { s.Reset() return err } + s.changeCounter++ return nil } @@ -367,6 +374,7 @@ func (s *CommitteeChain) InsertUpdate(update *types.LightClientUpdate, nextCommi return ErrWrongCommitteeRoot } } + s.changeCounter++ if reorg { if err := s.rollback(period + 1); err != nil { return err @@ -405,6 +413,13 @@ func (s *CommitteeChain) NextSyncPeriod() (uint64, bool) { return s.committees.periods.End - 1, true } +func (s *CommitteeChain) ChangeCounter() uint64 { + s.chainmu.RLock() + defer s.chainmu.RUnlock() + + return s.changeCounter +} + // rollback removes all committees and fixed roots from the given period and updates // starting from the previous period. func (s *CommitteeChain) rollback(period uint64) error { @@ -452,12 +467,12 @@ func (s *CommitteeChain) getSyncCommittee(period uint64) (syncCommittee, error) if sc, ok := s.committees.get(s.db, period); ok { c, err := s.sigVerifier.deserializeSyncCommittee(sc) if err != nil { - return nil, fmt.Errorf("Sync committee #%d deserialization error: %v", period, err) + return nil, fmt.Errorf("sync committee #%d deserialization error: %v", period, err) } s.committeeCache.Add(period, c) return c, nil } - return nil, fmt.Errorf("Missing serialized sync committee #%d", period) + return nil, fmt.Errorf("missing serialized sync committee #%d", period) } // VerifySignedHeader returns true if the given signed header has a valid signature diff --git a/beacon/light/committee_chain_test.go b/beacon/light/committee_chain_test.go index 60ea2a0efd..57b6d7175c 100644 --- a/beacon/light/committee_chain_test.go +++ b/beacon/light/committee_chain_test.go @@ -241,12 +241,12 @@ func newCommitteeChainTest(t *testing.T, config types.ChainConfig, signerThresho signerThreshold: signerThreshold, enforceTime: enforceTime, } - c.chain = newCommitteeChain(c.db, &config, signerThreshold, enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) }) + c.chain = NewTestCommitteeChain(c.db, &config, signerThreshold, enforceTime, c.clock) return c } func (c *committeeChainTest) reloadChain() { - c.chain = newCommitteeChain(c.db, &c.config, c.signerThreshold, c.enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) }) + c.chain = NewTestCommitteeChain(c.db, &c.config, c.signerThreshold, c.enforceTime, c.clock) } func (c *committeeChainTest) setClockPeriod(period float64) { diff --git a/beacon/light/head_tracker.go b/beacon/light/head_tracker.go new file mode 100644 index 0000000000..579e1b53da --- /dev/null +++ b/beacon/light/head_tracker.go @@ -0,0 +1,150 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package light + +import ( + "errors" + "sync" + "time" + + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/log" +) + +// HeadTracker keeps track of the latest validated head and the "prefetch" head +// which is the (not necessarily validated) head announced by the majority of +// servers. +type HeadTracker struct { + lock sync.RWMutex + committeeChain *CommitteeChain + minSignerCount int + signedHead types.SignedHeader + hasSignedHead bool + finalityUpdate types.FinalityUpdate + hasFinalityUpdate bool + prefetchHead types.HeadInfo + changeCounter uint64 +} + +// NewHeadTracker creates a new HeadTracker. +func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int) *HeadTracker { + return &HeadTracker{ + committeeChain: committeeChain, + minSignerCount: minSignerCount, + } +} + +// ValidatedHead returns the latest validated head. +func (h *HeadTracker) ValidatedHead() (types.SignedHeader, bool) { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.signedHead, h.hasSignedHead +} + +// ValidatedHead returns the latest validated head. +func (h *HeadTracker) ValidatedFinality() (types.FinalityUpdate, bool) { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.finalityUpdate, h.hasFinalityUpdate +} + +// Validate validates the given signed head. If the head is successfully validated +// and it is better than the old validated head (higher slot or same slot and more +// signers) then ValidatedHead is updated. The boolean return flag signals if +// ValidatedHead has been changed. +func (h *HeadTracker) ValidateHead(head types.SignedHeader) (bool, error) { + h.lock.Lock() + defer h.lock.Unlock() + + replace, err := h.validate(head, h.signedHead) + if replace { + h.signedHead, h.hasSignedHead = head, true + h.changeCounter++ + } + return replace, err +} + +func (h *HeadTracker) ValidateFinality(update types.FinalityUpdate) (bool, error) { + h.lock.Lock() + defer h.lock.Unlock() + + replace, err := h.validate(update.SignedHeader(), h.finalityUpdate.SignedHeader()) + if replace { + h.finalityUpdate, h.hasFinalityUpdate = update, true + h.changeCounter++ + } + return replace, err +} + +func (h *HeadTracker) validate(head, oldHead types.SignedHeader) (bool, error) { + signerCount := head.Signature.SignerCount() + if signerCount < h.minSignerCount { + return false, errors.New("low signer count") + } + if head.Header.Slot < oldHead.Header.Slot || (head.Header.Slot == oldHead.Header.Slot && signerCount <= oldHead.Signature.SignerCount()) { + return false, nil + } + sigOk, age, err := h.committeeChain.VerifySignedHeader(head) + if err != nil { + return false, err + } + if age < 0 { + log.Warn("Future signed head received", "age", age) + } + if age > time.Minute*2 { + log.Warn("Old signed head received", "age", age) + } + if !sigOk { + return false, errors.New("invalid header signature") + } + return true, nil +} + +// PrefetchHead returns the latest known prefetch head's head info. +// This head can be used to start fetching related data hoping that it will be +// validated soon. +// Note that the prefetch head cannot be validated cryptographically so it should +// only be used as a performance optimization hint. +func (h *HeadTracker) PrefetchHead() types.HeadInfo { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.prefetchHead +} + +// SetPrefetchHead sets the prefetch head info. +// Note that HeadTracker does not verify the prefetch head, just acts as a thread +// safe bulletin board. +func (h *HeadTracker) SetPrefetchHead(head types.HeadInfo) { + h.lock.Lock() + defer h.lock.Unlock() + + if head == h.prefetchHead { + return + } + h.prefetchHead = head + h.changeCounter++ +} + +func (h *HeadTracker) ChangeCounter() uint64 { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.changeCounter +} diff --git a/beacon/light/request/scheduler.go b/beacon/light/request/scheduler.go new file mode 100644 index 0000000000..20f811900e --- /dev/null +++ b/beacon/light/request/scheduler.go @@ -0,0 +1,401 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package request + +import ( + "sync" + + "github.com/ethereum/go-ethereum/log" +) + +// Module represents a mechanism which is typically responsible for downloading +// and updating a passive data structure. It does not directly interact with the +// servers. It can start requests using the Requester interface, maintain its +// internal state by receiving and processing Events and update its target data +// structure based on the obtained data. +// It is the Scheduler's responsibility to feed events to the modules, call +// Process as long as there might be something to process and then generate request +// candidates using MakeRequest and start the best possible requests. +// Modules are called by Scheduler whenever a global trigger is fired. All events +// fire the trigger. Changing a target data structure also triggers a next +// processing round as it could make further actions possible either by the same +// or another Module. +type Module interface { + // Process is a non-blocking function responsible for starting requests, + // processing events and updating the target data structures(s) and the + // internal state of the module. Module state typically consists of information + // about pending requests and registered servers. + // Process is always called after an event is received or after a target data + // structure has been changed. + // + // Note: Process functions of different modules are never called concurrently; + // they are called by Scheduler in the same order of priority as they were + // registered in. + Process(Requester, []Event) +} + +// Requester allows Modules to obtain the list of momentarily available servers, +// start new requests and report server failure when a response has been proven +// to be invalid in the processing phase. +// Note that all Requester functions should be safe to call from Module.Process. +type Requester interface { + CanSendTo() []Server + Send(Server, Request) ID + Fail(Server, string) +} + +// Scheduler is a modular network data retrieval framework that coordinates multiple +// servers and retrieval mechanisms (modules). It implements a trigger mechanism +// that calls the Process function of registered modules whenever either the state +// of existing data structures or events coming from registered servers could +// allow new operations. +type Scheduler struct { + lock sync.Mutex + modules []Module // first has highest priority + names map[Module]string + servers map[server]struct{} + targets map[targetData]uint64 + + requesterLock sync.RWMutex + serverOrder []server + pending map[ServerAndID]pendingRequest + + // eventLock guards access to the events list. Note that eventLock can be + // locked either while lock is locked or unlocked but lock cannot be locked + // while eventLock is locked. + eventLock sync.Mutex + events []Event + stopCh chan chan struct{} + + triggerCh chan struct{} // restarts waiting sync loop + // if trigger has already been fired then send to testWaitCh blocks until + // the triggered processing round is finished + testWaitCh chan struct{} +} + +type ( + // Server identifies a server without allowing any direct interaction. + // Note: server interface is used by Scheduler and Tracker but not used by + // the modules that do not interact with them directly. + // In order to make module testing easier, Server interface is used in + // events and modules. + Server any + Request any + Response any + ID uint64 + ServerAndID struct { + Server Server + ID ID + } +) + +// targetData represents a registered target data structure that increases its +// ChangeCounter whenever it has been changed. +type targetData interface { + ChangeCounter() uint64 +} + +// pendingRequest keeps track of sent and not yet finalized requests and their +// sender modules. +type pendingRequest struct { + request Request + module Module +} + +// NewScheduler creates a new Scheduler. +func NewScheduler() *Scheduler { + s := &Scheduler{ + servers: make(map[server]struct{}), + names: make(map[Module]string), + pending: make(map[ServerAndID]pendingRequest), + targets: make(map[targetData]uint64), + stopCh: make(chan chan struct{}), + // Note: testWaitCh should not have capacity in order to ensure + // that after a trigger happens testWaitCh will block until the resulting + // processing round has been finished + triggerCh: make(chan struct{}, 1), + testWaitCh: make(chan struct{}), + } + return s +} + +// RegisterTarget registers a target data structure, ensuring that any changes +// made to it trigger a new round of Module.Process calls, giving a chance to +// modules to react to the changes. +func (s *Scheduler) RegisterTarget(t targetData) { + s.lock.Lock() + defer s.lock.Unlock() + + s.targets[t] = 0 +} + +// RegisterModule registers a module. Should be called before starting the scheduler. +// In each processing round the order of module processing depends on the order of +// registration. +func (s *Scheduler) RegisterModule(m Module, name string) { + s.lock.Lock() + defer s.lock.Unlock() + + s.modules = append(s.modules, m) + s.names[m] = name +} + +// RegisterServer registers a new server. +func (s *Scheduler) RegisterServer(server server) { + s.lock.Lock() + defer s.lock.Unlock() + + s.addEvent(Event{Type: EvRegistered, Server: server}) + server.subscribe(func(event Event) { + event.Server = server + s.addEvent(event) + }) +} + +// UnregisterServer removes a registered server. +func (s *Scheduler) UnregisterServer(server server) { + s.lock.Lock() + defer s.lock.Unlock() + + server.unsubscribe() + s.addEvent(Event{Type: EvUnregistered, Server: server}) +} + +// Start starts the scheduler. It should be called after registering all modules +// and before registering any servers. +func (s *Scheduler) Start() { + go s.syncLoop() +} + +// Stop stops the scheduler. +func (s *Scheduler) Stop() { + stop := make(chan struct{}) + s.stopCh <- stop + <-stop + s.lock.Lock() + for server := range s.servers { + server.unsubscribe() + } + s.servers = nil + s.lock.Unlock() +} + +// syncLoop is the main event loop responsible for event/data processing and +// sending new requests. +// A round of processing starts whenever the global trigger is fired. Triggers +// fired during a processing round ensure that there is going to be a next round. +func (s *Scheduler) syncLoop() { + for { + s.lock.Lock() + s.processRound() + s.lock.Unlock() + loop: + for { + select { + case stop := <-s.stopCh: + close(stop) + return + case <-s.triggerCh: + break loop + case <-s.testWaitCh: + } + } + } +} + +// targetChanged returns true if a registered target data structure has been +// changed since the last call to this function. +func (s *Scheduler) targetChanged() (changed bool) { + for target, counter := range s.targets { + if newCounter := target.ChangeCounter(); newCounter != counter { + s.targets[target] = newCounter + changed = true + } + } + return +} + +// processRound runs an entire processing round. It calls the Process functions +// of all modules, passing all relevant events and repeating Process calls as +// long as any changes have been made to the registered target data structures. +// Once all events have been processed and a stable state has been achieved, +// requests are generated and sent if necessary and possible. +func (s *Scheduler) processRound() { + for { + log.Trace("Processing modules") + filteredEvents := s.filterEvents() + for _, module := range s.modules { + log.Trace("Processing module", "name", s.names[module], "events", len(filteredEvents[module])) + module.Process(requester{s, module}, filteredEvents[module]) + } + if !s.targetChanged() { + break + } + } +} + +// Trigger starts a new processing round. If fired during processing, it ensures +// another full round of processing all modules. +func (s *Scheduler) Trigger() { + select { + case s.triggerCh <- struct{}{}: + default: + } +} + +// addEvent adds an event to be processed in the next round. Note that it can be +// called regardless of the state of the lock mutex, making it safe for use in +// the server event callback. +func (s *Scheduler) addEvent(event Event) { + s.eventLock.Lock() + s.events = append(s.events, event) + s.eventLock.Unlock() + s.Trigger() +} + +// filterEvent sorts each Event either as a request event or a server event, +// depending on its type. Request events are also sorted in a map based on the +// module that originally initiated the request. It also ensures that no events +// related to a server are returned before EvRegistered or after EvUnregistered. +// In case of an EvUnregistered server event it also closes all pending requests +// to the given server by adding a failed request event (EvFail), ensuring that +// all requests get finalized and thereby allowing the module logic to be safe +// and simple. +func (s *Scheduler) filterEvents() map[Module][]Event { + s.eventLock.Lock() + events := s.events + s.events = nil + s.eventLock.Unlock() + + s.requesterLock.Lock() + defer s.requesterLock.Unlock() + + filteredEvents := make(map[Module][]Event) + for _, event := range events { + server := event.Server.(server) + if _, ok := s.servers[server]; !ok && event.Type != EvRegistered { + continue // before EvRegister or after EvUnregister, discard + } + + if event.IsRequestEvent() { + sid, _, _ := event.RequestInfo() + pending, ok := s.pending[sid] + if !ok { + continue // request already closed, ignore further events + } + if event.Type == EvResponse || event.Type == EvFail { + delete(s.pending, sid) // final event, close pending request + } + filteredEvents[pending.module] = append(filteredEvents[pending.module], event) + } else { + switch event.Type { + case EvRegistered: + s.servers[server] = struct{}{} + s.serverOrder = append(s.serverOrder, nil) + copy(s.serverOrder[1:], s.serverOrder[:len(s.serverOrder)-1]) + s.serverOrder[0] = server + case EvUnregistered: + s.closePending(event.Server, filteredEvents) + delete(s.servers, server) + for i, srv := range s.serverOrder { + if srv == server { + copy(s.serverOrder[i:len(s.serverOrder)-1], s.serverOrder[i+1:]) + s.serverOrder = s.serverOrder[:len(s.serverOrder)-1] + break + } + } + } + for _, module := range s.modules { + filteredEvents[module] = append(filteredEvents[module], event) + } + } + } + return filteredEvents +} + +// closePending closes all pending requests to the given server and adds an EvFail +// event to properly finalize them +func (s *Scheduler) closePending(server Server, filteredEvents map[Module][]Event) { + for sid, pending := range s.pending { + if sid.Server == server { + filteredEvents[pending.module] = append(filteredEvents[pending.module], Event{ + Type: EvFail, + Server: server, + Data: RequestResponse{ + ID: sid.ID, + Request: pending.request, + }, + }) + delete(s.pending, sid) + } + } +} + +// requester implements Requester. Note that while requester basically wraps +// Scheduler (with the added information of the currently processed Module), all +// functions are safe to call from Module.Process which is running while +// the Scheduler.lock mutex is held. +type requester struct { + *Scheduler + module Module +} + +// CanSendTo returns the list of currently available servers. It also returns +// them in an order of least to most recently used, ensuring a round-robin usage +// of suitable servers if the module always chooses the first suitable one. +func (s requester) CanSendTo() []Server { + s.requesterLock.RLock() + defer s.requesterLock.RUnlock() + + list := make([]Server, 0, len(s.serverOrder)) + for _, server := range s.serverOrder { + if server.canRequestNow() { + list = append(list, server) + } + } + return list +} + +// Send sends a request and adds an entry to Scheduler.pending map, ensuring that +// related request events will be delivered to the sender Module. +func (s requester) Send(srv Server, req Request) ID { + s.requesterLock.Lock() + defer s.requesterLock.Unlock() + + server := srv.(server) + id := server.sendRequest(req) + sid := ServerAndID{Server: srv, ID: id} + s.pending[sid] = pendingRequest{request: req, module: s.module} + for i, ss := range s.serverOrder { + if ss == server { + copy(s.serverOrder[i:len(s.serverOrder)-1], s.serverOrder[i+1:]) + s.serverOrder[len(s.serverOrder)-1] = server + return id + } + } + log.Error("Target server not found in ordered list of registered servers") + return id +} + +// Fail should be called when a server delivers invalid or useless information. +// Calling Fail disables the given server for a period that is initially short +// but is exponentially growing if it happens frequently. This results in a +// somewhat fault tolerant operation that avoids hammering servers with requests +// that they cannot serve but still gives them a chance periodically. +func (s requester) Fail(srv Server, desc string) { + srv.(server).fail(desc) +} diff --git a/beacon/light/request/scheduler_test.go b/beacon/light/request/scheduler_test.go new file mode 100644 index 0000000000..7d5a567078 --- /dev/null +++ b/beacon/light/request/scheduler_test.go @@ -0,0 +1,122 @@ +package request + +import ( + "reflect" + "testing" +) + +func TestEventFilter(t *testing.T) { + s := NewScheduler() + module1 := &testModule{name: "module1"} + module2 := &testModule{name: "module2"} + s.RegisterModule(module1, "module1") + s.RegisterModule(module2, "module2") + s.Start() + // startup process round without events + s.testWaitCh <- struct{}{} + module1.expProcess(t, nil) + module2.expProcess(t, nil) + srv := &testServer{} + // register server; both modules should receive server event + s.RegisterServer(srv) + s.testWaitCh <- struct{}{} + module1.expProcess(t, []Event{ + {Type: EvRegistered, Server: srv}, + }) + module2.expProcess(t, []Event{ + {Type: EvRegistered, Server: srv}, + }) + // let module1 send a request + srv.canRequest = 1 + module1.sendReq = testRequest + s.Trigger() + // in first triggered round module1 sends the request, no events yet + s.testWaitCh <- struct{}{} + module1.expProcess(t, nil) + module2.expProcess(t, nil) + // server emits EvTimeout; only module1 should receive it + srv.eventCb(Event{Type: EvTimeout, Data: RequestResponse{ID: 1, Request: testRequest}}) + s.testWaitCh <- struct{}{} + module1.expProcess(t, []Event{ + {Type: EvTimeout, Server: srv, Data: RequestResponse{ID: 1, Request: testRequest}}, + }) + module2.expProcess(t, nil) + // unregister server; both modules should receive server event + s.UnregisterServer(srv) + s.testWaitCh <- struct{}{} + module1.expProcess(t, []Event{ + // module1 should also receive EvFail on its pending request + {Type: EvFail, Server: srv, Data: RequestResponse{ID: 1, Request: testRequest}}, + {Type: EvUnregistered, Server: srv}, + }) + module2.expProcess(t, []Event{ + {Type: EvUnregistered, Server: srv}, + }) + // response after server unregistered; should be discarded + srv.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + s.testWaitCh <- struct{}{} + module1.expProcess(t, nil) + module2.expProcess(t, nil) + // no more process rounds expected; shut down + s.testWaitCh <- struct{}{} + module1.expNoMoreProcess(t) + module2.expNoMoreProcess(t) + s.Stop() +} + +type testServer struct { + eventCb func(Event) + lastID ID + canRequest int +} + +func (s *testServer) subscribe(eventCb func(Event)) { + s.eventCb = eventCb +} + +func (s *testServer) canRequestNow() bool { + return s.canRequest > 0 +} + +func (s *testServer) sendRequest(req Request) ID { + s.canRequest-- + s.lastID++ + return s.lastID +} + +func (s *testServer) fail(string) {} +func (s *testServer) unsubscribe() {} + +type testModule struct { + name string + processed [][]Event + sendReq Request +} + +func (m *testModule) Process(requester Requester, events []Event) { + m.processed = append(m.processed, events) + if m.sendReq != nil { + if cs := requester.CanSendTo(); len(cs) > 0 { + requester.Send(cs[0], m.sendReq) + } + } +} + +func (m *testModule) expProcess(t *testing.T, expEvents []Event) { + if len(m.processed) == 0 { + t.Errorf("Missing call to %s.Process", m.name) + return + } + events := m.processed[0] + m.processed = m.processed[1:] + if !reflect.DeepEqual(events, expEvents) { + t.Errorf("Call to %s.Process with wrong events (expected %v, got %v)", m.name, expEvents, events) + } +} + +func (m *testModule) expNoMoreProcess(t *testing.T) { + for len(m.processed) > 0 { + t.Errorf("Unexpected call to %s.Process with events %v", m.name, m.processed[0]) + m.processed = m.processed[1:] + } +} diff --git a/beacon/light/request/server.go b/beacon/light/request/server.go new file mode 100644 index 0000000000..407eb69f49 --- /dev/null +++ b/beacon/light/request/server.go @@ -0,0 +1,439 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package request + +import ( + "math" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/log" +) + +var ( + // request events + EvResponse = &EventType{Name: "response", requestEvent: true} // data: RequestResponse; sent by requestServer + EvFail = &EventType{Name: "fail", requestEvent: true} // data: RequestResponse; sent by requestServer + EvTimeout = &EventType{Name: "timeout", requestEvent: true} // data: RequestResponse; sent by serverWithTimeout + // server events + EvRegistered = &EventType{Name: "registered"} // data: nil; sent by Scheduler + EvUnregistered = &EventType{Name: "unregistered"} // data: nil; sent by Scheduler + EvCanRequestAgain = &EventType{Name: "canRequestAgain"} // data: nil; sent by serverWithLimits +) + +const ( + softRequestTimeout = time.Second // allow resending request to a different server but do not cancel yet + hardRequestTimeout = time.Second * 10 // cancel request +) + +const ( + // serverWithLimits parameters + parallelAdjustUp = 0.1 // adjust parallelLimit up in case of success under full load + parallelAdjustDown = 1 // adjust parallelLimit down in case of timeout/failure + minParallelLimit = 1 // parallelLimit lower bound + defaultParallelLimit = 3 // parallelLimit initial value + minFailureDelay = time.Millisecond * 100 // minimum disable time in case of request failure + maxFailureDelay = time.Minute // maximum disable time in case of request failure + maxServerEventBuffer = 5 // server event allowance buffer limit + maxServerEventRate = time.Second // server event allowance buffer recharge rate +) + +// requestServer can send requests in a non-blocking way and feed back events +// through the event callback. After each request it should send back either +// EvResponse or EvFail. Additionally, it may also send application-defined +// events that the Modules can interpret. +type requestServer interface { + Subscribe(eventCallback func(Event)) + SendRequest(ID, Request) + Unsubscribe() +} + +// server is implemented by a requestServer wrapped into serverWithTimeout and +// serverWithLimits and is used by Scheduler. +// In addition to requestServer functionality, server can also handle timeouts, +// limit the number of parallel in-flight requests and temporarily disable +// new requests based on timeouts and response failures. +type server interface { + subscribe(eventCallback func(Event)) + canRequestNow() bool + sendRequest(Request) ID + fail(string) + unsubscribe() +} + +// NewServer wraps a requestServer and returns a server +func NewServer(rs requestServer, clock mclock.Clock) server { + s := &serverWithLimits{} + s.parent = rs + s.serverWithTimeout.init(clock) + s.init() + return s +} + +// EventType identifies an event type, either related to a request or the server +// in general. Server events can also be externally defined. +type EventType struct { + Name string + requestEvent bool // all request events are pre-defined in request package +} + +// Event describes an event where the type of Data depends on Type. +// Server field is not required when sent through the event callback; it is filled +// out when processed by the Scheduler. Note that the Scheduler can also create +// and send events (EvRegistered, EvUnregistered) directly. +type Event struct { + Type *EventType + Server Server // filled by Scheduler + Data any +} + +// IsRequestEvent returns true if the event is a request event +func (e *Event) IsRequestEvent() bool { + return e.Type.requestEvent +} + +// RequestInfo assumes that the event is a request event and returns its contents +// in a convenient form. +func (e *Event) RequestInfo() (ServerAndID, Request, Response) { + data := e.Data.(RequestResponse) + return ServerAndID{Server: e.Server, ID: data.ID}, data.Request, data.Response +} + +// RequestResponse is the Data type of request events. +type RequestResponse struct { + ID ID + Request Request + Response Response +} + +// serverWithTimeout wraps a requestServer and introduces timeouts. +// The request's lifecycle is concluded if EvResponse or EvFail emitted by the +// parent requestServer. If this does not happen until softRequestTimeout then +// EvTimeout is emitted, after which the final EvResponse or EvFail is still +// guaranteed to follow. +// If the parent fails to send this final event for hardRequestTimeout then +// serverWithTimeout emits EvFail and discards any further events from the +// parent related to the given request. +type serverWithTimeout struct { + parent requestServer + lock sync.Mutex + clock mclock.Clock + childEventCb func(event Event) + timeouts map[ID]mclock.Timer + lastID ID +} + +// init initializes serverWithTimeout +func (s *serverWithTimeout) init(clock mclock.Clock) { + s.clock = clock + s.timeouts = make(map[ID]mclock.Timer) +} + +// subscribe subscribes to events which include parent (requestServer) events +// plus EvTimeout. +func (s *serverWithTimeout) subscribe(eventCallback func(event Event)) { + s.lock.Lock() + defer s.lock.Unlock() + + s.childEventCb = eventCallback + s.parent.Subscribe(s.eventCallback) +} + +// sendRequest generated a new request ID, emits EvRequest, sets up the timeout +// timer, then sends the request through the parent (requestServer). +func (s *serverWithTimeout) sendRequest(request Request) (reqId ID) { + s.lock.Lock() + s.lastID++ + id := s.lastID + s.startTimeout(RequestResponse{ID: id, Request: request}) + s.lock.Unlock() + s.parent.SendRequest(id, request) + return id +} + +// eventCallback is called by parent (requestServer) event subscription. +func (s *serverWithTimeout) eventCallback(event Event) { + s.lock.Lock() + defer s.lock.Unlock() + + switch event.Type { + case EvResponse, EvFail: + id := event.Data.(RequestResponse).ID + if timer, ok := s.timeouts[id]; ok { + // Note: if stopping the timer is unsuccessful then the resulting AfterFunc + // call will just do nothing + timer.Stop() + delete(s.timeouts, id) + s.childEventCb(event) + } + default: + s.childEventCb(event) + } +} + +// startTimeout starts a timeout timer for the given request. +func (s *serverWithTimeout) startTimeout(reqData RequestResponse) { + id := reqData.ID + s.timeouts[id] = s.clock.AfterFunc(softRequestTimeout, func() { + s.lock.Lock() + if _, ok := s.timeouts[id]; !ok { + s.lock.Unlock() + return + } + s.timeouts[id] = s.clock.AfterFunc(hardRequestTimeout-softRequestTimeout, func() { + s.lock.Lock() + if _, ok := s.timeouts[id]; !ok { + s.lock.Unlock() + return + } + delete(s.timeouts, id) + childEventCb := s.childEventCb + s.lock.Unlock() + childEventCb(Event{Type: EvFail, Data: reqData}) + }) + childEventCb := s.childEventCb + s.lock.Unlock() + childEventCb(Event{Type: EvTimeout, Data: reqData}) + }) +} + +// stop stops all goroutines associated with the server. +func (s *serverWithTimeout) unsubscribe() { + s.lock.Lock() + defer s.lock.Unlock() + + for _, timer := range s.timeouts { + if timer != nil { + timer.Stop() + } + } + s.childEventCb = nil + s.parent.Unsubscribe() +} + +// serverWithLimits wraps serverWithTimeout and implements server. It limits the +// number of parallel in-flight requests and prevents sending new requests when a +// pending one has already timed out. Server events are also rate limited. +// It also implements a failure delay mechanism that adds an exponentially growing +// delay each time a request fails (wrong answer or hard timeout). This makes the +// syncing mechanism less brittle as temporary failures of the server might happen +// sometimes, but still avoids hammering a non-functional server with requests. +type serverWithLimits struct { + serverWithTimeout + lock sync.Mutex + childEventCb func(event Event) + softTimeouts map[ID]struct{} + pendingCount, timeoutCount int + parallelLimit float32 + sendEvent bool + delayTimer mclock.Timer + delayCounter int + failureDelayEnd mclock.AbsTime + failureDelay float64 + serverEventBuffer int + eventBufferUpdated mclock.AbsTime +} + +// init initializes serverWithLimits +func (s *serverWithLimits) init() { + s.softTimeouts = make(map[ID]struct{}) + s.parallelLimit = defaultParallelLimit + s.serverEventBuffer = maxServerEventBuffer +} + +// subscribe subscribes to events which include parent (serverWithTimeout) events +// plus EvCanRequestAgain. +func (s *serverWithLimits) subscribe(eventCallback func(event Event)) { + s.lock.Lock() + defer s.lock.Unlock() + + s.childEventCb = eventCallback + s.serverWithTimeout.subscribe(s.eventCallback) +} + +// eventCallback is called by parent (serverWithTimeout) event subscription. +func (s *serverWithLimits) eventCallback(event Event) { + s.lock.Lock() + var sendCanRequestAgain bool + passEvent := true + switch event.Type { + case EvTimeout: + id := event.Data.(RequestResponse).ID + s.softTimeouts[id] = struct{}{} + s.timeoutCount++ + s.parallelLimit -= parallelAdjustDown + if s.parallelLimit < minParallelLimit { + s.parallelLimit = minParallelLimit + } + log.Debug("Server timeout", "count", s.timeoutCount, "parallelLimit", s.parallelLimit) + case EvResponse, EvFail: + id := event.Data.(RequestResponse).ID + if _, ok := s.softTimeouts[id]; ok { + delete(s.softTimeouts, id) + s.timeoutCount-- + log.Debug("Server timeout finalized", "count", s.timeoutCount, "parallelLimit", s.parallelLimit) + } + if event.Type == EvResponse && s.pendingCount >= int(s.parallelLimit) { + s.parallelLimit += parallelAdjustUp + } + s.pendingCount-- + if s.canRequest() { + sendCanRequestAgain = s.sendEvent + s.sendEvent = false + } + if event.Type == EvFail { + s.failLocked("failed request") + } + default: + // server event; check rate limit + if s.serverEventBuffer < maxServerEventBuffer { + now := s.clock.Now() + sinceUpdate := time.Duration(now - s.eventBufferUpdated) + if sinceUpdate >= maxServerEventRate*time.Duration(maxServerEventBuffer-s.serverEventBuffer) { + s.serverEventBuffer = maxServerEventBuffer + s.eventBufferUpdated = now + } else { + addBuffer := int(sinceUpdate / maxServerEventRate) + s.serverEventBuffer += addBuffer + s.eventBufferUpdated += mclock.AbsTime(maxServerEventRate * time.Duration(addBuffer)) + } + } + if s.serverEventBuffer > 0 { + s.serverEventBuffer-- + } else { + passEvent = false + } + } + childEventCb := s.childEventCb + s.lock.Unlock() + if passEvent { + childEventCb(event) + } + if sendCanRequestAgain { + childEventCb(Event{Type: EvCanRequestAgain}) + } +} + +// sendRequest sends a request through the parent (serverWithTimeout). +func (s *serverWithLimits) sendRequest(request Request) (reqId ID) { + s.lock.Lock() + s.pendingCount++ + s.lock.Unlock() + return s.serverWithTimeout.sendRequest(request) +} + +// stop stops all goroutines associated with the server. +func (s *serverWithLimits) unsubscribe() { + s.lock.Lock() + defer s.lock.Unlock() + + if s.delayTimer != nil { + s.delayTimer.Stop() + s.delayTimer = nil + } + s.childEventCb = nil + s.serverWithTimeout.unsubscribe() +} + +// canRequest checks whether a new request can be started. +func (s *serverWithLimits) canRequest() bool { + if s.delayTimer != nil || s.pendingCount >= int(s.parallelLimit) || s.timeoutCount > 0 { + return false + } + if s.parallelLimit < minParallelLimit { + s.parallelLimit = minParallelLimit + } + return true +} + +// canRequestNow checks whether a new request can be started, according to the +// current in-flight request count and parallelLimit, and also the failure delay +// timer. +// If it returns false then it is guaranteed that an EvCanRequestAgain will be +// sent whenever the server becomes available for requesting again. +func (s *serverWithLimits) canRequestNow() bool { + var sendCanRequestAgain bool + s.lock.Lock() + canRequest := s.canRequest() + if canRequest { + sendCanRequestAgain = s.sendEvent + s.sendEvent = false + } + childEventCb := s.childEventCb + s.lock.Unlock() + if sendCanRequestAgain { + childEventCb(Event{Type: EvCanRequestAgain}) + } + return canRequest +} + +// delay sets the delay timer to the given duration, disabling new requests for +// the given period. +func (s *serverWithLimits) delay(delay time.Duration) { + if s.delayTimer != nil { + // Note: if stopping the timer is unsuccessful then the resulting AfterFunc + // call will just do nothing + s.delayTimer.Stop() + s.delayTimer = nil + } + + s.delayCounter++ + delayCounter := s.delayCounter + log.Debug("Server delay started", "length", delay) + s.delayTimer = s.clock.AfterFunc(delay, func() { + log.Debug("Server delay ended", "length", delay) + var sendCanRequestAgain bool + s.lock.Lock() + if s.delayTimer != nil && s.delayCounter == delayCounter { // do nothing if there is a new timer now + s.delayTimer = nil + if s.canRequest() { + sendCanRequestAgain = s.sendEvent + s.sendEvent = false + } + } + childEventCb := s.childEventCb + s.lock.Unlock() + if sendCanRequestAgain { + childEventCb(Event{Type: EvCanRequestAgain}) + } + }) +} + +// fail reports that a response from the server was found invalid by the processing +// Module, disabling new requests for a dynamically adjusted time period. +func (s *serverWithLimits) fail(desc string) { + s.lock.Lock() + defer s.lock.Unlock() + + s.failLocked(desc) +} + +// failLocked calculates the dynamic failure delay and applies it. +func (s *serverWithLimits) failLocked(desc string) { + log.Debug("Server error", "description", desc) + s.failureDelay *= 2 + now := s.clock.Now() + if now > s.failureDelayEnd { + s.failureDelay *= math.Pow(2, -float64(now-s.failureDelayEnd)/float64(maxFailureDelay)) + } + if s.failureDelay < float64(minFailureDelay) { + s.failureDelay = float64(minFailureDelay) + } + s.failureDelayEnd = now + mclock.AbsTime(s.failureDelay) + s.delay(time.Duration(s.failureDelay)) +} diff --git a/beacon/light/request/server_test.go b/beacon/light/request/server_test.go new file mode 100644 index 0000000000..b6b9edf9a0 --- /dev/null +++ b/beacon/light/request/server_test.go @@ -0,0 +1,158 @@ +package request + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common/mclock" +) + +const ( + testRequest = "Life, the Universe, and Everything" + testResponse = 42 +) + +var testEventType = &EventType{Name: "testEvent"} + +func TestServerEvents(t *testing.T) { + rs := &testRequestServer{} + clock := &mclock.Simulated{} + srv := NewServer(rs, clock) + var lastEventType *EventType + srv.subscribe(func(event Event) { lastEventType = event.Type }) + evTypeName := func(evType *EventType) string { + if evType == nil { + return "none" + } + return evType.Name + } + expEvent := func(expType *EventType) { + if lastEventType != expType { + t.Errorf("Wrong event type (expected %s, got %s)", evTypeName(expType), evTypeName(lastEventType)) + } + lastEventType = nil + } + // user events should simply be passed through + rs.eventCb(Event{Type: testEventType}) + expEvent(testEventType) + // send request, soft timeout, then valid response + srv.sendRequest(testRequest) + clock.WaitForTimers(1) + clock.Run(softRequestTimeout) + expEvent(EvTimeout) + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expEvent(EvResponse) + // send request, hard timeout (response after hard timeout should be ignored) + srv.sendRequest(testRequest) + clock.WaitForTimers(1) + clock.Run(softRequestTimeout) + expEvent(EvTimeout) + clock.WaitForTimers(1) + clock.Run(hardRequestTimeout) + expEvent(EvFail) + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expEvent(nil) +} + +func TestServerParallel(t *testing.T) { + rs := &testRequestServer{} + srv := NewServer(rs, &mclock.Simulated{}) + srv.subscribe(func(event Event) {}) + + expSend := func(expSent int) { + var sent int + for sent <= expSent { + if !srv.canRequestNow() { + break + } + sent++ + srv.sendRequest(testRequest) + } + if sent != expSent { + t.Errorf("Wrong number of parallel requests accepted (expected %d, got %d)", expSent, sent) + } + } + // max out parallel allowance + expSend(defaultParallelLimit) + // 1 answered, should accept 1 more + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expSend(1) + // 2 answered, should accept 2 more + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 2, Request: testRequest, Response: testResponse}}) + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 3, Request: testRequest, Response: testResponse}}) + expSend(2) + // failed request, should decrease allowance and not accept more + rs.eventCb(Event{Type: EvFail, Data: RequestResponse{ID: 4, Request: testRequest}}) + expSend(0) + srv.unsubscribe() +} + +func TestServerFail(t *testing.T) { + rs := &testRequestServer{} + clock := &mclock.Simulated{} + srv := NewServer(rs, clock) + srv.subscribe(func(event Event) {}) + expCanRequest := func(expCanRequest bool) { + if canRequest := srv.canRequestNow(); canRequest != expCanRequest { + t.Errorf("Wrong result for canRequestNow (expected %v, got %v)", expCanRequest, canRequest) + } + } + // timed out request + expCanRequest(true) + srv.sendRequest(testRequest) + clock.WaitForTimers(1) + expCanRequest(true) + clock.Run(softRequestTimeout) + expCanRequest(false) // cannot request when there is a timed out request + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expCanRequest(true) + // explicit server.Fail + srv.fail("") + clock.WaitForTimers(1) + expCanRequest(false) // cannot request for a while after a failure + clock.Run(minFailureDelay) + expCanRequest(true) + // request returned with EvFail + srv.sendRequest(testRequest) + rs.eventCb(Event{Type: EvFail, Data: RequestResponse{ID: 2, Request: testRequest}}) + clock.WaitForTimers(1) + expCanRequest(false) // EvFail should also start failure delay + clock.Run(minFailureDelay) + expCanRequest(false) // second failure delay is longer, should still be disabled + clock.Run(minFailureDelay) + expCanRequest(true) + srv.unsubscribe() +} + +func TestServerEventRateLimit(t *testing.T) { + rs := &testRequestServer{} + clock := &mclock.Simulated{} + srv := NewServer(rs, clock) + var eventCount int + srv.subscribe(func(event Event) { + if !event.IsRequestEvent() { + eventCount++ + } + }) + expEvents := func(send, expAllowed int) { + eventCount = 0 + for sent := 0; sent < send; sent++ { + rs.eventCb(Event{Type: testEventType}) + } + if eventCount != expAllowed { + t.Errorf("Wrong number of server events passing rate limitation (sent %d, expected %d, got %d)", send, expAllowed, eventCount) + } + } + expEvents(maxServerEventBuffer+5, maxServerEventBuffer) + clock.Run(maxServerEventRate) + expEvents(5, 1) + clock.Run(maxServerEventRate * maxServerEventBuffer * 2) + expEvents(maxServerEventBuffer+5, maxServerEventBuffer) +} + +type testRequestServer struct { + eventCb func(Event) +} + +func (rs *testRequestServer) Subscribe(eventCb func(Event)) { rs.eventCb = eventCb } +func (rs *testRequestServer) SendRequest(ID, Request) {} +func (rs *testRequestServer) Unsubscribe() {} diff --git a/beacon/light/sync/head_sync.go b/beacon/light/sync/head_sync.go new file mode 100644 index 0000000000..9fef95b0df --- /dev/null +++ b/beacon/light/sync/head_sync.go @@ -0,0 +1,176 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" +) + +type headTracker interface { + ValidateHead(head types.SignedHeader) (bool, error) + ValidateFinality(head types.FinalityUpdate) (bool, error) + SetPrefetchHead(head types.HeadInfo) +} + +// HeadSync implements request.Module; it updates the validated and prefetch +// heads of HeadTracker based on the EvHead and EvSignedHead events coming from +// registered servers. +// It can also postpone the validation of the latest announced signed head +// until the committee chain is synced up to at least the required period. +type HeadSync struct { + headTracker headTracker + chain committeeChain + nextSyncPeriod uint64 + chainInit bool + unvalidatedHeads map[request.Server]types.SignedHeader + unvalidatedFinality map[request.Server]types.FinalityUpdate + serverHeads map[request.Server]types.HeadInfo + headServerCount map[types.HeadInfo]headServerCount + headCounter uint64 + prefetchHead types.HeadInfo +} + +// headServerCount is associated with most recently seen head infos; it counts +// the number of servers currently having the given head info as their announced +// head and a counter signaling how recent that head is. +// This data is used for selecting the prefetch head. +type headServerCount struct { + serverCount int + headCounter uint64 +} + +// NewHeadSync creates a new HeadSync. +func NewHeadSync(headTracker headTracker, chain committeeChain) *HeadSync { + s := &HeadSync{ + headTracker: headTracker, + chain: chain, + unvalidatedHeads: make(map[request.Server]types.SignedHeader), + unvalidatedFinality: make(map[request.Server]types.FinalityUpdate), + serverHeads: make(map[request.Server]types.HeadInfo), + headServerCount: make(map[types.HeadInfo]headServerCount), + } + return s +} + +// Process implements request.Module. +func (s *HeadSync) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + switch event.Type { + case EvNewHead: + s.setServerHead(event.Server, event.Data.(types.HeadInfo)) + case EvNewSignedHead: + s.newSignedHead(event.Server, event.Data.(types.SignedHeader)) + case EvNewFinalityUpdate: + s.newFinalityUpdate(event.Server, event.Data.(types.FinalityUpdate)) + case request.EvUnregistered: + s.setServerHead(event.Server, types.HeadInfo{}) + delete(s.serverHeads, event.Server) + delete(s.unvalidatedHeads, event.Server) + } + } + + nextPeriod, chainInit := s.chain.NextSyncPeriod() + if nextPeriod != s.nextSyncPeriod || chainInit != s.chainInit { + s.nextSyncPeriod, s.chainInit = nextPeriod, chainInit + s.processUnvalidated() + } +} + +// newSignedHead handles received signed head; either validates it if the chain +// is properly synced or stores it for further validation. +func (s *HeadSync) newSignedHead(server request.Server, signedHead types.SignedHeader) { + if !s.chainInit || types.SyncPeriod(signedHead.SignatureSlot) > s.nextSyncPeriod { + s.unvalidatedHeads[server] = signedHead + return + } + s.headTracker.ValidateHead(signedHead) +} + +// newSignedHead handles received signed head; either validates it if the chain +// is properly synced or stores it for further validation. +func (s *HeadSync) newFinalityUpdate(server request.Server, finalityUpdate types.FinalityUpdate) { + if !s.chainInit || types.SyncPeriod(finalityUpdate.SignatureSlot) > s.nextSyncPeriod { + s.unvalidatedFinality[server] = finalityUpdate + return + } + s.headTracker.ValidateFinality(finalityUpdate) +} + +// processUnvalidatedHeads iterates the list of unvalidated heads and validates +// those which can be validated. +func (s *HeadSync) processUnvalidated() { + if !s.chainInit { + return + } + for server, signedHead := range s.unvalidatedHeads { + if types.SyncPeriod(signedHead.SignatureSlot) <= s.nextSyncPeriod { + s.headTracker.ValidateHead(signedHead) + delete(s.unvalidatedHeads, server) + } + } + for server, finalityUpdate := range s.unvalidatedFinality { + if types.SyncPeriod(finalityUpdate.SignatureSlot) <= s.nextSyncPeriod { + s.headTracker.ValidateFinality(finalityUpdate) + delete(s.unvalidatedFinality, server) + } + } +} + +// setServerHead processes non-validated server head announcements and updates +// the prefetch head if necessary. +func (s *HeadSync) setServerHead(server request.Server, head types.HeadInfo) bool { + if oldHead, ok := s.serverHeads[server]; ok { + if head == oldHead { + return false + } + h := s.headServerCount[oldHead] + if h.serverCount--; h.serverCount > 0 { + s.headServerCount[oldHead] = h + } else { + delete(s.headServerCount, oldHead) + } + } + if head != (types.HeadInfo{}) { + h, ok := s.headServerCount[head] + if !ok { + s.headCounter++ + h.headCounter = s.headCounter + } + h.serverCount++ + s.headServerCount[head] = h + s.serverHeads[server] = head + } else { + delete(s.serverHeads, server) + } + var ( + bestHead types.HeadInfo + bestHeadInfo headServerCount + ) + for head, headServerCount := range s.headServerCount { + if headServerCount.serverCount > bestHeadInfo.serverCount || + (headServerCount.serverCount == bestHeadInfo.serverCount && headServerCount.headCounter > bestHeadInfo.headCounter) { + bestHead, bestHeadInfo = head, headServerCount + } + } + if bestHead == s.prefetchHead { + return false + } + s.prefetchHead = bestHead + s.headTracker.SetPrefetchHead(bestHead) + return true +} diff --git a/beacon/light/sync/head_sync_test.go b/beacon/light/sync/head_sync_test.go new file mode 100644 index 0000000000..12faad6292 --- /dev/null +++ b/beacon/light/sync/head_sync_test.go @@ -0,0 +1,151 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "testing" + + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" +) + +var ( + testServer1 = "testServer1" + testServer2 = "testServer2" + testServer3 = "testServer3" + testServer4 = "testServer4" + + testHead0 = types.HeadInfo{} + testHead1 = types.HeadInfo{Slot: 123, BlockRoot: common.Hash{1}} + testHead2 = types.HeadInfo{Slot: 124, BlockRoot: common.Hash{2}} + testHead3 = types.HeadInfo{Slot: 124, BlockRoot: common.Hash{3}} + testHead4 = types.HeadInfo{Slot: 125, BlockRoot: common.Hash{4}} + + testSHead1 = types.SignedHeader{SignatureSlot: 0x0124, Header: types.Header{Slot: 0x0123, StateRoot: common.Hash{1}}} + testSHead2 = types.SignedHeader{SignatureSlot: 0x2010, Header: types.Header{Slot: 0x200e, StateRoot: common.Hash{2}}} + // testSHead3 is at the end of period 1 but signed in period 2 + testSHead3 = types.SignedHeader{SignatureSlot: 0x4000, Header: types.Header{Slot: 0x3fff, StateRoot: common.Hash{3}}} + testSHead4 = types.SignedHeader{SignatureSlot: 0x6444, Header: types.Header{Slot: 0x6443, StateRoot: common.Hash{4}}} +) + +func TestValidatedHead(t *testing.T) { + chain := &TestCommitteeChain{} + ht := &TestHeadTracker{} + headSync := NewHeadSync(ht, chain) + ts := NewTestScheduler(t, headSync) + + ht.ExpValidated(t, 0, nil) + + ts.AddServer(testServer1, 1) + ts.ServerEvent(EvNewSignedHead, testServer1, testSHead1) + ts.Run(1) + // announced head should be queued because of uninitialized chain + ht.ExpValidated(t, 1, nil) + + chain.SetNextSyncPeriod(0) // initialize chain + ts.Run(2) + // expect previously queued head to be validated + ht.ExpValidated(t, 2, []types.SignedHeader{testSHead1}) + + chain.SetNextSyncPeriod(1) + ts.ServerEvent(EvNewSignedHead, testServer1, testSHead2) + ts.AddServer(testServer2, 1) + ts.ServerEvent(EvNewSignedHead, testServer2, testSHead2) + ts.Run(3) + // expect both head announcements to be validated instantly + ht.ExpValidated(t, 3, []types.SignedHeader{testSHead2, testSHead2}) + + ts.ServerEvent(EvNewSignedHead, testServer1, testSHead3) + ts.AddServer(testServer3, 1) + ts.ServerEvent(EvNewSignedHead, testServer3, testSHead4) + ts.Run(4) + // future period annonced heads should be queued + ht.ExpValidated(t, 4, nil) + + chain.SetNextSyncPeriod(2) + ts.Run(5) + // testSHead3 can be validated now but not testSHead4 + ht.ExpValidated(t, 5, []types.SignedHeader{testSHead3}) + + // server 3 disconnected without proving period 3, its announced head should be dropped + ts.RemoveServer(testServer3) + ts.Run(6) + ht.ExpValidated(t, 6, nil) + + chain.SetNextSyncPeriod(3) + ts.Run(7) + // testSHead4 could be validated now but it's not queued by any registered server + ht.ExpValidated(t, 7, nil) + + ts.ServerEvent(EvNewSignedHead, testServer2, testSHead4) + ts.Run(8) + // now testSHead4 should be validated + ht.ExpValidated(t, 8, []types.SignedHeader{testSHead4}) +} + +func TestPrefetchHead(t *testing.T) { + chain := &TestCommitteeChain{} + ht := &TestHeadTracker{} + headSync := NewHeadSync(ht, chain) + ts := NewTestScheduler(t, headSync) + + ht.ExpPrefetch(t, 0, testHead0) // no servers registered + + ts.AddServer(testServer1, 1) + ts.ServerEvent(EvNewHead, testServer1, testHead1) + ts.Run(1) + ht.ExpPrefetch(t, 1, testHead1) // s1: h1 + + ts.AddServer(testServer2, 1) + ts.ServerEvent(EvNewHead, testServer2, testHead2) + ts.Run(2) + ht.ExpPrefetch(t, 2, testHead2) // s1: h1, s2: h2 + + ts.ServerEvent(EvNewHead, testServer1, testHead2) + ts.Run(3) + ht.ExpPrefetch(t, 3, testHead2) // s1: h2, s2: h2 + + ts.AddServer(testServer3, 1) + ts.ServerEvent(EvNewHead, testServer3, testHead3) + ts.Run(4) + ht.ExpPrefetch(t, 4, testHead2) // s1: h2, s2: h2, s3: h3 + + ts.AddServer(testServer4, 1) + ts.ServerEvent(EvNewHead, testServer4, testHead4) + ts.Run(5) + ht.ExpPrefetch(t, 5, testHead2) // s1: h2, s2: h2, s3: h3, s4: h4 + + ts.ServerEvent(EvNewHead, testServer2, testHead3) + ts.Run(6) + ht.ExpPrefetch(t, 6, testHead3) // s1: h2, s2: h3, s3: h3, s4: h4 + + ts.RemoveServer(testServer3) + ts.Run(7) + ht.ExpPrefetch(t, 7, testHead4) // s1: h2, s2: h3, s4: h4 + + ts.RemoveServer(testServer1) + ts.Run(8) + ht.ExpPrefetch(t, 8, testHead4) // s2: h3, s4: h4 + + ts.RemoveServer(testServer4) + ts.Run(9) + ht.ExpPrefetch(t, 9, testHead3) // s2: h3 + + ts.RemoveServer(testServer2) + ts.Run(10) + ht.ExpPrefetch(t, 10, testHead0) // no servers registered +} diff --git a/beacon/light/sync/test_helpers.go b/beacon/light/sync/test_helpers.go new file mode 100644 index 0000000000..a1ca2b5909 --- /dev/null +++ b/beacon/light/sync/test_helpers.go @@ -0,0 +1,254 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/beacon/light" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" +) + +type requestWithID struct { + sid request.ServerAndID + request request.Request +} + +type TestScheduler struct { + t *testing.T + module request.Module + events []request.Event + servers []request.Server + allowance map[request.Server]int + sent map[int][]requestWithID + testIndex int + expFail map[request.Server]int // expected Server.Fail calls during next Run + lastId request.ID +} + +func NewTestScheduler(t *testing.T, module request.Module) *TestScheduler { + return &TestScheduler{ + t: t, + module: module, + allowance: make(map[request.Server]int), + expFail: make(map[request.Server]int), + sent: make(map[int][]requestWithID), + } +} + +func (ts *TestScheduler) Run(testIndex int, exp ...any) { + expReqs := make([]requestWithID, len(exp)/2) + id := ts.lastId + for i := range expReqs { + id++ + expReqs[i] = requestWithID{ + sid: request.ServerAndID{Server: exp[i*2].(request.Server), ID: id}, + request: exp[i*2+1].(request.Request), + } + } + if len(expReqs) == 0 { + expReqs = nil + } + + ts.testIndex = testIndex + ts.module.Process(ts, ts.events) + ts.events = nil + + for server, count := range ts.expFail { + delete(ts.expFail, server) + if count == 0 { + continue + } + ts.t.Errorf("Missing %d Server.Fail(s) from server %s in test case #%d", count, server.(string), testIndex) + } + + if !reflect.DeepEqual(ts.sent[testIndex], expReqs) { + ts.t.Errorf("Wrong sent requests in test case #%d (expected %v, got %v)", testIndex, expReqs, ts.sent[testIndex]) + } +} + +func (ts *TestScheduler) CanSendTo() (cs []request.Server) { + for _, server := range ts.servers { + if ts.allowance[server] > 0 { + cs = append(cs, server) + } + } + return +} + +func (ts *TestScheduler) Send(server request.Server, req request.Request) request.ID { + ts.lastId++ + ts.sent[ts.testIndex] = append(ts.sent[ts.testIndex], requestWithID{ + sid: request.ServerAndID{Server: server, ID: ts.lastId}, + request: req, + }) + ts.allowance[server]-- + return ts.lastId +} + +func (ts *TestScheduler) Fail(server request.Server, desc string) { + if ts.expFail[server] == 0 { + ts.t.Errorf("Unexpected Fail from server %s in test case #%d: %s", server.(string), ts.testIndex, desc) + return + } + ts.expFail[server]-- +} + +func (ts *TestScheduler) Request(testIndex, reqIndex int) requestWithID { + if len(ts.sent[testIndex]) < reqIndex { + ts.t.Errorf("Missing request from test case %d index %d", testIndex, reqIndex) + return requestWithID{} + } + return ts.sent[testIndex][reqIndex-1] +} + +func (ts *TestScheduler) ServerEvent(evType *request.EventType, server request.Server, data any) { + ts.events = append(ts.events, request.Event{ + Type: evType, + Server: server, + Data: data, + }) +} + +func (ts *TestScheduler) RequestEvent(evType *request.EventType, req requestWithID, resp request.Response) { + if req.request == nil { + return + } + ts.events = append(ts.events, request.Event{ + Type: evType, + Server: req.sid.Server, + Data: request.RequestResponse{ + ID: req.sid.ID, + Request: req.request, + Response: resp, + }, + }) +} + +func (ts *TestScheduler) AddServer(server request.Server, allowance int) { + ts.servers = append(ts.servers, server) + ts.allowance[server] = allowance + ts.ServerEvent(request.EvRegistered, server, nil) +} + +func (ts *TestScheduler) RemoveServer(server request.Server) { + ts.servers = append(ts.servers, server) + for i, s := range ts.servers { + if s == server { + copy(ts.servers[i:len(ts.servers)-1], ts.servers[i+1:]) + ts.servers = ts.servers[:len(ts.servers)-1] + break + } + } + delete(ts.allowance, server) + ts.ServerEvent(request.EvUnregistered, server, nil) +} + +func (ts *TestScheduler) AddAllowance(server request.Server, allowance int) { + ts.allowance[server] += allowance +} + +func (ts *TestScheduler) ExpFail(server request.Server) { + ts.expFail[server]++ +} + +type TestCommitteeChain struct { + fsp, nsp uint64 + init bool +} + +func (t *TestCommitteeChain) CheckpointInit(bootstrap types.BootstrapData) error { + t.fsp, t.nsp, t.init = bootstrap.Header.SyncPeriod(), bootstrap.Header.SyncPeriod()+2, true + return nil +} + +func (t *TestCommitteeChain) InsertUpdate(update *types.LightClientUpdate, nextCommittee *types.SerializedSyncCommittee) error { + period := update.AttestedHeader.Header.SyncPeriod() + if period < t.fsp || period > t.nsp || !t.init { + return light.ErrInvalidPeriod + } + if period == t.nsp { + t.nsp++ + } + return nil +} + +func (t *TestCommitteeChain) NextSyncPeriod() (uint64, bool) { + return t.nsp, t.init +} + +func (tc *TestCommitteeChain) ExpInit(t *testing.T, ExpInit bool) { + if tc.init != ExpInit { + t.Errorf("Incorrect init flag (expected %v, got %v)", ExpInit, tc.init) + } +} + +func (t *TestCommitteeChain) SetNextSyncPeriod(nsp uint64) { + t.init, t.nsp = true, nsp +} + +func (tc *TestCommitteeChain) ExpNextSyncPeriod(t *testing.T, expNsp uint64) { + tc.ExpInit(t, true) + if tc.nsp != expNsp { + t.Errorf("Incorrect NextSyncPeriod (expected %d, got %d)", expNsp, tc.nsp) + } +} + +type TestHeadTracker struct { + phead types.HeadInfo + validated []types.SignedHeader +} + +func (ht *TestHeadTracker) ValidateHead(head types.SignedHeader) (bool, error) { + ht.validated = append(ht.validated, head) + return true, nil +} + +// TODO add test case for finality +func (ht *TestHeadTracker) ValidateFinality(head types.FinalityUpdate) (bool, error) { + return true, nil +} + +func (ht *TestHeadTracker) ExpValidated(t *testing.T, tci int, expHeads []types.SignedHeader) { + for i, expHead := range expHeads { + if i >= len(ht.validated) { + t.Errorf("Missing validated head in test case #%d index #%d (expected {slot %d blockRoot %x}, got none)", tci, i, expHead.Header.Slot, expHead.Header.Hash()) + continue + } + if ht.validated[i] != expHead { + vhead := ht.validated[i].Header + t.Errorf("Wrong validated head in test case #%d index #%d (expected {slot %d blockRoot %x}, got {slot %d blockRoot %x})", tci, i, expHead.Header.Slot, expHead.Header.Hash(), vhead.Slot, vhead.Hash()) + } + } + for i := len(expHeads); i < len(ht.validated); i++ { + vhead := ht.validated[i].Header + t.Errorf("Unexpected validated head in test case #%d index #%d (expected none, got {slot %d blockRoot %x})", tci, i, vhead.Slot, vhead.Hash()) + } + ht.validated = nil +} + +func (ht *TestHeadTracker) SetPrefetchHead(head types.HeadInfo) { + ht.phead = head +} + +func (ht *TestHeadTracker) ExpPrefetch(t *testing.T, tci int, exp types.HeadInfo) { + if ht.phead != exp { + t.Errorf("Wrong prefetch head in test case #%d (expected {slot %d blockRoot %x}, got {slot %d blockRoot %x})", tci, exp.Slot, exp.BlockRoot, ht.phead.Slot, ht.phead.BlockRoot) + } +} diff --git a/beacon/light/sync/types.go b/beacon/light/sync/types.go new file mode 100644 index 0000000000..6449ae842d --- /dev/null +++ b/beacon/light/sync/types.go @@ -0,0 +1,42 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" +) + +var ( + EvNewHead = &request.EventType{Name: "newHead"} // data: types.HeadInfo + EvNewSignedHead = &request.EventType{Name: "newSignedHead"} // data: types.SignedHeader + EvNewFinalityUpdate = &request.EventType{Name: "newFinalityUpdate"} // data: types.FinalityUpdate +) + +type ( + ReqUpdates struct { + FirstPeriod, Count uint64 + } + RespUpdates struct { + Updates []*types.LightClientUpdate + Committees []*types.SerializedSyncCommittee + } + ReqHeader common.Hash + ReqCheckpointData common.Hash + ReqBeaconBlock common.Hash +) diff --git a/beacon/light/sync/update_sync.go b/beacon/light/sync/update_sync.go new file mode 100644 index 0000000000..533e470fb0 --- /dev/null +++ b/beacon/light/sync/update_sync.go @@ -0,0 +1,299 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "sort" + + "github.com/ethereum/go-ethereum/beacon/light" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +const maxUpdateRequest = 8 // maximum number of updates requested in a single request + +type committeeChain interface { + CheckpointInit(bootstrap types.BootstrapData) error + InsertUpdate(update *types.LightClientUpdate, nextCommittee *types.SerializedSyncCommittee) error + NextSyncPeriod() (uint64, bool) +} + +// CheckpointInit implements request.Module; it fetches the light client bootstrap +// data belonging to the given checkpoint hash and initializes the committee chain +// if successful. +type CheckpointInit struct { + chain committeeChain + checkpointHash common.Hash + locked request.ServerAndID + initialized bool +} + +// NewCheckpointInit creates a new CheckpointInit. +func NewCheckpointInit(chain committeeChain, checkpointHash common.Hash) *CheckpointInit { + return &CheckpointInit{ + chain: chain, + checkpointHash: checkpointHash, + } +} + +// Process implements request.Module. +func (s *CheckpointInit) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + if !event.IsRequestEvent() { + continue + } + sid, req, resp := event.RequestInfo() + if s.locked == sid { + s.locked = request.ServerAndID{} + } + if resp != nil { + if checkpoint := resp.(*types.BootstrapData); checkpoint.Header.Hash() == common.Hash(req.(ReqCheckpointData)) { + s.chain.CheckpointInit(*checkpoint) + s.initialized = true + return + } + + requester.Fail(event.Server, "invalid checkpoint data") + } + } + // start a request if possible + if s.initialized || s.locked != (request.ServerAndID{}) { + return + } + cs := requester.CanSendTo() + if len(cs) == 0 { + return + } + server := cs[0] + id := requester.Send(server, ReqCheckpointData(s.checkpointHash)) + s.locked = request.ServerAndID{Server: server, ID: id} +} + +// ForwardUpdateSync implements request.Module; it fetches updates between the +// committee chain head and each server's announced head. Updates are fetched +// in batches and multiple batches can also be requested in parallel. +// Out of order responses are also handled; if a batch of updates cannot be added +// to the chain immediately because of a gap then the future updates are +// remembered until they can be processed. +type ForwardUpdateSync struct { + chain committeeChain + rangeLock rangeLock + lockedIDs map[request.ServerAndID]struct{} + processQueue []updateResponse + nextSyncPeriod map[request.Server]uint64 +} + +// NewForwardUpdateSync creates a new ForwardUpdateSync. +func NewForwardUpdateSync(chain committeeChain) *ForwardUpdateSync { + return &ForwardUpdateSync{ + chain: chain, + rangeLock: make(rangeLock), + lockedIDs: make(map[request.ServerAndID]struct{}), + nextSyncPeriod: make(map[request.Server]uint64), + } +} + +// rangeLock allows locking sections of an integer space, preventing the syncing +// mechanism from making requests again for sections where a not timed out request +// is already pending or where already fetched and unprocessed data is available. +type rangeLock map[uint64]int + +// lock locks or unlocks the given section, depending on the sign of the add parameter. +func (r rangeLock) lock(first, count uint64, add int) { + for i := first; i < first+count; i++ { + if v := r[i] + add; v > 0 { + r[i] = v + } else { + delete(r, i) + } + } +} + +// firstUnlocked returns the first unlocked section starting at or after start +// and not longer than maxCount. +func (r rangeLock) firstUnlocked(start, maxCount uint64) (first, count uint64) { + first = start + for { + if _, ok := r[first]; !ok { + break + } + first++ + } + for { + count++ + if count == maxCount { + break + } + if _, ok := r[first+count]; ok { + break + } + } + return +} + +// lockRange locks the range belonging to the given update request, unless the +// same request has already been locked +func (s *ForwardUpdateSync) lockRange(sid request.ServerAndID, req ReqUpdates) { + if _, ok := s.lockedIDs[sid]; ok { + return + } + s.lockedIDs[sid] = struct{}{} + s.rangeLock.lock(req.FirstPeriod, req.Count, 1) +} + +// unlockRange unlocks the range belonging to the given update request, unless +// same request has already been unlocked +func (s *ForwardUpdateSync) unlockRange(sid request.ServerAndID, req ReqUpdates) { + if _, ok := s.lockedIDs[sid]; !ok { + return + } + delete(s.lockedIDs, sid) + s.rangeLock.lock(req.FirstPeriod, req.Count, -1) +} + +// verifyRange returns true if the number of updates and the individual update +// periods in the response match the requested section. +func (s *ForwardUpdateSync) verifyRange(request ReqUpdates, response RespUpdates) bool { + if uint64(len(response.Updates)) != request.Count || uint64(len(response.Committees)) != request.Count { + return false + } + for i, update := range response.Updates { + if update.AttestedHeader.Header.SyncPeriod() != request.FirstPeriod+uint64(i) { + return false + } + } + return true +} + +// updateResponse is a response that has passed initial verification and has been +// queued for processing. Note that an update response cannot be processed until +// the previous updates have also been added to the chain. +type updateResponse struct { + sid request.ServerAndID + request ReqUpdates + response RespUpdates +} + +// updateResponseList implements sort.Sort and sorts update request/response events by FirstPeriod. +type updateResponseList []updateResponse + +func (u updateResponseList) Len() int { return len(u) } +func (u updateResponseList) Swap(i, j int) { u[i], u[j] = u[j], u[i] } +func (u updateResponseList) Less(i, j int) bool { + return u[i].request.FirstPeriod < u[j].request.FirstPeriod +} + +// Process implements request.Module. +func (s *ForwardUpdateSync) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + switch event.Type { + case request.EvResponse, request.EvFail, request.EvTimeout: + sid, rq, rs := event.RequestInfo() + req := rq.(ReqUpdates) + var queued bool + if event.Type == request.EvResponse { + resp := rs.(RespUpdates) + if s.verifyRange(req, resp) { + // there is a response with a valid format; put it in the process queue + s.processQueue = append(s.processQueue, updateResponse{sid: sid, request: req, response: resp}) + s.lockRange(sid, req) + queued = true + } else { + requester.Fail(event.Server, "invalid update range") + } + } + if !queued { + s.unlockRange(sid, req) + } + case EvNewSignedHead: + signedHead := event.Data.(types.SignedHeader) + s.nextSyncPeriod[event.Server] = types.SyncPeriod(signedHead.SignatureSlot + 256) + case request.EvUnregistered: + delete(s.nextSyncPeriod, event.Server) + } + } + + // try processing ordered list of available responses + sort.Sort(updateResponseList(s.processQueue)) + for s.processQueue != nil { + u := s.processQueue[0] + if !s.processResponse(requester, u) { + break + } + s.unlockRange(u.sid, u.request) + s.processQueue = s.processQueue[1:] + if len(s.processQueue) == 0 { + s.processQueue = nil + } + } + + // start new requests if possible + startPeriod, chainInit := s.chain.NextSyncPeriod() + if !chainInit { + return + } + for { + firstPeriod, maxCount := s.rangeLock.firstUnlocked(startPeriod, maxUpdateRequest) + var ( + sendTo request.Server + bestCount uint64 + ) + for _, server := range requester.CanSendTo() { + nextPeriod := s.nextSyncPeriod[server] + if nextPeriod <= firstPeriod { + continue + } + count := maxCount + if nextPeriod < firstPeriod+maxCount { + count = nextPeriod - firstPeriod + } + if count > bestCount { + sendTo, bestCount = server, count + } + } + if sendTo == nil { + return + } + req := ReqUpdates{FirstPeriod: firstPeriod, Count: bestCount} + id := requester.Send(sendTo, req) + s.lockRange(request.ServerAndID{Server: sendTo, ID: id}, req) + } +} + +// processResponse adds the fetched updates and committees to the committee chain. +// Returns true in case of full or partial success. +func (s *ForwardUpdateSync) processResponse(requester request.Requester, u updateResponse) (success bool) { + for i, update := range u.response.Updates { + if err := s.chain.InsertUpdate(update, u.response.Committees[i]); err != nil { + if err == light.ErrInvalidPeriod { + // there is a gap in the update periods; stop processing without + // failing and try again next time + return + } + if err == light.ErrInvalidUpdate || err == light.ErrWrongCommitteeRoot || err == light.ErrCannotReorg { + requester.Fail(u.sid.Server, "invalid update received") + } else { + log.Error("Unexpected InsertUpdate error", "error", err) + } + return + } + success = true + } + return +} diff --git a/beacon/light/sync/update_sync_test.go b/beacon/light/sync/update_sync_test.go new file mode 100644 index 0000000000..1c4b3d6d76 --- /dev/null +++ b/beacon/light/sync/update_sync_test.go @@ -0,0 +1,219 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "testing" + + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" +) + +func TestCheckpointInit(t *testing.T) { + chain := &TestCommitteeChain{} + checkpoint := &types.BootstrapData{Header: types.Header{Slot: 0x2000*4 + 0x1000}} // period 4 + checkpointHash := checkpoint.Header.Hash() + chkInit := NewCheckpointInit(chain, checkpointHash) + ts := NewTestScheduler(t, chkInit) + // add 2 servers + ts.AddServer(testServer1, 1) + ts.AddServer(testServer2, 1) + + // expect bootstrap request to server 1 + ts.Run(1, testServer1, ReqCheckpointData(checkpointHash)) + + // server 1 times out; expect request to server 2 + ts.RequestEvent(request.EvTimeout, ts.Request(1, 1), nil) + ts.Run(2, testServer2, ReqCheckpointData(checkpointHash)) + + // invalid response from server 2; expect init state to still be false + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), &types.BootstrapData{Header: types.Header{Slot: 123456}}) + ts.ExpFail(testServer2) + ts.Run(3) + chain.ExpInit(t, false) + + // server 1 fails (hard timeout) + ts.RequestEvent(request.EvFail, ts.Request(1, 1), nil) + ts.Run(4) + chain.ExpInit(t, false) + + // server 3 is registered; expect bootstrap request to server 3 + ts.AddServer(testServer3, 1) + ts.Run(5, testServer3, ReqCheckpointData(checkpointHash)) + + // valid response from server 3; expect chain to be initialized + ts.RequestEvent(request.EvResponse, ts.Request(5, 1), checkpoint) + ts.Run(6) + chain.ExpInit(t, true) +} + +func TestUpdateSyncParallel(t *testing.T) { + chain := &TestCommitteeChain{} + chain.SetNextSyncPeriod(0) + updateSync := NewForwardUpdateSync(chain) + ts := NewTestScheduler(t, updateSync) + // add 2 servers, head at period 100; allow 3-3 parallel requests for each + ts.AddServer(testServer1, 3) + ts.ServerEvent(EvNewSignedHead, testServer1, types.SignedHeader{SignatureSlot: 0x2000*100 + 0x1000}) + ts.AddServer(testServer2, 3) + ts.ServerEvent(EvNewSignedHead, testServer2, types.SignedHeader{SignatureSlot: 0x2000*100 + 0x1000}) + + // expect 6 requests to be sent + ts.Run(1, + testServer1, ReqUpdates{FirstPeriod: 0, Count: 8}, + testServer1, ReqUpdates{FirstPeriod: 8, Count: 8}, + testServer1, ReqUpdates{FirstPeriod: 16, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 24, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 32, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 40, Count: 8}) + + // valid response to request 1; expect 8 periods synced and a new request started + ts.RequestEvent(request.EvResponse, ts.Request(1, 1), testRespUpdate(ts.Request(1, 1))) + ts.AddAllowance(testServer1, 1) + ts.Run(2, testServer1, ReqUpdates{FirstPeriod: 48, Count: 8}) + chain.ExpNextSyncPeriod(t, 8) + + // valid response to requests 4 and 5 + ts.RequestEvent(request.EvResponse, ts.Request(1, 4), testRespUpdate(ts.Request(1, 4))) + ts.RequestEvent(request.EvResponse, ts.Request(1, 5), testRespUpdate(ts.Request(1, 5))) + ts.AddAllowance(testServer2, 2) + // expect 2 more requests but no sync progress (responses 4 and 5 cannot be added before 2 and 3) + ts.Run(3, + testServer2, ReqUpdates{FirstPeriod: 56, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 64, Count: 8}) + chain.ExpNextSyncPeriod(t, 8) + + // soft timeout for requests 2 and 3 (server 1 is overloaded) + ts.RequestEvent(request.EvTimeout, ts.Request(1, 2), nil) + ts.RequestEvent(request.EvTimeout, ts.Request(1, 3), nil) + // no allowance, no more requests + ts.Run(4) + + // valid response to requests 6 and 8 and 9 + ts.RequestEvent(request.EvResponse, ts.Request(1, 6), testRespUpdate(ts.Request(1, 6))) + ts.RequestEvent(request.EvResponse, ts.Request(3, 1), testRespUpdate(ts.Request(3, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(3, 2), testRespUpdate(ts.Request(3, 2))) + ts.AddAllowance(testServer2, 3) + // server 2 can now resend requests 2 and 3 (timed out by server 1) and also send a new one + ts.Run(5, + testServer2, ReqUpdates{FirstPeriod: 8, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 16, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 72, Count: 8}) + + // server 1 finally answers timed out request 2 + ts.RequestEvent(request.EvResponse, ts.Request(1, 2), testRespUpdate(ts.Request(1, 2))) + ts.AddAllowance(testServer1, 1) + // expect sync progress and one new request + ts.Run(6, testServer1, ReqUpdates{FirstPeriod: 80, Count: 8}) + chain.ExpNextSyncPeriod(t, 16) + + // server 2 answers requests 11 and 12 (resends of requests 2 and 3) + ts.RequestEvent(request.EvResponse, ts.Request(5, 1), testRespUpdate(ts.Request(5, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(5, 2), testRespUpdate(ts.Request(5, 2))) + ts.AddAllowance(testServer2, 2) + ts.Run(7, + testServer2, ReqUpdates{FirstPeriod: 88, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 96, Count: 4}) + // finally the gap is filled, update can process responses up to req6 + chain.ExpNextSyncPeriod(t, 48) + + // all remaining requests are answered + ts.RequestEvent(request.EvResponse, ts.Request(1, 3), testRespUpdate(ts.Request(1, 3))) + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testRespUpdate(ts.Request(2, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(5, 3), testRespUpdate(ts.Request(5, 3))) + ts.RequestEvent(request.EvResponse, ts.Request(6, 1), testRespUpdate(ts.Request(6, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testRespUpdate(ts.Request(7, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(7, 2), testRespUpdate(ts.Request(7, 2))) + ts.Run(8) + // expect chain to be fully synced + chain.ExpNextSyncPeriod(t, 100) +} + +func TestUpdateSyncDifferentHeads(t *testing.T) { + chain := &TestCommitteeChain{} + chain.SetNextSyncPeriod(10) + updateSync := NewForwardUpdateSync(chain) + ts := NewTestScheduler(t, updateSync) + // add 3 servers with different announced head periods + ts.AddServer(testServer1, 1) + ts.ServerEvent(EvNewSignedHead, testServer1, types.SignedHeader{SignatureSlot: 0x2000*15 + 0x1000}) + ts.AddServer(testServer2, 1) + ts.ServerEvent(EvNewSignedHead, testServer2, types.SignedHeader{SignatureSlot: 0x2000*16 + 0x1000}) + ts.AddServer(testServer3, 1) + ts.ServerEvent(EvNewSignedHead, testServer3, types.SignedHeader{SignatureSlot: 0x2000*17 + 0x1000}) + + // expect request to the best announced head + ts.Run(1, testServer3, ReqUpdates{FirstPeriod: 10, Count: 7}) + + // request times out, expect request to the next best head + ts.RequestEvent(request.EvTimeout, ts.Request(1, 1), nil) + ts.Run(2, testServer2, ReqUpdates{FirstPeriod: 10, Count: 6}) + + // request times out, expect request to the last available server + ts.RequestEvent(request.EvTimeout, ts.Request(2, 1), nil) + ts.Run(3, testServer1, ReqUpdates{FirstPeriod: 10, Count: 5}) + + // valid response to request 3, expect chain synced to period 15 + ts.RequestEvent(request.EvResponse, ts.Request(3, 1), testRespUpdate(ts.Request(3, 1))) + ts.AddAllowance(testServer1, 1) + ts.Run(4) + chain.ExpNextSyncPeriod(t, 15) + + // invalid response to request 1, server can only deliver updates up to period 15 despite announced head + truncated := ts.Request(1, 1) + truncated.request = ReqUpdates{FirstPeriod: 10, Count: 5} + ts.RequestEvent(request.EvResponse, ts.Request(1, 1), testRespUpdate(truncated)) + ts.ExpFail(testServer3) + ts.Run(5) + // expect no progress of chain head + chain.ExpNextSyncPeriod(t, 15) + + // valid response to request 2, expect chain synced to period 16 + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testRespUpdate(ts.Request(2, 1))) + ts.AddAllowance(testServer2, 1) + ts.Run(6) + chain.ExpNextSyncPeriod(t, 16) + + // a new server is registered with announced head period 17 + ts.AddServer(testServer4, 1) + ts.ServerEvent(EvNewSignedHead, testServer4, types.SignedHeader{SignatureSlot: 0x2000*17 + 0x1000}) + // expect request to sync one more period + ts.Run(7, testServer4, ReqUpdates{FirstPeriod: 16, Count: 1}) + + // valid response, expect chain synced to period 17 + ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testRespUpdate(ts.Request(7, 1))) + ts.AddAllowance(testServer4, 1) + ts.Run(8) + chain.ExpNextSyncPeriod(t, 17) +} + +func testRespUpdate(request requestWithID) request.Response { + var resp RespUpdates + if request.request == nil { + return resp + } + req := request.request.(ReqUpdates) + resp.Updates = make([]*types.LightClientUpdate, int(req.Count)) + resp.Committees = make([]*types.SerializedSyncCommittee, int(req.Count)) + period := req.FirstPeriod + for i := range resp.Updates { + resp.Updates[i] = &types.LightClientUpdate{AttestedHeader: types.SignedHeader{Header: types.Header{Slot: 0x2000*period + 0x1000}}} + resp.Committees[i] = new(types.SerializedSyncCommittee) + period++ + } + return resp +} diff --git a/beacon/params/params.go b/beacon/params/params.go index ee9feb1acb..e4e0d00934 100644 --- a/beacon/params/params.go +++ b/beacon/params/params.go @@ -41,4 +41,6 @@ const ( StateIndexNextSyncCommittee = 55 StateIndexExecPayload = 56 StateIndexExecHead = 908 + + BodyIndexExecPayload = 25 ) diff --git a/beacon/types/light_sync.go b/beacon/types/light_sync.go index 3284081e4d..ed62d237f1 100644 --- a/beacon/types/light_sync.go +++ b/beacon/types/light_sync.go @@ -20,11 +20,20 @@ import ( "errors" "fmt" + "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/beacon/merkle" "github.com/ethereum/go-ethereum/beacon/params" "github.com/ethereum/go-ethereum/common" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/ztyp/tree" ) +// HeadInfo represents an unvalidated new head announcement. +type HeadInfo struct { + Slot uint64 + BlockRoot common.Hash +} + // BootstrapData contains a sync committee where light sync can be started, // together with a proof through a beacon header and corresponding state. // Note: BootstrapData is fetched from a server based on a known checkpoint hash. @@ -134,3 +143,50 @@ func (u UpdateScore) BetterThan(w UpdateScore) bool { } return u.SignerCount > w.SignerCount } + +type HeaderWithExecProof struct { + Header + PayloadHeader *capella.ExecutionPayloadHeader + PayloadBranch merkle.Values +} + +func (h *HeaderWithExecProof) Validate() error { + payloadRoot := merkle.Value(h.PayloadHeader.HashTreeRoot(tree.GetHashFn())) + return merkle.VerifyProof(h.BodyRoot, params.BodyIndexExecPayload, h.PayloadBranch, payloadRoot) +} + +type FinalityUpdate struct { + Attested, Finalized HeaderWithExecProof + FinalityBranch merkle.Values + // Sync committee BLS signature aggregate + Signature SyncAggregate + // Slot in which the signature has been created (newer than Header.Slot, + // determines the signing sync committee) + SignatureSlot uint64 +} + +func (u *FinalityUpdate) SignedHeader() SignedHeader { + return SignedHeader{ + Header: u.Attested.Header, + Signature: u.Signature, + SignatureSlot: u.SignatureSlot, + } +} + +func (u *FinalityUpdate) Validate() error { + if err := u.Attested.Validate(); err != nil { + return err + } + if err := u.Finalized.Validate(); err != nil { + return err + } + return merkle.VerifyProof(u.Attested.StateRoot, params.StateIndexFinalBlock, u.FinalityBranch, merkle.Value(u.Finalized.Hash())) +} + +// ChainHeadEvent returns an authenticated execution payload associated with the +// latest accepted head of the beacon chain, along with the hash of the latest +// finalized execution block. +type ChainHeadEvent struct { + HeadBlock *engine.ExecutableData + Finalized common.Hash +} diff --git a/build/checksums.txt b/build/checksums.txt index 96815ff791..03a53946df 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,9 +1,9 @@ # This file contains sha256 checksums of optional build dependencies. -# version:spec-tests 1.0.6 +# version:spec-tests 2.1.0 # https://github.com/ethereum/execution-spec-tests/releases -# https://github.com/ethereum/execution-spec-tests/releases/download/v1.0.6/ -485af7b66cf41eb3a8c1bd46632913b8eb95995df867cf665617bbc9b4beedd1 fixtures_develop.tar.gz +# https://github.com/ethereum/execution-spec-tests/releases/download/v2.1.0/ +ca89c76851b0900bfcc3cbb9a26cbece1f3d7c64a3bed38723e914713290df6c fixtures_develop.tar.gz # version:golang 1.21.6 # https://go.dev/dl/ diff --git a/build/ci.go b/build/ci.go index 1ffbf3074d..4d8dba6ce2 100644 --- a/build/ci.go +++ b/build/ci.go @@ -121,14 +121,13 @@ var ( // Note: vivid is unsupported because there is no golang-1.6 package for it. // Note: the following Ubuntu releases have been officially deprecated on Launchpad: // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite, impish, - // kinetic + // kinetic, lunar debDistroGoBoots = map[string]string{ "trusty": "golang-1.11", // 14.04, EOL: 04/2024 "xenial": "golang-go", // 16.04, EOL: 04/2026 "bionic": "golang-go", // 18.04, EOL: 04/2028 "focal": "golang-go", // 20.04, EOL: 04/2030 "jammy": "golang-go", // 22.04, EOL: 04/2032 - "lunar": "golang-go", // 23.04, EOL: 01/2024 "mantic": "golang-go", // 23.10, EOL: 07/2024 } diff --git a/cmd/blsync/engine_api.go b/cmd/blsync/engine_api.go new file mode 100644 index 0000000000..d10750e295 --- /dev/null +++ b/cmd/blsync/engine_api.go @@ -0,0 +1,69 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "context" + "time" + + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" +) + +func updateEngineApi(client *rpc.Client, headCh chan types.ChainHeadEvent) { + for event := range headCh { + if client == nil { // dry run, no engine API specified + log.Info("New execution block retrieved", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "finalized block hash", event.Finalized) + } else { + if status, err := callNewPayloadV2(client, event.HeadBlock); err == nil { + log.Info("Successful NewPayload", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "status", status) + } else { + log.Error("Failed NewPayload", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "error", err) + } + if status, err := callForkchoiceUpdatedV1(client, event.HeadBlock.BlockHash, event.Finalized); err == nil { + log.Info("Successful ForkchoiceUpdated", "head", event.HeadBlock.BlockHash, "status", status) + } else { + log.Error("Failed ForkchoiceUpdated", "head", event.HeadBlock.BlockHash, "error", err) + } + } + } +} + +func callNewPayloadV2(client *rpc.Client, execData *engine.ExecutableData) (string, error) { + var resp engine.PayloadStatusV1 + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + err := client.CallContext(ctx, &resp, "engine_newPayloadV2", execData) + cancel() + return resp.Status, err +} + +func callForkchoiceUpdatedV1(client *rpc.Client, headHash, finalizedHash common.Hash) (string, error) { + var resp engine.ForkChoiceResponse + update := engine.ForkchoiceStateV1{ + HeadBlockHash: headHash, + SafeBlockHash: finalizedHash, + FinalizedBlockHash: finalizedHash, + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + err := client.CallContext(ctx, &resp, "engine_forkchoiceUpdatedV1", update, nil) + cancel() + return resp.PayloadStatus.Status, err +} diff --git a/cmd/blsync/main.go b/cmd/blsync/main.go new file mode 100644 index 0000000000..fd22761d3c --- /dev/null +++ b/cmd/blsync/main.go @@ -0,0 +1,125 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/ethereum/go-ethereum/beacon/blsync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/rpc" + "github.com/mattn/go-colorable" + "github.com/mattn/go-isatty" + "github.com/urfave/cli/v2" +) + +var ( + verbosityFlag = &cli.IntFlag{ + Name: "verbosity", + Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail", + Value: 3, + Category: flags.LoggingCategory, + } + vmoduleFlag = &cli.StringFlag{ + Name: "vmodule", + Usage: "Per-module verbosity: comma-separated list of = (e.g. eth/*=5,p2p=4)", + Value: "", + Hidden: true, + Category: flags.LoggingCategory, + } +) + +func main() { + app := flags.NewApp("beacon light syncer tool") + app.Flags = []cli.Flag{ + utils.BeaconApiFlag, + utils.BeaconApiHeaderFlag, + utils.BeaconThresholdFlag, + utils.BeaconNoFilterFlag, + utils.BeaconConfigFlag, + utils.BeaconGenesisRootFlag, + utils.BeaconGenesisTimeFlag, + utils.BeaconCheckpointFlag, + //TODO datadir for optional permanent database + utils.MainnetFlag, + utils.SepoliaFlag, + utils.GoerliFlag, + utils.BlsyncApiFlag, + utils.BlsyncJWTSecretFlag, + verbosityFlag, + vmoduleFlag, + } + app.Action = sync + + if err := app.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func sync(ctx *cli.Context) error { + usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb" + output := io.Writer(os.Stderr) + if usecolor { + output = colorable.NewColorable(os.Stderr) + } + verbosity := log.FromLegacyLevel(ctx.Int(verbosityFlag.Name)) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(output, verbosity, usecolor))) + + headCh := make(chan types.ChainHeadEvent, 16) + client := blsync.NewClient(ctx) + sub := client.SubscribeChainHeadEvent(headCh) + go updateEngineApi(makeRPCClient(ctx), headCh) + client.Start() + // run until stopped + <-ctx.Done() + client.Stop() + sub.Unsubscribe() + close(headCh) + return nil +} + +func makeRPCClient(ctx *cli.Context) *rpc.Client { + if !ctx.IsSet(utils.BlsyncApiFlag.Name) { + log.Warn("No engine API target specified, performing a dry run") + return nil + } + if !ctx.IsSet(utils.BlsyncJWTSecretFlag.Name) { + utils.Fatalf("JWT secret parameter missing") //TODO use default if datadir is specified + } + + engineApiUrl, jwtFileName := ctx.String(utils.BlsyncApiFlag.Name), ctx.String(utils.BlsyncJWTSecretFlag.Name) + var jwtSecret [32]byte + if jwt, err := node.ObtainJWTSecret(jwtFileName); err == nil { + copy(jwtSecret[:], jwt) + } else { + utils.Fatalf("Error loading or generating JWT secret: %v", err) + } + auth := node.NewJWTAuth(jwtSecret) + cl, err := rpc.DialOptions(context.Background(), engineApiUrl, rpc.WithHTTPAuth(auth)) + if err != nil { + utils.Fatalf("Could not create RPC client: %v", err) + } + return cl +} diff --git a/cmd/clef/README.md b/cmd/clef/README.md index 3a43db8c95..cf09265136 100644 --- a/cmd/clef/README.md +++ b/cmd/clef/README.md @@ -916,7 +916,7 @@ There are a couple of implementation for a UI. We'll try to keep this list up to | Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters| | ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- | -| QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| -| GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | -| Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | -| Clef UI| https://github.com/ethereum/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)| +| QtSigner| https://github.com/holiman/qtsigner/ | Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| +| GtkSigner| https://github.com/holiman/gtksigner | Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | +| Frame | https://github.com/floating/frame/commits/go-signer | Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | +| Clef UI| https://github.com/ethereum/clef-ui | Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)| diff --git a/cmd/clef/datatypes.md b/cmd/clef/datatypes.md index dd8cda5846..8456edfa35 100644 --- a/cmd/clef/datatypes.md +++ b/cmd/clef/datatypes.md @@ -75,7 +75,7 @@ Example: }, { "type": "Info", - "message": "User should see this aswell" + "message": "User should see this as well" } ], "meta": { diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index e8b3725b17..a34a41dacd 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -26,7 +26,7 @@ import ( "io" "math/big" "os" - "path" + "path/filepath" "sort" "strings" @@ -56,21 +56,21 @@ type Chain struct { // NewChain takes the given chain.rlp file, and decodes and returns // the blocks from the file. func NewChain(dir string) (*Chain, error) { - gen, err := loadGenesis(path.Join(dir, "genesis.json")) + gen, err := loadGenesis(filepath.Join(dir, "genesis.json")) if err != nil { return nil, err } gblock := gen.ToBlock() - blocks, err := blocksFromFile(path.Join(dir, "chain.rlp"), gblock) + blocks, err := blocksFromFile(filepath.Join(dir, "chain.rlp"), gblock) if err != nil { return nil, err } - state, err := readState(path.Join(dir, "headstate.json")) + state, err := readState(filepath.Join(dir, "headstate.json")) if err != nil { return nil, err } - accounts, err := readAccounts(path.Join(dir, "accounts.json")) + accounts, err := readAccounts(filepath.Join(dir, "accounts.json")) if err != nil { return nil, err } diff --git a/cmd/devp2p/internal/ethtest/conn.go b/cmd/devp2p/internal/ethtest/conn.go index 2d36ccb423..ba3c0585fd 100644 --- a/cmd/devp2p/internal/ethtest/conn.go +++ b/cmd/devp2p/internal/ethtest/conn.go @@ -166,7 +166,7 @@ func (c *Conn) ReadEth() (any, error) { case eth.TransactionsMsg: msg = new(eth.TransactionsPacket) case eth.NewPooledTransactionHashesMsg: - msg = new(eth.NewPooledTransactionHashesPacket68) + msg = new(eth.NewPooledTransactionHashesPacket) case eth.GetPooledTransactionsMsg: msg = new(eth.GetPooledTransactionsPacket) case eth.PooledTransactionsMsg: diff --git a/cmd/devp2p/internal/ethtest/engine.go b/cmd/devp2p/internal/ethtest/engine.go index ea4fc76e6f..0e94efa5bd 100644 --- a/cmd/devp2p/internal/ethtest/engine.go +++ b/cmd/devp2p/internal/ethtest/engine.go @@ -22,7 +22,7 @@ import ( "io" "net/http" "os" - "path" + "path/filepath" "time" "github.com/ethereum/go-ethereum/common" @@ -38,7 +38,7 @@ type EngineClient struct { // NewEngineClient creates a new engine client. func NewEngineClient(dir, url, jwt string) (*EngineClient, error) { - headfcu, err := os.ReadFile(path.Join(dir, "headfcu.json")) + headfcu, err := os.ReadFile(filepath.Join(dir, "headfcu.json")) if err != nil { return nil, fmt.Errorf("failed to read headfcu: %w", err) } diff --git a/cmd/devp2p/internal/ethtest/snap.go b/cmd/devp2p/internal/ethtest/snap.go index 64e0633585..8ff3f1f71a 100644 --- a/cmd/devp2p/internal/ethtest/snap.go +++ b/cmd/devp2p/internal/ethtest/snap.go @@ -648,7 +648,7 @@ The server should reject the request.`, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8}}, }, nBytes: 5000, - expHashes: []common.Hash{common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")}, + expHashes: []common.Hash{types.EmptyCodeHash}, }, { diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 4f499d41d8..b5cc27a2b5 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -64,23 +64,23 @@ func NewSuite(dest *enode.Node, chainDir, engineURL, jwt string) (*Suite, error) func (s *Suite) EthTests() []utesting.Test { return []utesting.Test{ // status - {Name: "TestStatus", Fn: s.TestStatus}, + {Name: "Status", Fn: s.TestStatus}, // get block headers - {Name: "TestGetBlockHeaders", Fn: s.TestGetBlockHeaders}, - {Name: "TestSimultaneousRequests", Fn: s.TestSimultaneousRequests}, - {Name: "TestSameRequestID", Fn: s.TestSameRequestID}, - {Name: "TestZeroRequestID", Fn: s.TestZeroRequestID}, + {Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders}, + {Name: "SimultaneousRequests", Fn: s.TestSimultaneousRequests}, + {Name: "SameRequestID", Fn: s.TestSameRequestID}, + {Name: "ZeroRequestID", Fn: s.TestZeroRequestID}, // get block bodies - {Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies}, + {Name: "GetBlockBodies", Fn: s.TestGetBlockBodies}, // // malicious handshakes + status - {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, - {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, + {Name: "MaliciousHandshake", Fn: s.TestMaliciousHandshake}, + {Name: "MaliciousStatus", Fn: s.TestMaliciousStatus}, // test transactions - {Name: "TestLargeTxRequest", Fn: s.TestLargeTxRequest, Slow: true}, - {Name: "TestTransaction", Fn: s.TestTransaction}, - {Name: "TestInvalidTxs", Fn: s.TestInvalidTxs}, - {Name: "TestNewPooledTxs", Fn: s.TestNewPooledTxs}, - {Name: "TestBlobViolations", Fn: s.TestBlobViolations}, + {Name: "LargeTxRequest", Fn: s.TestLargeTxRequest, Slow: true}, + {Name: "Transaction", Fn: s.TestTransaction}, + {Name: "InvalidTxs", Fn: s.TestInvalidTxs}, + {Name: "NewPooledTxs", Fn: s.TestNewPooledTxs}, + {Name: "BlobViolations", Fn: s.TestBlobViolations}, } } @@ -94,9 +94,9 @@ func (s *Suite) SnapTests() []utesting.Test { } } -// TestStatus attempts to connect to the given node and exchange a status -// message with it on the eth protocol. func (s *Suite) TestStatus(t *utesting.T) { + t.Log(`This test is just a sanity check. It performs an eth protocol handshake.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -112,9 +112,9 @@ func headersMatch(expected []*types.Header, headers []*types.Header) bool { return reflect.DeepEqual(expected, headers) } -// TestGetBlockHeaders tests whether the given node can respond to an eth -// `GetBlockHeaders` request and that the response is accurate. func (s *Suite) TestGetBlockHeaders(t *utesting.T) { + t.Log(`This test requests block headers from the node.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -154,10 +154,10 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) { } } -// TestSimultaneousRequests sends two simultaneous `GetBlockHeader` requests -// from the same connection with different request IDs and checks to make sure -// the node responds with the correct headers per request. func (s *Suite) TestSimultaneousRequests(t *utesting.T) { + t.Log(`This test requests blocks headers from the node, performing two requests +concurrently, with different request IDs.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -228,9 +228,10 @@ func (s *Suite) TestSimultaneousRequests(t *utesting.T) { } } -// TestSameRequestID sends two requests with the same request ID to a single -// node. func (s *Suite) TestSameRequestID(t *utesting.T) { + t.Log(`This test requests block headers, performing two concurrent requests with the +same request ID. The node should handle the request by responding to both requests.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -298,9 +299,10 @@ func (s *Suite) TestSameRequestID(t *utesting.T) { } } -// TestZeroRequestID checks that a message with a request ID of zero is still handled -// by the node. func (s *Suite) TestZeroRequestID(t *utesting.T) { + t.Log(`This test sends a GetBlockHeaders message with a request-id of zero, +and expects a response.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -333,9 +335,9 @@ func (s *Suite) TestZeroRequestID(t *utesting.T) { } } -// TestGetBlockBodies tests whether the given node can respond to a -// `GetBlockBodies` request and that the response is accurate. func (s *Suite) TestGetBlockBodies(t *utesting.T) { + t.Log(`This test sends GetBlockBodies requests to the node for known blocks in the test chain.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -376,12 +378,12 @@ func randBuf(size int) []byte { return buf } -// TestMaliciousHandshake tries to send malicious data during the handshake. func (s *Suite) TestMaliciousHandshake(t *utesting.T) { - key, _ := crypto.GenerateKey() + t.Log(`This test tries to send malicious data during the devp2p handshake, in various ways.`) // Write hello to client. var ( + key, _ = crypto.GenerateKey() pub0 = crypto.FromECDSAPub(&key.PublicKey)[1:] version = eth.ProtocolVersions[0] ) @@ -451,8 +453,9 @@ func (s *Suite) TestMaliciousHandshake(t *utesting.T) { } } -// TestMaliciousStatus sends a status package with a large total difficulty. func (s *Suite) TestMaliciousStatus(t *utesting.T) { + t.Log(`This test sends a malicious eth Status message to the node and expects a disconnect.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -486,9 +489,10 @@ func (s *Suite) TestMaliciousStatus(t *utesting.T) { } } -// TestTransaction sends a valid transaction to the node and checks if the -// transaction gets propagated. func (s *Suite) TestTransaction(t *utesting.T) { + t.Log(`This test sends a valid transaction to the node and checks if the +transaction gets propagated.`) + // Nudge client out of syncing mode to accept pending txs. if err := s.engine.sendForkchoiceUpdated(); err != nil { t.Fatalf("failed to send next block: %v", err) @@ -507,15 +511,16 @@ func (s *Suite) TestTransaction(t *utesting.T) { if err != nil { t.Fatalf("failed to sign tx: %v", err) } - if err := s.sendTxs([]*types.Transaction{tx}); err != nil { + if err := s.sendTxs(t, []*types.Transaction{tx}); err != nil { t.Fatal(err) } s.chain.IncNonce(from, 1) } -// TestInvalidTxs sends several invalid transactions and tests whether -// the node will propagate them. func (s *Suite) TestInvalidTxs(t *utesting.T) { + t.Log(`This test sends several kinds of invalid transactions and checks that the node +does not propagate them.`) + // Nudge client out of syncing mode to accept pending txs. if err := s.engine.sendForkchoiceUpdated(); err != nil { t.Fatalf("failed to send next block: %v", err) @@ -534,7 +539,7 @@ func (s *Suite) TestInvalidTxs(t *utesting.T) { if err != nil { t.Fatalf("failed to sign tx: %v", err) } - if err := s.sendTxs([]*types.Transaction{tx}); err != nil { + if err := s.sendTxs(t, []*types.Transaction{tx}); err != nil { t.Fatalf("failed to send txs: %v", err) } s.chain.IncNonce(from, 1) @@ -590,14 +595,15 @@ func (s *Suite) TestInvalidTxs(t *utesting.T) { } txs = append(txs, tx) } - if err := s.sendInvalidTxs(txs); err != nil { + if err := s.sendInvalidTxs(t, txs); err != nil { t.Fatalf("failed to send invalid txs: %v", err) } } -// TestLargeTxRequest tests whether a node can fulfill a large GetPooledTransactions -// request. func (s *Suite) TestLargeTxRequest(t *utesting.T) { + t.Log(`This test first send ~2000 transactions to the node, then requests them +on another peer connection using GetPooledTransactions.`) + // Nudge client out of syncing mode to accept pending txs. if err := s.engine.sendForkchoiceUpdated(); err != nil { t.Fatalf("failed to send next block: %v", err) @@ -630,7 +636,7 @@ func (s *Suite) TestLargeTxRequest(t *utesting.T) { s.chain.IncNonce(from, uint64(count)) // Send txs. - if err := s.sendTxs(txs); err != nil { + if err := s.sendTxs(t, txs); err != nil { t.Fatalf("failed to send txs: %v", err) } @@ -667,13 +673,15 @@ func (s *Suite) TestLargeTxRequest(t *utesting.T) { } } -// TestNewPooledTxs tests whether a node will do a GetPooledTransactions request -// upon receiving a NewPooledTransactionHashes announcement. func (s *Suite) TestNewPooledTxs(t *utesting.T) { + t.Log(`This test announces transaction hashes to the node and expects it to fetch +the transactions using a GetPooledTransactions request.`) + // Nudge client out of syncing mode to accept pending txs. if err := s.engine.sendForkchoiceUpdated(); err != nil { t.Fatalf("failed to send next block: %v", err) } + var ( count = 50 from, nonce = s.chain.GetSender(1) @@ -710,7 +718,7 @@ func (s *Suite) TestNewPooledTxs(t *utesting.T) { } // Send announcement. - ann := eth.NewPooledTransactionHashesPacket68{Types: txTypes, Sizes: sizes, Hashes: hashes} + ann := eth.NewPooledTransactionHashesPacket{Types: txTypes, Sizes: sizes, Hashes: hashes} err = conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann) if err != nil { t.Fatalf("failed to write to connection: %v", err) @@ -728,7 +736,7 @@ func (s *Suite) TestNewPooledTxs(t *utesting.T) { t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsRequest)) } return - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: continue case *eth.TransactionsPacket: continue @@ -746,8 +754,8 @@ func makeSidecar(data ...byte) *types.BlobTxSidecar { ) for i := range blobs { blobs[i][0] = data[i] - c, _ := kzg4844.BlobToCommitment(blobs[i]) - p, _ := kzg4844.ComputeBlobProof(blobs[i], c) + c, _ := kzg4844.BlobToCommitment(&blobs[i]) + p, _ := kzg4844.ComputeBlobProof(&blobs[i], c) commitments = append(commitments, c) proofs = append(proofs, p) } @@ -762,7 +770,7 @@ func (s *Suite) makeBlobTxs(count, blobs int, discriminator byte) (txs types.Tra from, nonce := s.chain.GetSender(5) for i := 0; i < count; i++ { // Make blob data, max of 2 blobs per tx. - blobdata := make([]byte, blobs%2) + blobdata := make([]byte, blobs%3) for i := range blobdata { blobdata[i] = discriminator blobs -= 1 @@ -787,6 +795,8 @@ func (s *Suite) makeBlobTxs(count, blobs int, discriminator byte) (txs types.Tra } func (s *Suite) TestBlobViolations(t *utesting.T) { + t.Log(`This test sends some invalid blob tx announcements and expects the node to disconnect.`) + if err := s.engine.sendForkchoiceUpdated(); err != nil { t.Fatalf("send fcu failed: %v", err) } @@ -796,12 +806,12 @@ func (s *Suite) TestBlobViolations(t *utesting.T) { t2 = s.makeBlobTxs(2, 3, 0x2) ) for _, test := range []struct { - ann eth.NewPooledTransactionHashesPacket68 + ann eth.NewPooledTransactionHashesPacket resp eth.PooledTransactionsResponse }{ // Invalid tx size. { - ann: eth.NewPooledTransactionHashesPacket68{ + ann: eth.NewPooledTransactionHashesPacket{ Types: []byte{types.BlobTxType, types.BlobTxType}, Sizes: []uint32{uint32(t1[0].Size()), uint32(t1[1].Size() + 10)}, Hashes: []common.Hash{t1[0].Hash(), t1[1].Hash()}, @@ -810,7 +820,7 @@ func (s *Suite) TestBlobViolations(t *utesting.T) { }, // Wrong tx type. { - ann: eth.NewPooledTransactionHashesPacket68{ + ann: eth.NewPooledTransactionHashesPacket{ Types: []byte{types.DynamicFeeTxType, types.BlobTxType}, Sizes: []uint32{uint32(t2[0].Size()), uint32(t2[1].Size())}, Hashes: []common.Hash{t2[0].Hash(), t2[1].Hash()}, diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index ad73bc9f90..d70adda51f 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -20,7 +20,7 @@ import ( crand "crypto/rand" "fmt" "os" - "path" + "path/filepath" "testing" "time" @@ -39,7 +39,7 @@ func makeJWTSecret() (string, [32]byte, error) { if _, err := crand.Read(secret[:]); err != nil { return "", secret, fmt.Errorf("failed to create jwt secret: %v", err) } - jwtPath := path.Join(os.TempDir(), "jwt_secret") + jwtPath := filepath.Join(os.TempDir(), "jwt_secret") if err := os.WriteFile(jwtPath, []byte(hexutil.Encode(secret[:])), 0600); err != nil { return "", secret, fmt.Errorf("failed to prepare jwt secret file: %v", err) } diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index 0ea7c32752..80b5d80745 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -25,11 +25,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/internal/utesting" ) // sendTxs sends the given transactions to the node and // expects the node to accept and propagate them. -func (s *Suite) sendTxs(txs []*types.Transaction) error { +func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error { // Open sending conn. sendConn, err := s.dial() if err != nil { @@ -70,10 +71,19 @@ func (s *Suite) sendTxs(txs []*types.Transaction) error { for _, tx := range *msg { got[tx.Hash()] = true } - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: for _, hash := range msg.Hashes { got[hash] = true } + case *eth.GetBlockHeadersPacket: + headers, err := s.chain.GetHeaders(msg) + if err != nil { + t.Logf("invalid GetBlockHeaders request: %v", err) + } + recvConn.Write(ethProto, eth.BlockHeadersMsg, ð.BlockHeadersPacket{ + RequestId: msg.RequestId, + BlockHeadersRequest: headers, + }) default: return fmt.Errorf("unexpected eth wire msg: %s", pretty.Sdump(msg)) } @@ -95,7 +105,7 @@ func (s *Suite) sendTxs(txs []*types.Transaction) error { return fmt.Errorf("timed out waiting for txs") } -func (s *Suite) sendInvalidTxs(txs []*types.Transaction) error { +func (s *Suite) sendInvalidTxs(t *utesting.T, txs []*types.Transaction) error { // Open sending conn. sendConn, err := s.dial() if err != nil { @@ -146,12 +156,21 @@ func (s *Suite) sendInvalidTxs(txs []*types.Transaction) error { return fmt.Errorf("received bad tx: %s", tx.Hash()) } } - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: for _, hash := range msg.Hashes { if _, ok := invalids[hash]; ok { return fmt.Errorf("received bad tx: %s", hash) } } + case *eth.GetBlockHeadersPacket: + headers, err := s.chain.GetHeaders(msg) + if err != nil { + t.Logf("invalid GetBlockHeaders request: %v", err) + } + recvConn.Write(ethProto, eth.BlockHeadersMsg, ð.BlockHeadersPacket{ + RequestId: msg.RequestId, + BlockHeadersRequest: headers, + }) default: return fmt.Errorf("unexpected eth message: %v", pretty.Sdump(msg)) } diff --git a/cmd/devp2p/main.go b/cmd/devp2p/main.go index 8461a8b9b5..66974bba58 100644 --- a/cmd/devp2p/main.go +++ b/cmd/devp2p/main.go @@ -66,9 +66,15 @@ func commandHasFlag(ctx *cli.Context, flag cli.Flag) bool { for _, name := range names { set[name] = struct{}{} } - for _, fn := range ctx.FlagNames() { - if _, ok := set[fn]; ok { - return true + for _, ctx := range ctx.Lineage() { + if ctx.Command != nil { + for _, f := range ctx.Command.Flags { + for _, name := range f.Names() { + if _, ok := set[name]; ok { + return true + } + } + } } } return false diff --git a/cmd/era/main.go b/cmd/era/main.go new file mode 100644 index 0000000000..8b57fd695c --- /dev/null +++ b/cmd/era/main.go @@ -0,0 +1,325 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "math/big" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" + "github.com/urfave/cli/v2" +) + +var app = flags.NewApp("go-ethereum era tool") + +var ( + dirFlag = &cli.StringFlag{ + Name: "dir", + Usage: "directory storing all relevant era1 files", + Value: "eras", + } + networkFlag = &cli.StringFlag{ + Name: "network", + Usage: "network name associated with era1 files", + Value: "mainnet", + } + eraSizeFlag = &cli.IntFlag{ + Name: "size", + Usage: "number of blocks per era", + Value: era.MaxEra1Size, + } + txsFlag = &cli.BoolFlag{ + Name: "txs", + Usage: "print full transaction values", + } +) + +var ( + blockCommand = &cli.Command{ + Name: "block", + Usage: "get block data", + ArgsUsage: "", + Action: block, + Flags: []cli.Flag{ + txsFlag, + }, + } + infoCommand = &cli.Command{ + Name: "info", + ArgsUsage: "", + Usage: "get epoch information", + Action: info, + } + verifyCommand = &cli.Command{ + Name: "verify", + ArgsUsage: "", + Usage: "verifies each era1 against expected accumulator root", + Action: verify, + } +) + +func init() { + app.Commands = []*cli.Command{ + blockCommand, + infoCommand, + verifyCommand, + } + app.Flags = []cli.Flag{ + dirFlag, + networkFlag, + eraSizeFlag, + } +} + +func main() { + if err := app.Run(os.Args); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +// block prints the specified block from an era1 store. +func block(ctx *cli.Context) error { + num, err := strconv.ParseUint(ctx.Args().First(), 10, 64) + if err != nil { + return fmt.Errorf("invalid block number: %w", err) + } + e, err := open(ctx, num/uint64(ctx.Int(eraSizeFlag.Name))) + if err != nil { + return fmt.Errorf("error opening era1: %w", err) + } + defer e.Close() + // Read block with number. + block, err := e.GetBlockByNumber(num) + if err != nil { + return fmt.Errorf("error reading block %d: %w", num, err) + } + // Convert block to JSON and print. + val := ethapi.RPCMarshalBlock(block, ctx.Bool(txsFlag.Name), ctx.Bool(txsFlag.Name), params.MainnetChainConfig) + b, err := json.MarshalIndent(val, "", " ") + if err != nil { + return fmt.Errorf("error marshaling json: %w", err) + } + fmt.Println(string(b)) + return nil +} + +// info prints some high-level information about the era1 file. +func info(ctx *cli.Context) error { + epoch, err := strconv.ParseUint(ctx.Args().First(), 10, 64) + if err != nil { + return fmt.Errorf("invalid epoch number: %w", err) + } + e, err := open(ctx, epoch) + if err != nil { + return err + } + defer e.Close() + acc, err := e.Accumulator() + if err != nil { + return fmt.Errorf("error reading accumulator: %w", err) + } + td, err := e.InitialTD() + if err != nil { + return fmt.Errorf("error reading total difficulty: %w", err) + } + info := struct { + Accumulator common.Hash `json:"accumulator"` + TotalDifficulty *big.Int `json:"totalDifficulty"` + StartBlock uint64 `json:"startBlock"` + Count uint64 `json:"count"` + }{ + acc, td, e.Start(), e.Count(), + } + b, _ := json.MarshalIndent(info, "", " ") + fmt.Println(string(b)) + return nil +} + +// open opens an era1 file at a certain epoch. +func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { + var ( + dir = ctx.String(dirFlag.Name) + network = ctx.String(networkFlag.Name) + ) + entries, err := era.ReadDir(dir, network) + if err != nil { + return nil, fmt.Errorf("error reading era dir: %w", err) + } + if epoch >= uint64(len(entries)) { + return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch) + } + return era.Open(filepath.Join(dir, entries[epoch])) +} + +// verify checks each era1 file in a directory to ensure it is well-formed and +// that the accumulator matches the expected value. +func verify(ctx *cli.Context) error { + if ctx.Args().Len() != 1 { + return errors.New("missing accumulators file") + } + + roots, err := readHashes(ctx.Args().First()) + if err != nil { + return fmt.Errorf("unable to read expected roots file: %w", err) + } + + var ( + dir = ctx.String(dirFlag.Name) + network = ctx.String(networkFlag.Name) + start = time.Now() + reported = time.Now() + ) + + entries, err := era.ReadDir(dir, network) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + + if len(entries) != len(roots) { + return errors.New("number of era1 files should match the number of accumulator hashes") + } + + // Verify each epoch matches the expected root. + for i, want := range roots { + // Wrap in function so defers don't stack. + err := func() error { + name := entries[i] + e, err := era.Open(filepath.Join(dir, name)) + if err != nil { + return fmt.Errorf("error opening era1 file %s: %w", name, err) + } + defer e.Close() + // Read accumulator and check against expected. + if got, err := e.Accumulator(); err != nil { + return fmt.Errorf("error retrieving accumulator for %s: %w", name, err) + } else if got != want { + return fmt.Errorf("invalid root %s: got %s, want %s", name, got, want) + } + // Recompute accumulator. + if err := checkAccumulator(e); err != nil { + return fmt.Errorf("error verify era1 file %s: %w", name, err) + } + // Give the user some feedback that something is happening. + if time.Since(reported) >= 8*time.Second { + fmt.Printf("Verifying Era1 files \t\t verified=%d,\t elapsed=%s\n", i, common.PrettyDuration(time.Since(start))) + reported = time.Now() + } + return nil + }() + if err != nil { + return err + } + } + + return nil +} + +// checkAccumulator verifies the accumulator matches the data in the Era. +func checkAccumulator(e *era.Era) error { + var ( + err error + want common.Hash + td *big.Int + tds = make([]*big.Int, 0) + hashes = make([]common.Hash, 0) + ) + if want, err = e.Accumulator(); err != nil { + return fmt.Errorf("error reading accumulator: %w", err) + } + if td, err = e.InitialTD(); err != nil { + return fmt.Errorf("error reading total difficulty: %w", err) + } + it, err := era.NewIterator(e) + if err != nil { + return fmt.Errorf("error making era iterator: %w", err) + } + // To fully verify an era the following attributes must be checked: + // 1) the block index is constructed correctly + // 2) the tx root matches the value in the block + // 3) the receipts root matches the value in the block + // 4) the starting total difficulty value is correct + // 5) the accumulator is correct by recomputing it locally, which verifies + // the blocks are all correct (via hash) + // + // The attributes 1), 2), and 3) are checked for each block. 4) and 5) require + // accumulation across the entire set and are verified at the end. + for it.Next() { + // 1) next() walks the block index, so we're able to implicitly verify it. + if it.Error() != nil { + return fmt.Errorf("error reading block %d: %w", it.Number(), err) + } + block, receipts, err := it.BlockAndReceipts() + if it.Error() != nil { + return fmt.Errorf("error reading block %d: %w", it.Number(), err) + } + // 2) recompute tx root and verify against header. + tr := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)) + if tr != block.TxHash() { + return fmt.Errorf("tx root in block %d mismatch: want %s, got %s", block.NumberU64(), block.TxHash(), tr) + } + // 3) recompute receipt root and check value against block. + rr := types.DeriveSha(receipts, trie.NewStackTrie(nil)) + if rr != block.ReceiptHash() { + return fmt.Errorf("receipt root in block %d mismatch: want %s, got %s", block.NumberU64(), block.ReceiptHash(), rr) + } + hashes = append(hashes, block.Hash()) + td.Add(td, block.Difficulty()) + tds = append(tds, new(big.Int).Set(td)) + } + // 4+5) Verify accumulator and total difficulty. + got, err := era.ComputeAccumulator(hashes, tds) + if err != nil { + return fmt.Errorf("error computing accumulator: %w", err) + } + if got != want { + return fmt.Errorf("expected accumulator root does not match calculated: got %s, want %s", got, want) + } + return nil +} + +// readHashes reads a file of newline-delimited hashes. +func readHashes(f string) ([]common.Hash, error) { + b, err := os.ReadFile(f) + if err != nil { + return nil, errors.New("unable to open accumulators file") + } + s := strings.Split(string(b), "\n") + // Remove empty last element, if present. + if s[len(s)-1] == "" { + s = s[:len(s)-1] + } + // Convert to hashes. + r := make([]common.Hash, len(s)) + for i := range s { + r[i] = common.HexToHash(s[i]) + } + return r, nil +} diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go index c5d836e0ea..7bfb8c812b 100644 --- a/cmd/evm/blockrunner.go +++ b/cmd/evm/blockrunner.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" @@ -51,7 +51,7 @@ func blockTestCmd(ctx *cli.Context) error { return errors.New("path-to-test argument required") } - var tracer vm.EVMLogger + var tracer *tracing.Hooks // Configure the EVM logger if ctx.Bool(MachineFlag.Name) { tracer = logger.NewJSONLogger(&logger.Config{ @@ -59,7 +59,7 @@ func blockTestCmd(ctx *cli.Context) error { DisableStack: ctx.Bool(DisableStackFlag.Name), DisableStorage: ctx.Bool(DisableStorageFlag.Name), EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name), - }, os.Stderr) + }, os.Stderr).Hooks() } // Load the test content from the input file src, err := os.ReadFile(ctx.Args().First()) diff --git a/cmd/evm/internal/t8ntool/block.go b/cmd/evm/internal/t8ntool/block.go index a2dc473437..62c8593a1d 100644 --- a/cmd/evm/internal/t8ntool/block.go +++ b/cmd/evm/internal/t8ntool/block.go @@ -242,7 +242,7 @@ func readInput(ctx *cli.Context) (*bbInput, error) { if headerStr == stdinSelector || ommersStr == stdinSelector || txsStr == stdinSelector || cliqueStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { - return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) } } if cliqueStr != stdinSelector && cliqueStr != "" { diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 18a4fb836c..3da6cb877d 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -17,7 +17,9 @@ package t8ntool import ( + "encoding/json" "fmt" + "io" "math/big" "github.com/ethereum/go-ethereum/common" @@ -28,21 +30,24 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/tracers/directory" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) type Prestate struct { - Env stEnv `json:"env"` - Pre core.GenesisAlloc `json:"pre"` + Env stEnv `json:"env"` + Pre types.GenesisAlloc `json:"pre"` } // ExecutionResult contains the execution status after running a state test, any @@ -118,7 +123,7 @@ type rejectedTx struct { // Apply applies a set of transactions to a pre-state func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, txIt txIterator, miningReward int64, - getTracerFn func(txIndex int, txHash common.Hash) (vm.EVMLogger, error)) (*state.StateDB, *ExecutionResult, []byte, error) { + getTracerFn func(txIndex int, txHash common.Hash) (*directory.Tracer, io.WriteCloser, error)) (*state.StateDB, *ExecutionResult, []byte, error) { // Capture errors for BLOCKHASH operation, if we haven't been supplied the // required blockhashes var hashError error @@ -189,7 +194,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } if beaconRoot := pre.Env.ParentBeaconBlockRoot; beaconRoot != nil { evm := vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vmConfig) - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb, nil) + core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) } for i := 0; txIt.Next(); i++ { @@ -221,11 +226,13 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, continue } } - tracer, err := getTracerFn(txIndex, tx.Hash()) + tracer, traceOutput, err := getTracerFn(txIndex, tx.Hash()) if err != nil { return nil, nil, nil, err } - vmConfig.Tracer = tracer + if vmConfig.Tracer != nil { + vmConfig.Tracer = tracer.Hooks + } statedb.SetTxContext(tx.Hash(), txIndex) var ( @@ -236,7 +243,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, evm := vm.NewEVM(vmContext, txContext, statedb, chainConfig, vmConfig) if tracer != nil { - tracer.CaptureTxStart(evm, tx, msg.From) + tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } // (ret []byte, usedGas uint64, failed bool, err error) msgResult, err := core.ApplyMessage(evm, msg, gaspool) @@ -246,7 +253,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()}) gaspool.SetGas(prevGas) if tracer != nil { - tracer.CaptureTxEnd(nil, err) + tracer.OnTxEnd(nil, err) + if err := writeTraceResult(tracer, traceOutput); err != nil { + log.Warn("Error writing tracer output", "err", err) + } } continue } @@ -291,7 +301,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, receipt.TransactionIndex = uint(txIndex) receipts = append(receipts, receipt) if tracer != nil { - tracer.CaptureTxEnd(receipt, nil) + tracer.Hooks.OnTxEnd(receipt, nil) + writeTraceResult(tracer, traceOutput) } } @@ -318,15 +329,15 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, reward.Sub(reward, new(big.Int).SetUint64(ommer.Delta)) reward.Mul(reward, blockReward) reward.Div(reward, big.NewInt(8)) - statedb.AddBalance(ommer.Address, uint256.MustFromBig(reward), state.BalanceIncreaseRewardMineUncle) + statedb.AddBalance(ommer.Address, uint256.MustFromBig(reward), tracing.BalanceIncreaseRewardMineUncle) } - statedb.AddBalance(pre.Env.Coinbase, uint256.MustFromBig(minerReward), state.BalanceIncreaseRewardMineBlock) + statedb.AddBalance(pre.Env.Coinbase, uint256.MustFromBig(minerReward), tracing.BalanceIncreaseRewardMineBlock) } // Apply withdrawals for _, w := range pre.Env.Withdrawals { // Amount is in gwei, turn into wei amount := new(big.Int).Mul(new(big.Int).SetUint64(w.Amount), big.NewInt(params.GWei)) - statedb.AddBalance(w.Address, uint256.MustFromBig(amount), state.BalanceIncreaseWithdrawal) + statedb.AddBalance(w.Address, uint256.MustFromBig(amount), tracing.BalanceIncreaseWithdrawal) } // Commit block root, err := statedb.Commit(vmContext.BlockNumber.Uint64(), chainConfig.IsEIP158(vmContext.BlockNumber)) @@ -363,13 +374,13 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return statedb, execRs, body, nil } -func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB { - sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) +func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB { + sdb := state.NewDatabaseWithConfig(db, &triedb.Config{Preimages: true}) statedb, _ := state.New(types.EmptyRootHash, sdb, nil) for addr, a := range accounts { statedb.SetCode(addr, a.Code) statedb.SetNonce(addr, a.Nonce) - statedb.SetBalance(addr, uint256.MustFromBig(a.Balance), state.BalanceIncreaseGenesisBalance) + statedb.SetBalance(addr, uint256.MustFromBig(a.Balance), tracing.BalanceIncreaseGenesisBalance) for k, v := range a.Storage { statedb.SetState(addr, k, v) } @@ -406,3 +417,16 @@ func calcDifficulty(config *params.ChainConfig, number, currentTime, parentTime } return ethash.CalcDifficulty(config, currentTime, parent) } + +func writeTraceResult(tracer *directory.Tracer, f io.WriteCloser) error { + defer f.Close() + result, err := tracer.GetResult() + if err != nil { + return err + } + err = json.NewEncoder(f).Encode(result) + if err != nil { + return err + } + return nil +} diff --git a/cmd/evm/internal/t8ntool/tracewriter.go b/cmd/evm/internal/t8ntool/tracewriter.go deleted file mode 100644 index 8a850f1d98..0000000000 --- a/cmd/evm/internal/t8ntool/tracewriter.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package t8ntool - -import ( - "encoding/json" - "io" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/tracers/directory" - "github.com/ethereum/go-ethereum/log" -) - -// traceWriter is an vm.EVMLogger which also holds an inner logger/tracer. -// When the TxEnd event happens, the inner tracer result is written to the file, and -// the file is closed. -type traceWriter struct { - inner vm.EVMLogger - f io.WriteCloser -} - -// Compile-time interface check -var _ = vm.EVMLogger((*traceWriter)(nil)) - -func (t *traceWriter) CaptureTxEnd(receipt *types.Receipt, err error) { - t.inner.CaptureTxEnd(receipt, err) - defer t.f.Close() - - if tracer, ok := t.inner.(directory.Tracer); ok { - result, err := tracer.GetResult() - if err != nil { - log.Warn("Error in tracer", "err", err) - return - } - err = json.NewEncoder(t.f).Encode(result) - if err != nil { - log.Warn("Error writing tracer output", "err", err) - return - } - } -} - -func (t *traceWriter) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common.Address) { - t.inner.CaptureTxStart(env, tx, from) -} -func (t *traceWriter) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { - t.inner.CaptureStart(from, to, create, input, gas, value) -} - -func (t *traceWriter) CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) { - t.inner.CaptureEnd(output, gasUsed, err, reverted) -} - -func (t *traceWriter) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { - t.inner.CaptureEnter(typ, from, to, input, gas, value) -} - -func (t *traceWriter) CaptureExit(output []byte, gasUsed uint64, err error, reverted bool) { - t.inner.CaptureExit(output, gasUsed, err, reverted) -} - -func (t *traceWriter) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { - t.inner.CaptureState(pc, op, gas, cost, scope, rData, depth, err) -} -func (t *traceWriter) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { - t.inner.CaptureFault(pc, op, gas, cost, scope, depth, err) -} - -func (t *traceWriter) CaptureKeccakPreimage(hash common.Hash, data []byte) { - t.inner.CaptureKeccakPreimage(hash, data) -} - -func (t *traceWriter) OnGasChange(old, new uint64, reason vm.GasChangeReason) { - t.inner.OnGasChange(old, new, reason) -} diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 8533b78637..7f66ba4d85 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -86,7 +86,7 @@ func Transaction(ctx *cli.Context) error { if txStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) } // Decode the body of already signed transactions body = common.FromHex(inputData.TxRlp) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index f8ec76aa69..b91f468112 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -20,14 +20,14 @@ import ( "encoding/json" "errors" "fmt" + "io" "math/big" "os" - "path" + "path/filepath" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -74,14 +74,14 @@ var ( ) type input struct { - Alloc core.GenesisAlloc `json:"alloc,omitempty"` - Env *stEnv `json:"env,omitempty"` - Txs []*txWithKey `json:"txs,omitempty"` - TxRlp string `json:"txsRlp,omitempty"` + Alloc types.GenesisAlloc `json:"alloc,omitempty"` + Env *stEnv `json:"env,omitempty"` + Txs []*txWithKey `json:"txs,omitempty"` + TxRlp string `json:"txsRlp,omitempty"` } func Transition(ctx *cli.Context) error { - var getTracer = func(txIndex int, txHash common.Hash) (vm.EVMLogger, error) { return nil, nil } + var getTracer = func(txIndex int, txHash common.Hash) (*directory.Tracer, io.WriteCloser, error) { return nil, nil, nil } baseDir, err := createBasedir(ctx) if err != nil { @@ -96,28 +96,35 @@ func Transition(ctx *cli.Context) error { EnableReturnData: ctx.Bool(TraceEnableReturnDataFlag.Name), Debug: true, } - getTracer = func(txIndex int, txHash common.Hash) (vm.EVMLogger, error) { - traceFile, err := os.Create(path.Join(baseDir, fmt.Sprintf("trace-%d-%v.jsonl", txIndex, txHash.String()))) + getTracer = func(txIndex int, txHash common.Hash) (*directory.Tracer, io.WriteCloser, error) { + traceFile, err := os.Create(filepath.Join(baseDir, fmt.Sprintf("trace-%d-%v.jsonl", txIndex, txHash.String()))) if err != nil { - return nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err)) + return nil, nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err)) } - return &traceWriter{logger.NewJSONLogger(logConfig, traceFile), traceFile}, nil + logger := logger.NewJSONLogger(logConfig, traceFile).Hooks() + tracer := &directory.Tracer{ + Hooks: logger, + // JSONLogger streams out result to file. + GetResult: func() (json.RawMessage, error) { return nil, nil }, + Stop: func(err error) {}, + } + return tracer, traceFile, nil } } else if ctx.IsSet(TraceTracerFlag.Name) { var config json.RawMessage if ctx.IsSet(TraceTracerConfigFlag.Name) { config = []byte(ctx.String(TraceTracerConfigFlag.Name)) } - getTracer = func(txIndex int, txHash common.Hash) (vm.EVMLogger, error) { - traceFile, err := os.Create(path.Join(baseDir, fmt.Sprintf("trace-%d-%v.json", txIndex, txHash.String()))) + getTracer = func(txIndex int, txHash common.Hash) (*directory.Tracer, io.WriteCloser, error) { + traceFile, err := os.Create(filepath.Join(baseDir, fmt.Sprintf("trace-%d-%v.json", txIndex, txHash.String()))) if err != nil { - return nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err)) + return nil, nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err)) } tracer, err := directory.DefaultDirectory.New(ctx.String(TraceTracerFlag.Name), nil, config) if err != nil { - return nil, NewError(ErrorConfig, fmt.Errorf("failed instantiating tracer: %w", err)) + return nil, nil, NewError(ErrorConfig, fmt.Errorf("failed instantiating tracer: %w", err)) } - return &traceWriter{tracer, traceFile}, nil + return tracer, traceFile, nil } } // We need to load three things: alloc, env and transactions. May be either in @@ -136,7 +143,7 @@ func Transition(ctx *cli.Context) error { if allocStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) } } if allocStr != stdinSelector { @@ -272,7 +279,7 @@ func applyCancunChecks(env *stEnv, chainConfig *params.ChainConfig) error { return nil } -type Alloc map[common.Address]core.GenesisAccount +type Alloc map[common.Address]types.Account func (g Alloc) OnRoot(common.Hash) {} @@ -288,7 +295,7 @@ func (g Alloc) OnAccount(addr *common.Address, dumpAccount state.DumpAccount) { storage[k] = common.HexToHash(v) } } - genesisAccount := core.GenesisAccount{ + genesisAccount := types.Account{ Code: dumpAccount.Code, Storage: storage, Balance: balance, @@ -303,7 +310,7 @@ func saveFile(baseDir, filename string, data interface{}) error { if err != nil { return NewError(ErrorJson, fmt.Errorf("failed marshalling output: %v", err)) } - location := path.Join(baseDir, filename) + location := filepath.Join(baseDir, filename) if err = os.WriteFile(location, b, 0644); err != nil { return NewError(ErrorIO, fmt.Errorf("failed writing output: %v", err)) } diff --git a/cmd/evm/internal/t8ntool/tx_iterator.go b/cmd/evm/internal/t8ntool/tx_iterator.go index 8f28dc7022..046f62314d 100644 --- a/cmd/evm/internal/t8ntool/tx_iterator.go +++ b/cmd/evm/internal/t8ntool/tx_iterator.go @@ -127,7 +127,7 @@ func loadTransactions(txStr string, inputData *input, env stEnv, chainConfig *pa return newRlpTxIterator(body), nil } if err := json.Unmarshal(data, &txsWithKeys); err != nil { - return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err)) + return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshalling txs-file: %v", err)) } } else { if len(inputData.TxRlp) > 0 { diff --git a/cmd/evm/internal/t8ntool/utils.go b/cmd/evm/internal/t8ntool/utils.go index 8ec38c7618..42f5471e7b 100644 --- a/cmd/evm/internal/t8ntool/utils.go +++ b/cmd/evm/internal/t8ntool/utils.go @@ -33,7 +33,7 @@ func readFile(path, desc string, dest interface{}) error { defer inFile.Close() decoder := json.NewDecoder(inFile) if err := decoder.Decode(dest); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed unmarshaling %s file: %v", desc, err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshalling %s file: %v", desc, err)) } return nil } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index f3ffb3ed9f..cd711893d0 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -33,13 +33,14 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm/runtime" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" "github.com/urfave/cli/v2" ) @@ -116,7 +117,7 @@ func runCmd(ctx *cli.Context) error { } var ( - tracer vm.EVMLogger + tracer *tracing.Hooks debugLogger *logger.StructLogger statedb *state.StateDB chainConfig *params.ChainConfig @@ -127,10 +128,10 @@ func runCmd(ctx *cli.Context) error { blobBaseFee = new(big.Int) // TODO (MariusVanDerWijden) implement blob fee in state tests ) if ctx.Bool(MachineFlag.Name) { - tracer = logger.NewJSONLogger(logconfig, os.Stdout) + tracer = logger.NewJSONLogger(logconfig, os.Stdout).Hooks() } else if ctx.Bool(DebugFlag.Name) { debugLogger = logger.NewStructLogger(logconfig) - tracer = debugLogger + tracer = debugLogger.Hooks() } else { debugLogger = logger.NewStructLogger(logconfig) } @@ -148,7 +149,7 @@ func runCmd(ctx *cli.Context) error { } db := rawdb.NewMemoryDatabase() - triedb := trie.NewDatabase(db, &trie.Config{ + triedb := triedb.NewDatabase(db, &triedb.Config{ Preimages: preimages, HashDB: hashdb.Defaults, }) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 6e751b630f..3a23aa7fa5 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/tests" @@ -61,10 +60,10 @@ func stateTestCmd(ctx *cli.Context) error { var cfg vm.Config switch { case ctx.Bool(MachineFlag.Name): - cfg.Tracer = logger.NewJSONLogger(config, os.Stderr) + cfg.Tracer = logger.NewJSONLogger(config, os.Stderr).Hooks() case ctx.Bool(DebugFlag.Name): - cfg.Tracer = logger.NewStructLogger(config) + cfg.Tracer = logger.NewStructLogger(config).Hooks() } // Load the test content from the input file if len(ctx.Args().First()) != 0 { @@ -90,26 +89,27 @@ func runStateTest(fname string, cfg vm.Config, jsonOut, dump bool) error { if err != nil { return err } - var tests map[string]tests.StateTest - if err := json.Unmarshal(src, &tests); err != nil { + var testsByName map[string]tests.StateTest + if err := json.Unmarshal(src, &testsByName); err != nil { return err } + // Iterate over all the tests, run them and aggregate the results - results := make([]StatetestResult, 0, len(tests)) - for key, test := range tests { + results := make([]StatetestResult, 0, len(testsByName)) + for key, test := range testsByName { for _, st := range test.Subtests() { // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} - test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, statedb *state.StateDB) { + test.Run(st, cfg, false, rawdb.HashScheme, func(err error, tstate *tests.StateTestState) { var root common.Hash - if statedb != nil { - root = statedb.IntermediateRoot(false) + if tstate.StateDB != nil { + root = tstate.StateDB.IntermediateRoot(false) result.Root = &root if jsonOut { fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) } if dump { // Dump any state to aid debugging - cpy, _ := state.New(root, statedb.Database(), nil) + cpy, _ := state.New(root, tstate.StateDB.Database(), nil) dump := cpy.RawDump(nil) result.State = &dump } diff --git a/cmd/evm/transition-test.sh b/cmd/evm/transition-test.sh index 8cc6aa41de..2ddda2d473 100644 --- a/cmd/evm/transition-test.sh +++ b/cmd/evm/transition-test.sh @@ -103,7 +103,7 @@ type Env struct { CurrentTimestamp uint64 `json:"currentTimestamp"` Withdrawals []*Withdrawal `json:"withdrawals"` // optional - CurrentDifficulty *big.Int `json:"currentDifficuly"` + CurrentDifficulty *big.Int `json:"currentDifficulty"` CurrentRandom *big.Int `json:"currentRandom"` CurrentBaseFee *big.Int `json:"currentBaseFee"` ParentDifficulty *big.Int `json:"parentDifficulty"` diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index a7715e6e7a..dc45661eae 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -35,10 +35,12 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/urfave/cli/v2" ) @@ -98,6 +100,7 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.MetricsInfluxDBOrganizationFlag, utils.TxLookupLimitFlag, utils.VMTraceFlag, + utils.VMTraceConfigFlag, utils.TransactionHistoryFlag, utils.StateHistoryFlag, }, utils.DatabaseFlags), @@ -123,6 +126,33 @@ Optional second and third arguments control the first and last block to write. In this mode, the file will be appended if already existing. If the file ends with .gz, the output will be gzipped.`, + } + importHistoryCommand = &cli.Command{ + Action: importHistory, + Name: "import-history", + Usage: "Import an Era archive", + ArgsUsage: "", + Flags: flags.Merge([]cli.Flag{ + utils.TxLookupLimitFlag, + }, + utils.DatabaseFlags, + utils.NetworkFlags, + ), + Description: ` +The import-history command will import blocks and their corresponding receipts +from Era archives. +`, + } + exportHistoryCommand = &cli.Command{ + Action: exportHistory, + Name: "export-history", + Usage: "Export blockchain history to Era archives", + ArgsUsage: " ", + Flags: flags.Merge(utils.DatabaseFlags), + Description: ` +The export-history command will export blocks and their corresponding receipts +into Era archives. Eras are typically packaged in steps of 8192 blocks. +`, } importPreimagesCommand = &cli.Command{ Action: importPreimages, @@ -365,7 +395,97 @@ func exportChain(ctx *cli.Context) error { } err = utils.ExportAppendChain(chain, fp, uint64(first), uint64(last)) } + if err != nil { + utils.Fatalf("Export error: %v\n", err) + } + fmt.Printf("Export done in %v\n", time.Since(start)) + return nil +} +func importHistory(ctx *cli.Context) error { + if ctx.Args().Len() != 1 { + utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) + } + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, db := utils.MakeChain(ctx, stack, false) + defer db.Close() + + var ( + start = time.Now() + dir = ctx.Args().Get(0) + network string + ) + + // Determine network. + if utils.IsNetworkPreset(ctx) { + switch { + case ctx.Bool(utils.MainnetFlag.Name): + network = "mainnet" + case ctx.Bool(utils.SepoliaFlag.Name): + network = "sepolia" + case ctx.Bool(utils.GoerliFlag.Name): + network = "goerli" + } + } else { + // No network flag set, try to determine network based on files + // present in directory. + var networks []string + for _, n := range params.NetworkNames { + entries, err := era.ReadDir(dir, n) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + if len(entries) > 0 { + networks = append(networks, n) + } + } + if len(networks) == 0 { + return fmt.Errorf("no era1 files found in %s", dir) + } + if len(networks) > 1 { + return errors.New("multiple networks found, use a network flag to specify desired network") + } + network = networks[0] + } + + if err := utils.ImportHistory(chain, db, dir, network); err != nil { + return err + } + fmt.Printf("Import done in %v\n", time.Since(start)) + return nil +} + +// exportHistory exports chain history in Era archives at a specified +// directory. +func exportHistory(ctx *cli.Context) error { + if ctx.Args().Len() != 3 { + utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) + } + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, _ := utils.MakeChain(ctx, stack, true) + start := time.Now() + + var ( + dir = ctx.Args().Get(0) + first, ferr = strconv.ParseInt(ctx.Args().Get(1), 10, 64) + last, lerr = strconv.ParseInt(ctx.Args().Get(2), 10, 64) + ) + if ferr != nil || lerr != nil { + utils.Fatalf("Export error in parsing parameters: block number not an integer\n") + } + if first < 0 || last < 0 { + utils.Fatalf("Export error: block number must be greater than 0\n") + } + if head := chain.CurrentSnapBlock(); uint64(last) > head.Number.Uint64() { + utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.Number.Uint64()) + } + err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size)) if err != nil { utils.Fatalf("Export error: %v\n", err) } @@ -396,13 +516,10 @@ func importPreimages(ctx *cli.Context) error { return nil } -func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) { - db := utils.MakeChainDatabase(ctx, stack, true) - defer db.Close() - +func parseDumpConfig(ctx *cli.Context, stack *node.Node, db ethdb.Database) (*state.DumpConfig, common.Hash, error) { var header *types.Header if ctx.NArg() > 1 { - return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) + return nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) } if ctx.NArg() == 1 { arg := ctx.Args().First() @@ -411,17 +528,17 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth if number := rawdb.ReadHeaderNumber(db, hash); number != nil { header = rawdb.ReadHeader(db, hash, *number) } else { - return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash) + return nil, common.Hash{}, fmt.Errorf("block %x not found", hash) } } else { number, err := strconv.ParseUint(arg, 10, 64) if err != nil { - return nil, nil, common.Hash{}, err + return nil, common.Hash{}, err } if hash := rawdb.ReadCanonicalHash(db, number); hash != (common.Hash{}) { header = rawdb.ReadHeader(db, hash, number) } else { - return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) + return nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) } } } else { @@ -429,7 +546,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth header = rawdb.ReadHeadHeader(db) } if header == nil { - return nil, nil, common.Hash{}, errors.New("no head block found") + return nil, common.Hash{}, errors.New("no head block found") } startArg := common.FromHex(ctx.String(utils.StartKeyFlag.Name)) var start common.Hash @@ -441,7 +558,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth start = crypto.Keccak256Hash(startArg) log.Info("Converting start-address to hash", "address", common.BytesToAddress(startArg), "hash", start.Hex()) default: - return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg) + return nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg) } var conf = &state.DumpConfig{ SkipCode: ctx.Bool(utils.ExcludeCodeFlag.Name), @@ -453,14 +570,17 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(), "skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage, "start", hexutil.Encode(conf.Start), "limit", conf.Max) - return conf, db, header.Root, nil + return conf, header.Root, nil } func dump(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - conf, db, root, err := parseDumpConfig(ctx, stack) + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + conf, root, err := parseDumpConfig(ctx, stack, db) if err != nil { return err } diff --git a/cmd/geth/config.go b/cmd/geth/config.go index edc907ff3f..23a2d01e73 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -31,12 +31,12 @@ import ( "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/accounts/scwallet" "github.com/ethereum/go-ethereum/accounts/usbwallet" + "github.com/ethereum/go-ethereum/beacon/blsync" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/eth/tracers/directory" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/internal/version" @@ -179,17 +179,6 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { cfg.Eth.OverrideVerkle = &v } - if ctx.IsSet(utils.VMTraceFlag.Name) { - if name := ctx.String(utils.VMTraceFlag.Name); name != "" { - t, err := directory.LiveDirectory.New(name) - if err != nil { - utils.Fatalf("Failed to create tracer %q: %v", name, err) - } - - cfg.Eth.VMTracer = t - } - } - backend, eth := utils.RegisterEthService(stack, &cfg.Eth) // Create gauge with geth system and build information @@ -234,6 +223,8 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { } catalyst.RegisterSimulatedBeaconAPIs(stack, simBeacon) stack.RegisterLifecycle(simBeacon) + } else if ctx.IsSet(utils.BeaconApiFlag.Name) { + stack.RegisterLifecycle(catalyst.NewBlsync(blsync.NewClient(ctx), eth)) } else { err := catalyst.Register(stack, eth) if err != nil { @@ -277,7 +268,7 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { cfg.Metrics.Enabled = ctx.Bool(utils.MetricsEnabledFlag.Name) } if ctx.IsSet(utils.MetricsEnabledExpensiveFlag.Name) { - cfg.Metrics.EnabledExpensive = ctx.Bool(utils.MetricsEnabledExpensiveFlag.Name) + log.Warn("Expensive metrics are collected by default, please remove this flag", "flag", utils.MetricsEnabledExpensiveFlag.Name) } if ctx.IsSet(utils.MetricsHTTPFlag.Name) { cfg.Metrics.HTTP = ctx.String(utils.MetricsHTTPFlag.Name) diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index ef6ef5f288..4d62206417 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -71,7 +71,6 @@ func TestConsoleWelcome(t *testing.T) { Welcome to the Geth JavaScript console! instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}} -coinbase: {{.Etherbase}} at block: 0 ({{niltime}}) datadir: {{.Datadir}} modules: {{apis}} @@ -131,7 +130,6 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) { attach.SetTemplateFunc("goarch", func() string { return runtime.GOARCH }) attach.SetTemplateFunc("gover", runtime.Version) attach.SetTemplateFunc("gethver", func() string { return params.VersionWithCommit("", "") }) - attach.SetTemplateFunc("etherbase", func() string { return geth.Etherbase }) attach.SetTemplateFunc("niltime", func() string { return time.Unix(1548854791, 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)") }) @@ -144,7 +142,6 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) { Welcome to the Geth JavaScript console! instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}} -coinbase: {{etherbase}} at block: 0 ({{niltime}}){{if ipc}} datadir: {{datadir}}{{end}} modules: {{apis}} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 5cd0411bfa..8ec70aedf9 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console/prompt" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/internal/debug" @@ -117,13 +116,14 @@ var ( utils.DiscoveryPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, - utils.MiningEnabledFlag, + utils.MiningEnabledFlag, // deprecated utils.MinerGasLimitFlag, utils.MinerGasPriceFlag, - utils.MinerEtherbaseFlag, + utils.MinerEtherbaseFlag, // deprecated utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, - utils.MinerNewPayloadTimeout, + utils.MinerPendingFeeRecipientFlag, + utils.MinerNewPayloadTimeoutFlag, // deprecated utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV4Flag, @@ -138,6 +138,7 @@ var ( utils.DeveloperPeriodFlag, utils.VMEnableDebugFlag, utils.VMTraceFlag, + utils.VMTraceConfigFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, utils.NoCompactionFlag, @@ -148,6 +149,14 @@ var ( configFileFlag, utils.LogDebugFlag, utils.LogBacktraceAtFlag, + utils.BeaconApiFlag, + utils.BeaconApiHeaderFlag, + utils.BeaconThresholdFlag, + utils.BeaconNoFilterFlag, + utils.BeaconConfigFlag, + utils.BeaconGenesisRootFlag, + utils.BeaconGenesisTimeFlag, + utils.BeaconCheckpointFlag, }, utils.NetworkFlags, utils.DatabaseFlags) rpcFlags = []cli.Flag{ @@ -210,6 +219,8 @@ func init() { initCommand, importCommand, exportCommand, + importHistoryCommand, + exportHistoryCommand, importPreimagesCommand, removedbCommand, dumpCommand, @@ -421,24 +432,6 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon } }() } - - // Start auxiliary services if enabled - if ctx.Bool(utils.MiningEnabledFlag.Name) { - // Mining only makes sense if a full Ethereum node is running - if ctx.String(utils.SyncModeFlag.Name) == "light" { - utils.Fatalf("Light clients do not support mining") - } - ethBackend, ok := backend.(*eth.EthAPIBackend) - if !ok { - utils.Fatalf("Ethereum service not running") - } - // Set the gas price to the limits from the CLI and start mining - gasprice := flags.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) - ethBackend.TxPool().SetGasTip(gasprice) - if err := ethBackend.StartMining(); err != nil { - utils.Fatalf("Failed to start mining: %v", err) - } - } } // unlockAccounts unlocks any account specifically requested. diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 4284005a02..192c850868 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -36,7 +36,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - cli "github.com/urfave/cli/v2" + "github.com/urfave/cli/v2" ) var ( @@ -541,7 +541,10 @@ func dumpState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - conf, db, root, err := parseDumpConfig(ctx, stack) + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + conf, root, err := parseDumpConfig(ctx, stack, db) if err != nil { return err } diff --git a/cmd/geth/testdata/clique.json b/cmd/geth/testdata/clique.json index b54b4a7d3b..36f5c31057 100644 --- a/cmd/geth/testdata/clique.json +++ b/cmd/geth/testdata/clique.json @@ -8,6 +8,7 @@ "byzantiumBlock": 0, "constantinopleBlock": 0, "petersburgBlock": 0, + "terminalTotalDifficultyPassed": true, "clique": { "period": 5, "epoch": 30000 diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index 420b063d8b..ff3931356e 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/gballet/go-verkle" - cli "github.com/urfave/cli/v2" + "github.com/urfave/cli/v2" ) var ( diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 8b571be1ef..fc66e11dca 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -19,12 +19,15 @@ package utils import ( "bufio" + "bytes" "compress/gzip" + "crypto/sha256" "errors" "fmt" "io" "os" "os/signal" + "path/filepath" "runtime" "strings" "syscall" @@ -39,8 +42,10 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/urfave/cli/v2" ) @@ -228,6 +233,105 @@ func ImportChain(chain *core.BlockChain, fn string) error { return nil } +func readList(filename string) ([]string, error) { + b, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + return strings.Split(string(b), "\n"), nil +} + +// ImportHistory imports Era1 files containing historical block information, +// starting from genesis. +func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error { + if chain.CurrentSnapBlock().Number.BitLen() != 0 { + return errors.New("history import only supported when starting from genesis") + } + entries, err := era.ReadDir(dir, network) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + checksums, err := readList(filepath.Join(dir, "checksums.txt")) + if err != nil { + return fmt.Errorf("unable to read checksums.txt: %w", err) + } + if len(checksums) != len(entries) { + return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", len(checksums), len(entries)) + } + var ( + start = time.Now() + reported = time.Now() + imported = 0 + forker = core.NewForkChoice(chain, nil) + h = sha256.New() + buf = bytes.NewBuffer(nil) + ) + for i, filename := range entries { + err := func() error { + f, err := os.Open(filepath.Join(dir, filename)) + if err != nil { + return fmt.Errorf("unable to open era: %w", err) + } + defer f.Close() + + // Validate checksum. + if _, err := io.Copy(h, f); err != nil { + return fmt.Errorf("unable to recalculate checksum: %w", err) + } + if have, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; have != want { + return fmt.Errorf("checksum mismatch: have %s, want %s", have, want) + } + h.Reset() + buf.Reset() + + // Import all block data from Era1. + e, err := era.From(f) + if err != nil { + return fmt.Errorf("error opening era: %w", err) + } + it, err := era.NewIterator(e) + if err != nil { + return fmt.Errorf("error making era reader: %w", err) + } + for it.Next() { + block, err := it.Block() + if err != nil { + return fmt.Errorf("error reading block %d: %w", it.Number(), err) + } + if block.Number().BitLen() == 0 { + continue // skip genesis + } + receipts, err := it.Receipts() + if err != nil { + return fmt.Errorf("error reading receipts %d: %w", it.Number(), err) + } + if status, err := chain.HeaderChain().InsertHeaderChain([]*types.Header{block.Header()}, start, forker); err != nil { + return fmt.Errorf("error inserting header %d: %w", it.Number(), err) + } else if status != core.CanonStatTy { + return fmt.Errorf("error inserting header %d, not canon: %v", it.Number(), status) + } + if _, err := chain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{receipts}, 2^64-1); err != nil { + return fmt.Errorf("error inserting body %d: %w", it.Number(), err) + } + imported += 1 + + // Give the user some feedback that something is happening. + if time.Since(reported) >= 8*time.Second { + log.Info("Importing Era files", "head", it.Number(), "imported", imported, "elapsed", common.PrettyDuration(time.Since(start))) + imported = 0 + reported = time.Now() + } + } + return nil + }() + if err != nil { + return err + } + } + + return nil +} + func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block { head := chain.CurrentBlock() for i, block := range blocks { @@ -297,6 +401,93 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las return nil } +// ExportHistory exports blockchain history into the specified directory, +// following the Era format. +func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) error { + log.Info("Exporting blockchain history", "dir", dir) + if head := bc.CurrentBlock().Number.Uint64(); head < last { + log.Warn("Last block beyond head, setting last = head", "head", head, "last", last) + last = head + } + network := "unknown" + if name, ok := params.NetworkNames[bc.Config().ChainID.String()]; ok { + network = name + } + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return fmt.Errorf("error creating output directory: %w", err) + } + var ( + start = time.Now() + reported = time.Now() + h = sha256.New() + buf = bytes.NewBuffer(nil) + checksums []string + ) + for i := first; i <= last; i += step { + err := func() error { + filename := filepath.Join(dir, era.Filename(network, int(i/step), common.Hash{})) + f, err := os.Create(filename) + if err != nil { + return fmt.Errorf("could not create era file: %w", err) + } + defer f.Close() + + w := era.NewBuilder(f) + for j := uint64(0); j < step && j <= last-i; j++ { + var ( + n = i + j + block = bc.GetBlockByNumber(n) + ) + if block == nil { + return fmt.Errorf("export failed on #%d: not found", n) + } + receipts := bc.GetReceiptsByHash(block.Hash()) + if receipts == nil { + return fmt.Errorf("export failed on #%d: receipts not found", n) + } + td := bc.GetTd(block.Hash(), block.NumberU64()) + if td == nil { + return fmt.Errorf("export failed on #%d: total difficulty not found", n) + } + if err := w.Add(block, receipts, td); err != nil { + return err + } + } + root, err := w.Finalize() + if err != nil { + return fmt.Errorf("export failed to finalize %d: %w", step/i, err) + } + // Set correct filename with root. + os.Rename(filename, filepath.Join(dir, era.Filename(network, int(i/step), root))) + + // Compute checksum of entire Era1. + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + if _, err := io.Copy(h, f); err != nil { + return fmt.Errorf("unable to calculate checksum: %w", err) + } + checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex()) + h.Reset() + buf.Reset() + return nil + }() + if err != nil { + return err + } + if time.Since(reported) >= 8*time.Second { + log.Info("Exporting blocks", "exported", i, "elapsed", common.PrettyDuration(time.Since(start))) + reported = time.Now() + } + } + + os.WriteFile(filepath.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) + + log.Info("Exported blockchain to", "dir", dir) + + return nil +} + // ImportPreimages imports a batch of exported hash preimages into the database. // It's a part of the deprecated functionality, should be removed in the future. func ImportPreimages(db ethdb.Database, fn string) error { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 12ee94d725..574fc9554d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -21,6 +21,7 @@ import ( "context" "crypto/ecdsa" "encoding/hex" + "encoding/json" "errors" "fmt" "math" @@ -36,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" + bparams "github.com/ethereum/go-ethereum/beacon/params" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/fdlimit" "github.com/ethereum/go-ethereum/core" @@ -51,7 +53,7 @@ import ( "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" - "github.com/ethereum/go-ethereum/eth/tracers/directory" + "github.com/ethereum/go-ethereum/eth/tracers/directory/live" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/remotedb" "github.com/ethereum/go-ethereum/ethstats" @@ -70,9 +72,9 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" pcsclite "github.com/gballet/go-libpcsclite" gopsutil "github.com/shirou/gopsutil/mem" "github.com/urfave/cli/v2" @@ -282,6 +284,58 @@ var ( Value: ethconfig.Defaults.TransactionHistory, Category: flags.StateCategory, } + // Beacon client light sync settings + BeaconApiFlag = &cli.StringSliceFlag{ + Name: "beacon.api", + Usage: "Beacon node (CL) light client API URL. This flag can be given multiple times.", + Category: flags.BeaconCategory, + } + BeaconApiHeaderFlag = &cli.StringSliceFlag{ + Name: "beacon.api.header", + Usage: "Pass custom HTTP header fields to the emote beacon node API in \"key:value\" format. This flag can be given multiple times.", + Category: flags.BeaconCategory, + } + BeaconThresholdFlag = &cli.IntFlag{ + Name: "beacon.threshold", + Usage: "Beacon sync committee participation threshold", + Value: bparams.SyncCommitteeSupermajority, + Category: flags.BeaconCategory, + } + BeaconNoFilterFlag = &cli.BoolFlag{ + Name: "beacon.nofilter", + Usage: "Disable future slot signature filter", + Category: flags.BeaconCategory, + } + BeaconConfigFlag = &cli.StringFlag{ + Name: "beacon.config", + Usage: "Beacon chain config YAML file", + Category: flags.BeaconCategory, + } + BeaconGenesisRootFlag = &cli.StringFlag{ + Name: "beacon.genesis.gvroot", + Usage: "Beacon chain genesis validators root", + Category: flags.BeaconCategory, + } + BeaconGenesisTimeFlag = &cli.Uint64Flag{ + Name: "beacon.genesis.time", + Usage: "Beacon chain genesis time", + Category: flags.BeaconCategory, + } + BeaconCheckpointFlag = &cli.StringFlag{ + Name: "beacon.checkpoint", + Usage: "Beacon chain weak subjectivity checkpoint block hash", + Category: flags.BeaconCategory, + } + BlsyncApiFlag = &cli.StringFlag{ + Name: "blsync.engine.api", + Usage: "Target EL engine API URL", + Category: flags.BeaconCategory, + } + BlsyncJWTSecretFlag = &cli.StringFlag{ + Name: "blsync.jwtsecret", + Usage: "Path to a JWT secret to use for target engine API endpoint", + Category: flags.BeaconCategory, + } // Transaction pool settings TxPoolLocalsFlag = &cli.StringFlag{ Name: "txpool.locals", @@ -426,11 +480,6 @@ var ( } // Miner settings - MiningEnabledFlag = &cli.BoolFlag{ - Name: "mine", - Usage: "Enable mining", - Category: flags.MinerCategory, - } MinerGasLimitFlag = &cli.Uint64Flag{ Name: "miner.gaslimit", Usage: "Target gas ceiling for mined blocks", @@ -443,11 +492,6 @@ var ( Value: ethconfig.Defaults.Miner.GasPrice, Category: flags.MinerCategory, } - MinerEtherbaseFlag = &cli.StringFlag{ - Name: "miner.etherbase", - Usage: "0x prefixed public address for block mining rewards", - Category: flags.MinerCategory, - } MinerExtraDataFlag = &cli.StringFlag{ Name: "miner.extradata", Usage: "Block extra data set by the miner (default = client version)", @@ -459,10 +503,9 @@ var ( Value: ethconfig.Defaults.Miner.Recommit, Category: flags.MinerCategory, } - MinerNewPayloadTimeout = &cli.DurationFlag{ - Name: "miner.newpayload-timeout", - Usage: "Specify the maximum time allowance for creating a new payload", - Value: ethconfig.Defaults.Miner.NewPayloadTimeout, + MinerPendingFeeRecipientFlag = &cli.StringFlag{ + Name: "miner.pending.feeRecipient", + Usage: "0x prefixed public address for the pending block producer (not used for actual block production)", Category: flags.MinerCategory, } @@ -502,7 +545,11 @@ var ( Usage: "Name of tracer which should record internal VM operations (costly)", Category: flags.VMCategory, } - + VMTraceConfigFlag = &cli.StringFlag{ + Name: "vmtrace.config", + Usage: "Tracer configuration (JSON)", + Category: flags.VMCategory, + } // API options. RPCGlobalGasCapFlag = &cli.Uint64Flag{ Name: "rpc.gascap", @@ -826,12 +873,6 @@ var ( Usage: "Enable metrics collection and reporting", Category: flags.MetricsCategory, } - MetricsEnabledExpensiveFlag = &cli.BoolFlag{ - Name: "metrics.expensive", - Usage: "Enable expensive metrics collection and reporting", - Category: flags.MetricsCategory, - } - // MetricsHTTPFlag defines the endpoint for a stand-alone metrics HTTP endpoint. // Since the pprof service enables sensitive/vulnerable behavior, this allows a user // to enable a public-OK metrics endpoint without having to worry about ALSO exposing @@ -1274,19 +1315,23 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error // setEtherbase retrieves the etherbase from the directly specified command line flags. func setEtherbase(ctx *cli.Context, cfg *ethconfig.Config) { - if !ctx.IsSet(MinerEtherbaseFlag.Name) { + if ctx.IsSet(MinerEtherbaseFlag.Name) { + log.Warn("Option --miner.etherbase is deprecated as the etherbase is set by the consensus client post-merge") return } - addr := ctx.String(MinerEtherbaseFlag.Name) + if !ctx.IsSet(MinerPendingFeeRecipientFlag.Name) { + return + } + addr := ctx.String(MinerPendingFeeRecipientFlag.Name) if strings.HasPrefix(addr, "0x") || strings.HasPrefix(addr, "0X") { addr = addr[2:] } b, err := hex.DecodeString(addr) if err != nil || len(b) != common.AddressLength { - Fatalf("-%s: invalid etherbase address %q", MinerEtherbaseFlag.Name, addr) + Fatalf("-%s: invalid pending block producer address %q", MinerPendingFeeRecipientFlag.Name, addr) return } - cfg.Miner.Etherbase = common.BytesToAddress(b) + cfg.Miner.PendingFeeRecipient = common.BytesToAddress(b) } // MakePasswordList reads password lines from the file specified by the global --password flag. @@ -1502,6 +1547,9 @@ func setTxPool(ctx *cli.Context, cfg *legacypool.Config) { } func setMiner(ctx *cli.Context, cfg *miner.Config) { + if ctx.Bool(MiningEnabledFlag.Name) { + log.Warn("The flag --mine is deprecated and will be removed") + } if ctx.IsSet(MinerExtraDataFlag.Name) { cfg.ExtraData = []byte(ctx.String(MinerExtraDataFlag.Name)) } @@ -1514,8 +1562,9 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.IsSet(MinerRecommitIntervalFlag.Name) { cfg.Recommit = ctx.Duration(MinerRecommitIntervalFlag.Name) } - if ctx.IsSet(MinerNewPayloadTimeout.Name) { - cfg.NewPayloadTimeout = ctx.Duration(MinerNewPayloadTimeout.Name) + if ctx.IsSet(MinerNewPayloadTimeoutFlag.Name) { + log.Warn("The flag --miner.newpayload-timeout is deprecated and will be removed, please use --miner.recommit") + cfg.Recommit = ctx.Duration(MinerNewPayloadTimeoutFlag.Name) } } @@ -1674,6 +1723,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 { cfg.TransactionHistory = 0 log.Warn("Disabled transaction unindexing for archive node") + + cfg.StateScheme = rawdb.HashScheme + log.Warn("Forcing hash state-scheme for archive mode") } if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) { cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100 @@ -1789,8 +1841,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // Figure out the dev account address. // setEtherbase has been called above, configuring the miner address from command line flags. - if cfg.Miner.Etherbase != (common.Address{}) { - developer = accounts.Account{Address: cfg.Miner.Etherbase} + if cfg.Miner.PendingFeeRecipient != (common.Address{}) { + developer = accounts.Account{Address: cfg.Miner.PendingFeeRecipient} } else if accs := ks.Accounts(); len(accs) > 0 { developer = ks.Accounts()[0] } else { @@ -1801,7 +1853,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } // Make sure the address is configured as fee recipient, otherwise // the miner will fail to start. - cfg.Miner.Etherbase = developer.Address + cfg.Miner.PendingFeeRecipient = developer.Address if err := ks.Unlock(developer, passphrase); err != nil { Fatalf("Failed to unlock developer account: %v", err) @@ -1848,6 +1900,18 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if err := kzg4844.UseCKZG(ctx.String(CryptoKZGFlag.Name) == "ckzg"); err != nil { Fatalf("Failed to set KZG library implementation to %s: %v", ctx.String(CryptoKZGFlag.Name), err) } + // VM tracing config. + if ctx.IsSet(VMTraceFlag.Name) { + if name := ctx.String(VMTraceFlag.Name); name != "" { + var config string + if ctx.IsSet(VMTraceConfigFlag.Name) { + config = ctx.String(VMTraceConfigFlag.Name) + } + + cfg.VMTrace = name + cfg.VMTraceConfig = config + } + } } // SetDNSDiscoveryDefaults configures DNS discovery with the given URL if @@ -2128,7 +2192,11 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh vmcfg := vm.Config{EnablePreimageRecording: ctx.Bool(VMEnableDebugFlag.Name)} if ctx.IsSet(VMTraceFlag.Name) { if name := ctx.String(VMTraceFlag.Name); name != "" { - t, err := directory.LiveDirectory.New(name) + var config string + if ctx.IsSet(VMTraceConfigFlag.Name) { + config = ctx.String(VMTraceConfigFlag.Name) + } + t, err := live.Directory.New(name, json.RawMessage(config)) if err != nil { Fatalf("Failed to create tracer %q: %v", name, err) } @@ -2161,8 +2229,8 @@ func MakeConsolePreloads(ctx *cli.Context) []string { } // MakeTrieDatabase constructs a trie database based on the configured scheme. -func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool, isVerkle bool) *trie.Database { - config := &trie.Config{ +func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool, isVerkle bool) *triedb.Database { + config := &triedb.Config{ Preimages: preimage, IsVerkle: isVerkle, } @@ -2175,12 +2243,12 @@ func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, read // ignore the parameter silently. TODO(rjl493456442) // please config it if read mode is implemented. config.HashDB = hashdb.Defaults - return trie.NewDatabase(disk, config) + return triedb.NewDatabase(disk, config) } if readOnly { config.PathDB = pathdb.ReadOnly } else { config.PathDB = pathdb.Defaults } - return trie.NewDatabase(disk, config) + return triedb.NewDatabase(disk, config) } diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index 243abd8311..1dfd1a5f89 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -47,6 +47,9 @@ var DeprecatedFlags = []cli.Flag{ LightNoSyncServeFlag, LogBacktraceAtFlag, LogDebugFlag, + MinerNewPayloadTimeoutFlag, + MinerEtherbaseFlag, + MiningEnabledFlag, } var ( @@ -90,35 +93,35 @@ var ( Name: "light.serve", Usage: "Maximum percentage of time allowed for serving LES requests (deprecated)", Value: ethconfig.Defaults.LightServ, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightIngressFlag = &cli.IntFlag{ Name: "light.ingress", Usage: "Incoming bandwidth limit for serving light clients (deprecated)", Value: ethconfig.Defaults.LightIngress, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightEgressFlag = &cli.IntFlag{ Name: "light.egress", Usage: "Outgoing bandwidth limit for serving light clients (deprecated)", Value: ethconfig.Defaults.LightEgress, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightMaxPeersFlag = &cli.IntFlag{ Name: "light.maxpeers", Usage: "Maximum number of light clients to serve, or light servers to attach to (deprecated)", Value: ethconfig.Defaults.LightPeers, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightNoPruneFlag = &cli.BoolFlag{ Name: "light.nopruning", Usage: "Disable ancient light chain data pruning (deprecated)", - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightNoSyncServeFlag = &cli.BoolFlag{ Name: "light.nosyncserve", Usage: "Enables serving light clients before syncing (deprecated)", - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } // Deprecated November 2023 LogBacktraceAtFlag = &cli.StringFlag{ @@ -132,6 +135,28 @@ var ( Usage: "Prepends log messages with call-site location (deprecated)", Category: flags.DeprecatedCategory, } + // Deprecated February 2024 + MinerNewPayloadTimeoutFlag = &cli.DurationFlag{ + Name: "miner.newpayload-timeout", + Usage: "Specify the maximum time allowance for creating a new payload (deprecated)", + Value: ethconfig.Defaults.Miner.Recommit, + Category: flags.DeprecatedCategory, + } + MinerEtherbaseFlag = &cli.StringFlag{ + Name: "miner.etherbase", + Usage: "0x prefixed public address for block mining rewards (deprecated)", + Category: flags.DeprecatedCategory, + } + MiningEnabledFlag = &cli.BoolFlag{ + Name: "mine", + Usage: "Enable mining (deprecated)", + Category: flags.DeprecatedCategory, + } + MetricsEnabledExpensiveFlag = &cli.BoolFlag{ + Name: "metrics.expensive", + Usage: "Enable expensive metrics collection and reporting (deprecated)", + Category: flags.DeprecatedCategory, + } ) // showDeprecated displays deprecated flags that will be soon removed from the codebase. diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go new file mode 100644 index 0000000000..1d8e48344a --- /dev/null +++ b/cmd/utils/history_test.go @@ -0,0 +1,185 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package utils + +import ( + "bytes" + "crypto/sha256" + "io" + "math/big" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" +) + +var ( + count uint64 = 128 + step uint64 = 16 +) + +func TestHistoryImportAndExport(t *testing.T) { + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}}, + } + signer = types.LatestSigner(genesis.Config) + ) + + // Generate chain. + db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) { + if i == 0 { + return + } + tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(i - 1), + GasTipCap: common.Big0, + GasFeeCap: g.PrevBlock(0).BaseFee(), + Gas: 50000, + To: &common.Address{0xaa}, + Value: big.NewInt(int64(i)), + Data: nil, + AccessList: nil, + }) + if err != nil { + t.Fatalf("error creating tx: %v", err) + } + g.AddTx(tx) + }) + + // Initialize BlockChain. + chain, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if _, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("error insterting chain: %v", err) + } + + // Make temp directory for era files. + dir, err := os.MkdirTemp("", "history-export-test") + if err != nil { + t.Fatalf("error creating temp test directory: %v", err) + } + defer os.RemoveAll(dir) + + // Export history to temp directory. + if err := ExportHistory(chain, dir, 0, count, step); err != nil { + t.Fatalf("error exporting history: %v", err) + } + + // Read checksums. + b, err := os.ReadFile(filepath.Join(dir, "checksums.txt")) + if err != nil { + t.Fatalf("failed to read checksums: %v", err) + } + checksums := strings.Split(string(b), "\n") + + // Verify each Era. + entries, _ := era.ReadDir(dir, "mainnet") + for i, filename := range entries { + func() { + f, err := os.Open(filepath.Join(dir, filename)) + if err != nil { + t.Fatalf("error opening era file: %v", err) + } + var ( + h = sha256.New() + buf = bytes.NewBuffer(nil) + ) + if _, err := io.Copy(h, f); err != nil { + t.Fatalf("unable to recalculate checksum: %v", err) + } + if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want { + t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) + } + e, err := era.From(f) + if err != nil { + t.Fatalf("error opening era: %v", err) + } + defer e.Close() + it, err := era.NewIterator(e) + if err != nil { + t.Fatalf("error making era reader: %v", err) + } + for j := 0; it.Next(); j++ { + n := i*int(step) + j + if it.Error() != nil { + t.Fatalf("error reading block entry %d: %v", n, it.Error()) + } + block, receipts, err := it.BlockAndReceipts() + if err != nil { + t.Fatalf("error reading block entry %d: %v", n, err) + } + want := chain.GetBlockByNumber(uint64(n)) + if want, got := uint64(n), block.NumberU64(); want != got { + t.Fatalf("blocks out of order: want %d, got %d", want, got) + } + if want.Hash() != block.Hash() { + t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex()) + } + if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() { + t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got) + } + if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() { + t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got) + } + if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() { + t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got) + } + } + }() + } + + // Now import Era. + freezer := t.TempDir() + db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false) + if err != nil { + panic(err) + } + t.Cleanup(func() { + db2.Close() + }) + + genesis.MustCommit(db2, triedb.NewDatabase(db, triedb.HashDefaults)) + imported, err := core.NewBlockChain(db2, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if err := ImportHistory(imported, db2, dir, "mainnet"); err != nil { + t.Fatalf("failed to import chain: %v", err) + } + if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { + t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash()) + } +} diff --git a/common/bitutil/bitutil.go b/common/bitutil/bitutil.go index cd3e72169f..a18a6d18ee 100644 --- a/common/bitutil/bitutil.go +++ b/common/bitutil/bitutil.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Adapted from: https://golang.org/src/crypto/cipher/xor.go +// Adapted from: https://go.dev/src/crypto/subtle/xor_generic.go // Package bitutil implements fast bitwise operations. package bitutil diff --git a/common/bitutil/bitutil_test.go b/common/bitutil/bitutil_test.go index 307bf731f7..12f3fe24a6 100644 --- a/common/bitutil/bitutil_test.go +++ b/common/bitutil/bitutil_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Adapted from: https://golang.org/src/crypto/cipher/xor_test.go +// Adapted from: https://go.dev/src/crypto/subtle/xor_test.go package bitutil diff --git a/common/math/big.go b/common/math/big.go index 013c0ba4b6..721b297c9c 100644 --- a/common/math/big.go +++ b/common/math/big.go @@ -224,7 +224,7 @@ func ReadBits(bigint *big.Int, buf []byte) { } } -// U256 encodes as a 256 bit two's complement number. This operation is destructive. +// U256 encodes x as a 256 bit two's complement number. This operation is destructive. func U256(x *big.Int) *big.Int { return x.And(x, tt256m1) } @@ -255,14 +255,15 @@ func S256(x *big.Int) *big.Int { // // Courtesy @karalabe and @chfast func Exp(base, exponent *big.Int) *big.Int { + copyBase := new(big.Int).Set(base) result := big.NewInt(1) for _, word := range exponent.Bits() { for i := 0; i < wordBits; i++ { if word&1 == 1 { - U256(result.Mul(result, base)) + U256(result.Mul(result, copyBase)) } - U256(base.Mul(base, base)) + U256(copyBase.Mul(copyBase, copyBase)) word >>= 1 } } diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 9d84e7d934..c3dcae3ce2 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip1559" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" @@ -358,7 +359,7 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types. // Convert amount from gwei to wei. amount := new(uint256.Int).SetUint64(w.Amount) amount = amount.Mul(amount, uint256.NewInt(params.GWei)) - stateDB.AddBalance(w.Address, amount, state.BalanceIncreaseWithdrawal) + stateDB.AddBalance(w.Address, amount, tracing.BalanceIncreaseWithdrawal) } // No block reward which is issued by consensus layer instead. } diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index c693189ea5..59f0e96ebe 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - lru "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go index 7cd5919c5e..8ef8dbffa9 100644 --- a/consensus/clique/clique_test.go +++ b/consensus/clique/clique_test.go @@ -47,7 +47,7 @@ func TestReimportMirroredState(t *testing.T) { genspec := &core.Genesis{ Config: params.AllCliqueProtocolChanges, ExtraData: make([]byte, extraVanity+common.AddressLength+extraSeal), - Alloc: map[common.Address]core.GenesisAccount{ + Alloc: map[common.Address]types.Account{ addr: {Balance: big.NewInt(10000000000000000)}, }, BaseFee: big.NewInt(params.InitialBaseFee), diff --git a/consensus/consensus.go b/consensus/consensus.go index 3a2c2d2229..5cc052cb0f 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -119,11 +119,3 @@ type Engine interface { // Close terminates any background threads maintained by the consensus engine. Close() error } - -// PoW is a consensus engine based on proof-of-work. -type PoW interface { - Engine - - // Hashrate returns the current mining hashrate of a PoW consensus engine. - Hashrate() float64 -} diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index d254ec38cf..5d4fc56665 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" @@ -589,10 +590,10 @@ func accumulateRewards(config *params.ChainConfig, stateDB *state.StateDB, heade r.Sub(r, hNum) r.Mul(r, blockReward) r.Div(r, u256_8) - stateDB.AddBalance(uncle.Coinbase, r, state.BalanceIncreaseRewardMineUncle) + stateDB.AddBalance(uncle.Coinbase, r, tracing.BalanceIncreaseRewardMineUncle) r.Div(blockReward, u256_32) reward.Add(reward, r) } - stateDB.AddBalance(header.Coinbase, reward, state.BalanceIncreaseRewardMineBlock) + stateDB.AddBalance(header.Coinbase, reward, tracing.BalanceIncreaseRewardMineBlock) } diff --git a/consensus/merger.go b/consensus/merger.go deleted file mode 100644 index ffbcbf2b85..0000000000 --- a/consensus/merger.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2021 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package consensus - -import ( - "fmt" - "sync" - - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" -) - -// transitionStatus describes the status of eth1/2 transition. This switch -// between modes is a one-way action which is triggered by corresponding -// consensus-layer message. -type transitionStatus struct { - LeftPoW bool // The flag is set when the first NewHead message received - EnteredPoS bool // The flag is set when the first FinalisedBlock message received -} - -// Merger is an internal help structure used to track the eth1/2 transition status. -// It's a common structure can be used in both full node and light client. -type Merger struct { - db ethdb.KeyValueStore - status transitionStatus - mu sync.RWMutex -} - -// NewMerger creates a new Merger which stores its transition status in the provided db. -func NewMerger(db ethdb.KeyValueStore) *Merger { - var status transitionStatus - blob := rawdb.ReadTransitionStatus(db) - if len(blob) != 0 { - if err := rlp.DecodeBytes(blob, &status); err != nil { - log.Crit("Failed to decode the transition status", "err", err) - } - } - return &Merger{ - db: db, - status: status, - } -} - -// ReachTTD is called whenever the first NewHead message received -// from the consensus-layer. -func (m *Merger) ReachTTD() { - m.mu.Lock() - defer m.mu.Unlock() - - if m.status.LeftPoW { - return - } - m.status = transitionStatus{LeftPoW: true} - blob, err := rlp.EncodeToBytes(m.status) - if err != nil { - panic(fmt.Sprintf("Failed to encode the transition status: %v", err)) - } - rawdb.WriteTransitionStatus(m.db, blob) - log.Info("Left PoW stage") -} - -// FinalizePoS is called whenever the first FinalisedBlock message received -// from the consensus-layer. -func (m *Merger) FinalizePoS() { - m.mu.Lock() - defer m.mu.Unlock() - - if m.status.EnteredPoS { - return - } - m.status = transitionStatus{LeftPoW: true, EnteredPoS: true} - blob, err := rlp.EncodeToBytes(m.status) - if err != nil { - panic(fmt.Sprintf("Failed to encode the transition status: %v", err)) - } - rawdb.WriteTransitionStatus(m.db, blob) - log.Info("Entered PoS stage") -} - -// TDDReached reports whether the chain has left the PoW stage. -func (m *Merger) TDDReached() bool { - m.mu.RLock() - defer m.mu.RUnlock() - - return m.status.LeftPoW -} - -// PoSFinalized reports whether the chain has entered the PoS stage. -func (m *Merger) PoSFinalized() bool { - m.mu.RLock() - defer m.mu.RUnlock() - - return m.status.EnteredPoS -} diff --git a/consensus/misc/dao.go b/consensus/misc/dao.go index 803b577ea2..45669d0bce 100644 --- a/consensus/misc/dao.go +++ b/consensus/misc/dao.go @@ -22,6 +22,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" @@ -81,7 +82,7 @@ func ApplyDAOHardFork(statedb *state.StateDB) { // Move every DAO account and extra-balance account funds into the refund contract for _, addr := range params.DAODrainList() { - statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr), state.BalanceIncreaseDaoContract) - statedb.SetBalance(addr, new(uint256.Int), state.BalanceDecreaseDaoAccount) + statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr), tracing.BalanceIncreaseDaoContract) + statedb.SetBalance(addr, new(uint256.Int), tracing.BalanceDecreaseDaoAccount) } } diff --git a/console/console.go b/console/console.go index cdee53684e..5acb4cdccb 100644 --- a/console/console.go +++ b/console/console.go @@ -325,9 +325,6 @@ func (c *Console) Welcome() { // Print some generic Geth metadata if res, err := c.jsre.Run(` var message = "instance: " + web3.version.node + "\n"; - try { - message += "coinbase: " + eth.coinbase + "\n"; - } catch (err) {} message += "at block: " + eth.blockNumber + " (" + new Date(1000 * eth.getBlock(eth.blockNumber).timestamp) + ")\n"; try { message += " datadir: " + admin.datadir + "\n"; diff --git a/console/console_test.go b/console/console_test.go index a13be6a99d..d210a993c2 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -96,7 +96,7 @@ func newTester(t *testing.T, confOverride func(*ethconfig.Config)) *tester { ethConf := ðconfig.Config{ Genesis: core.DeveloperGenesisBlock(11_500_000, nil), Miner: miner.Config{ - Etherbase: common.HexToAddress(testAddress), + PendingFeeRecipient: common.HexToAddress(testAddress), }, } if confOverride != nil { @@ -152,8 +152,7 @@ func (env *tester) Close(t *testing.T) { } // Tests that the node lists the correct welcome message, notably that it contains -// the instance name, coinbase account, block number, data directory and supported -// console modules. +// the instance name, block number, data directory and supported console modules. func TestWelcome(t *testing.T) { tester := newTester(t, nil) defer tester.Close(t) @@ -167,14 +166,14 @@ func TestWelcome(t *testing.T) { if want := fmt.Sprintf("instance: %s", testInstance); !strings.Contains(output, want) { t.Fatalf("console output missing instance: have\n%s\nwant also %s", output, want) } - if want := fmt.Sprintf("coinbase: %s", testAddress); !strings.Contains(output, want) { - t.Fatalf("console output missing coinbase: have\n%s\nwant also %s", output, want) - } if want := "at block: 0"; !strings.Contains(output, want) { t.Fatalf("console output missing sync status: have\n%s\nwant also %s", output, want) } if want := fmt.Sprintf("datadir: %s", tester.workspace); !strings.Contains(output, want) { - t.Fatalf("console output missing coinbase: have\n%s\nwant also %s", output, want) + t.Fatalf("console output missing datadir: have\n%s\nwant also %s", output, want) + } + if want := "modules: "; !strings.Contains(output, want) { + t.Fatalf("console output missing modules: have\n%s\nwant also %s", output, want) } } diff --git a/core/bench_test.go b/core/bench_test.go index 951ce2a08c..97713868a5 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -189,7 +189,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // generator function. gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{benchRootAddr: {Balance: benchRootFunds}}, + Alloc: types.GenesisAlloc{benchRootAddr: {Balance: benchRootFunds}}, } _, chain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), b.N, gen) diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 48bdceff62..2f86b2d751 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -94,7 +94,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { preBlocks []*types.Block postBlocks []*types.Block engine consensus.Engine - merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) ) if isClique { var ( @@ -106,7 +105,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { gspec = &Genesis{ Config: &config, ExtraData: make([]byte, 32+common.AddressLength+crypto.SignatureLength), - Alloc: map[common.Address]GenesisAccount{ + Alloc: map[common.Address]types.Account{ addr: {Balance: big.NewInt(1)}, }, BaseFee: big.NewInt(params.InitialBaseFee), @@ -186,11 +185,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { } chain.InsertChain(preBlocks[i : i+1]) } - - // Make the transition - merger.ReachTTD() - merger.FinalizePoS() - // Verify the blocks after the merging for i := 0; i < len(postBlocks); i++ { _, results := engine.VerifyHeaders(chain, []*types.Header{postHeaders[i]}) diff --git a/core/blockchain.go b/core/blockchain.go index f6d40fe1c1..70ccce50c8 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" @@ -47,10 +48,9 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" - "golang.org/x/exp/slices" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) var ( @@ -62,26 +62,26 @@ var ( chainInfoGauge = metrics.NewRegisteredGaugeInfo("chain/info", nil) - accountReadTimer = metrics.NewRegisteredTimer("chain/account/reads", nil) - accountHashTimer = metrics.NewRegisteredTimer("chain/account/hashes", nil) - accountUpdateTimer = metrics.NewRegisteredTimer("chain/account/updates", nil) - accountCommitTimer = metrics.NewRegisteredTimer("chain/account/commits", nil) + accountReadTimer = metrics.NewRegisteredResettingTimer("chain/account/reads", nil) + accountHashTimer = metrics.NewRegisteredResettingTimer("chain/account/hashes", nil) + accountUpdateTimer = metrics.NewRegisteredResettingTimer("chain/account/updates", nil) + accountCommitTimer = metrics.NewRegisteredResettingTimer("chain/account/commits", nil) - storageReadTimer = metrics.NewRegisteredTimer("chain/storage/reads", nil) - storageHashTimer = metrics.NewRegisteredTimer("chain/storage/hashes", nil) - storageUpdateTimer = metrics.NewRegisteredTimer("chain/storage/updates", nil) - storageCommitTimer = metrics.NewRegisteredTimer("chain/storage/commits", nil) + storageReadTimer = metrics.NewRegisteredResettingTimer("chain/storage/reads", nil) + storageHashTimer = metrics.NewRegisteredResettingTimer("chain/storage/hashes", nil) + storageUpdateTimer = metrics.NewRegisteredResettingTimer("chain/storage/updates", nil) + storageCommitTimer = metrics.NewRegisteredResettingTimer("chain/storage/commits", nil) - snapshotAccountReadTimer = metrics.NewRegisteredTimer("chain/snapshot/account/reads", nil) - snapshotStorageReadTimer = metrics.NewRegisteredTimer("chain/snapshot/storage/reads", nil) - snapshotCommitTimer = metrics.NewRegisteredTimer("chain/snapshot/commits", nil) + snapshotAccountReadTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/account/reads", nil) + snapshotStorageReadTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/storage/reads", nil) + snapshotCommitTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/commits", nil) - triedbCommitTimer = metrics.NewRegisteredTimer("chain/triedb/commits", nil) + triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) - blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) - blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil) - blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil) - blockWriteTimer = metrics.NewRegisteredTimer("chain/write", nil) + blockInsertTimer = metrics.NewRegisteredResettingTimer("chain/inserts", nil) + blockValidationTimer = metrics.NewRegisteredResettingTimer("chain/validation", nil) + blockExecutionTimer = metrics.NewRegisteredResettingTimer("chain/execution", nil) + blockWriteTimer = metrics.NewRegisteredResettingTimer("chain/write", nil) blockReorgMeter = metrics.NewRegisteredMeter("chain/reorg/executes", nil) blockReorgAddMeter = metrics.NewRegisteredMeter("chain/reorg/add", nil) @@ -97,13 +97,11 @@ var ( ) const ( - bodyCacheLimit = 256 - blockCacheLimit = 256 - receiptsCacheLimit = 32 - txLookupCacheLimit = 1024 - maxFutureBlocks = 256 - maxTimeFutureBlocks = 30 - TriesInMemory = 128 + bodyCacheLimit = 256 + blockCacheLimit = 256 + receiptsCacheLimit = 32 + txLookupCacheLimit = 1024 + TriesInMemory = 128 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. // @@ -149,8 +147,8 @@ type CacheConfig struct { } // triedbConfig derives the configures for trie database. -func (c *CacheConfig) triedbConfig() *trie.Config { - config := &trie.Config{Preimages: c.Preimages} +func (c *CacheConfig) triedbConfig() *triedb.Config { + config := &triedb.Config{Preimages: c.Preimages} if c.StateScheme == rawdb.HashScheme { config.HashDB = &hashdb.Config{ CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, @@ -185,20 +183,6 @@ func DefaultCacheConfigWithScheme(scheme string) *CacheConfig { return &config } -// BlockchainLogger is used to collect traces during chain processing. -// Please make a copy of the referenced types if you intend to retain them. -type BlockchainLogger interface { - vm.EVMLogger - state.StateLogger - // OnBlockStart is called before executing `block`. - // `td` is the total difficulty prior to `block`. - OnBlockStart(block *types.Block, td *big.Int, finalized *types.Header, safe *types.Header, chainConfig *params.ChainConfig) - OnBlockEnd(err error) - OnGenesisBlock(genesis *types.Block, alloc GenesisAlloc, chainConfig *params.ChainConfig) - OnBeaconBlockRootStart(root common.Hash) - OnBeaconBlockRootEnd() -} - // txLookup is wrapper over transaction lookup along with the corresponding // transaction object. type txLookup struct { @@ -230,7 +214,7 @@ type BlockChain struct { gcproc time.Duration // Accumulates canonical block processing for trie dumping lastWrite uint64 // Last block when the state was flushed flushInterval atomic.Int64 // Time interval (processing time) after which to flush a state - triedb *trie.Database // The database handler for maintaining trie nodes. + triedb *triedb.Database // The database handler for maintaining trie nodes. stateCache state.Database // State database to reuse between imports (contains state cache) txIndexer *txIndexer // Transaction indexer, might be nil if not enabled @@ -259,9 +243,6 @@ type BlockChain struct { blockCache *lru.Cache[common.Hash, *types.Block] txLookupCache *lru.Cache[common.Hash, txLookup] - // future blocks are blocks added for later processing - futureBlocks *lru.Cache[common.Hash, *types.Block] - wg sync.WaitGroup quit chan struct{} // shutdown signal, closed in Stop. stopping atomic.Bool // false if chain is running, true when stopped @@ -273,7 +254,7 @@ type BlockChain struct { processor Processor // Block transaction processor interface forker *ForkChoice vmConfig vm.Config - logger BlockchainLogger + logger *tracing.Hooks } // NewBlockChain returns a fully initialised block chain using information @@ -284,15 +265,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis cacheConfig = defaultCacheConfig } // Open trie database with provided config - triedb := trie.NewDatabase(db, cacheConfig.triedbConfig()) - var logger BlockchainLogger - if vmConfig.Tracer != nil { - l, ok := vmConfig.Tracer.(BlockchainLogger) - if !ok { - return nil, errors.New("only extended tracers are supported for live mode") - } - logger = l - } + triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig()) + // Setup the genesis block, commit the provided genesis specification // to database if the genesis block is not present yet, or load the // stored one from database. @@ -321,10 +295,9 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit), - futureBlocks: lru.NewCache[common.Hash, *types.Block](maxFutureBlocks), engine: engine, vmConfig: vmConfig, - logger: logger, + logger: vmConfig.Tracer, } bc.flushInterval.Store(int64(cacheConfig.TrieTimeLimit)) bc.forker = NewForkChoice(bc, shouldPreserve) @@ -451,7 +424,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis } } - if bc.logger != nil { + if bc.logger != nil && bc.logger.OnGenesisBlock != nil { if block := bc.CurrentBlock(); block.Number.Uint64() == 0 { alloc, err := getGenesisState(bc.db, block.Hash()) if err != nil { @@ -462,7 +435,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis return nil, fmt.Errorf("live blockchain tracer requires genesis alloc to be set") } - bc.logger.OnGenesisBlock(bc.genesisBlock, alloc, bc.chainConfig) + bc.logger.OnGenesisBlock(bc.genesisBlock, alloc) } } @@ -487,11 +460,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis } bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root) } - - // Start future block processor. - bc.wg.Add(1) - go bc.updateFutureBlocks() - // Rewind the chain in case of an incompatible config upgrade. if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) @@ -502,6 +470,9 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis } rawdb.WriteChainConfig(db, genesisHash, chainConfig) } + if bc.logger != nil && bc.logger.OnBlockchainInit != nil { + bc.logger.OnBlockchainInit(chainConfig) + } // Start tx indexer if it's enabled. if txLookupLimit != nil { bc.txIndexer = newTxIndexer(*txLookupLimit, bc) @@ -666,6 +637,172 @@ func (bc *BlockChain) SetSafe(header *types.Header) { } } +// rewindPathHead implements the logic of rewindHead in the context of hash scheme. +func (bc *BlockChain) rewindHashHead(head *types.Header, root common.Hash) (*types.Header, uint64) { + var ( + limit uint64 // The oldest block that will be searched for this rewinding + beyondRoot = root == common.Hash{} // Flag whether we're beyond the requested root (no root, always true) + pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot point state + rootNumber uint64 // Associated block number of requested root + + start = time.Now() // Timestamp the rewinding is restarted + logged = time.Now() // Timestamp last progress log was printed + ) + // The oldest block to be searched is determined by the pivot block or a constant + // searching threshold. The rationale behind this is as follows: + // + // - Snap sync is selected if the pivot block is available. The earliest available + // state is the pivot block itself, so there is no sense in going further back. + // + // - Full sync is selected if the pivot block does not exist. The hash database + // periodically flushes the state to disk, and the used searching threshold is + // considered sufficient to find a persistent state, even for the testnet. It + // might be not enough for a chain that is nearly empty. In the worst case, + // the entire chain is reset to genesis, and snap sync is re-enabled on top, + // which is still acceptable. + if pivot != nil { + limit = *pivot + } else if head.Number.Uint64() > params.FullImmutabilityThreshold { + limit = head.Number.Uint64() - params.FullImmutabilityThreshold + } + for { + logger := log.Trace + if time.Since(logged) > time.Second*8 { + logged = time.Now() + logger = log.Info + } + logger("Block state missing, rewinding further", "number", head.Number, "hash", head.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) + + // If a root threshold was requested but not yet crossed, check + if !beyondRoot && head.Root == root { + beyondRoot, rootNumber = true, head.Number.Uint64() + } + // If search limit is reached, return the genesis block as the + // new chain head. + if head.Number.Uint64() < limit { + log.Info("Rewinding limit reached, resetting to genesis", "number", head.Number, "hash", head.Hash(), "limit", limit) + return bc.genesisBlock.Header(), rootNumber + } + // If the associated state is not reachable, continue searching + // backwards until an available state is found. + if !bc.HasState(head.Root) { + // If the chain is gapped in the middle, return the genesis + // block as the new chain head. + parent := bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) + if parent == nil { + log.Error("Missing block in the middle, resetting to genesis", "number", head.Number.Uint64()-1, "hash", head.ParentHash) + return bc.genesisBlock.Header(), rootNumber + } + head = parent + + // If the genesis block is reached, stop searching. + if head.Number.Uint64() == 0 { + log.Info("Genesis block reached", "number", head.Number, "hash", head.Hash()) + return head, rootNumber + } + continue // keep rewinding + } + // Once the available state is found, ensure that the requested root + // has already been crossed. If not, continue rewinding. + if beyondRoot || head.Number.Uint64() == 0 { + log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) + return head, rootNumber + } + log.Debug("Skipping block with threshold state", "number", head.Number, "hash", head.Hash(), "root", head.Root) + head = bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) // Keep rewinding + } +} + +// rewindPathHead implements the logic of rewindHead in the context of path scheme. +func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*types.Header, uint64) { + var ( + pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot block + rootNumber uint64 // Associated block number of requested root + + // BeyondRoot represents whether the requested root is already + // crossed. The flag value is set to true if the root is empty. + beyondRoot = root == common.Hash{} + + // noState represents if the target state requested for search + // is unavailable and impossible to be recovered. + noState = !bc.HasState(root) && !bc.stateRecoverable(root) + + start = time.Now() // Timestamp the rewinding is restarted + logged = time.Now() // Timestamp last progress log was printed + ) + // Rewind the head block tag until an available state is found. + for { + logger := log.Trace + if time.Since(logged) > time.Second*8 { + logged = time.Now() + logger = log.Info + } + logger("Block state missing, rewinding further", "number", head.Number, "hash", head.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) + + // If a root threshold was requested but not yet crossed, check + if !beyondRoot && head.Root == root { + beyondRoot, rootNumber = true, head.Number.Uint64() + } + // If the root threshold hasn't been crossed but the available + // state is reached, quickly determine if the target state is + // possible to be reached or not. + if !beyondRoot && noState && bc.HasState(head.Root) { + beyondRoot = true + log.Info("Disable the search for unattainable state", "root", root) + } + // Check if the associated state is available or recoverable if + // the requested root has already been crossed. + if beyondRoot && (bc.HasState(head.Root) || bc.stateRecoverable(head.Root)) { + break + } + // If pivot block is reached, return the genesis block as the + // new chain head. Theoretically there must be a persistent + // state before or at the pivot block, prevent endless rewinding + // towards the genesis just in case. + if pivot != nil && *pivot >= head.Number.Uint64() { + log.Info("Pivot block reached, resetting to genesis", "number", head.Number, "hash", head.Hash()) + return bc.genesisBlock.Header(), rootNumber + } + // If the chain is gapped in the middle, return the genesis + // block as the new chain head + parent := bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) // Keep rewinding + if parent == nil { + log.Error("Missing block in the middle, resetting to genesis", "number", head.Number.Uint64()-1, "hash", head.ParentHash) + return bc.genesisBlock.Header(), rootNumber + } + head = parent + + // If the genesis block is reached, stop searching. + if head.Number.Uint64() == 0 { + log.Info("Genesis block reached", "number", head.Number, "hash", head.Hash()) + return head, rootNumber + } + } + // Recover if the target state if it's not available yet. + if !bc.HasState(head.Root) { + if err := bc.triedb.Recover(head.Root); err != nil { + log.Crit("Failed to rollback state", "err", err) + } + } + log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) + return head, rootNumber +} + +// rewindHead searches the available states in the database and returns the associated +// block as the new head block. +// +// If the given root is not empty, then the rewind should attempt to pass the specified +// state root and return the associated block number as well. If the root, typically +// representing the state corresponding to snapshot disk layer, is deemed impassable, +// then block number zero is returned, indicating that snapshot recovery is disabled +// and the whole snapshot should be auto-generated in case of head mismatch. +func (bc *BlockChain) rewindHead(head *types.Header, root common.Hash) (*types.Header, uint64) { + if bc.triedb.Scheme() == rawdb.PathScheme { + return bc.rewindPathHead(head, root) + } + return bc.rewindHashHead(head, root) +} + // setHeadBeyondRoot rewinds the local chain to a new head with the extra condition // that the rewind must pass the specified state root. This method is meant to be // used when rewinding with snapshots enabled to ensure that we go back further than @@ -684,79 +821,40 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha } defer bc.chainmu.Unlock() - // Track the block number of the requested root hash - var rootNumber uint64 // (no root == always 0) - - // Retrieve the last pivot block to short circuit rollbacks beyond it and the - // current freezer limit to start nuking id underflown - pivot := rawdb.ReadLastPivotNumber(bc.db) - frozen, _ := bc.db.Ancients() + var ( + // Track the block number of the requested root hash + rootNumber uint64 // (no root == always 0) + // Retrieve the last pivot block to short circuit rollbacks beyond it + // and the current freezer limit to start nuking it's underflown. + pivot = rawdb.ReadLastPivotNumber(bc.db) + ) updateFn := func(db ethdb.KeyValueWriter, header *types.Header) (*types.Header, bool) { // Rewind the blockchain, ensuring we don't end up with a stateless head // block. Note, depth equality is permitted to allow using SetHead as a // chain reparation mechanism without deleting any data! if currentBlock := bc.CurrentBlock(); currentBlock != nil && header.Number.Uint64() <= currentBlock.Number.Uint64() { - newHeadBlock := bc.GetBlock(header.Hash(), header.Number.Uint64()) - if newHeadBlock == nil { - log.Error("Gap in the chain, rewinding to genesis", "number", header.Number, "hash", header.Hash()) - newHeadBlock = bc.genesisBlock - } else { - // Block exists. Keep rewinding until either we find one with state - // or until we exceed the optional threshold root hash - beyondRoot := (root == common.Hash{}) // Flag whether we're beyond the requested root (no root, always true) - - for { - // If a root threshold was requested but not yet crossed, check - if root != (common.Hash{}) && !beyondRoot && newHeadBlock.Root() == root { - beyondRoot, rootNumber = true, newHeadBlock.NumberU64() - } - if !bc.HasState(newHeadBlock.Root()) && !bc.stateRecoverable(newHeadBlock.Root()) { - log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) - if pivot == nil || newHeadBlock.NumberU64() > *pivot { - parent := bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) - if parent != nil { - newHeadBlock = parent - continue - } - log.Error("Missing block in the middle, aiming genesis", "number", newHeadBlock.NumberU64()-1, "hash", newHeadBlock.ParentHash()) - newHeadBlock = bc.genesisBlock - } else { - log.Trace("Rewind passed pivot, aiming genesis", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "pivot", *pivot) - newHeadBlock = bc.genesisBlock - } - } - if beyondRoot || newHeadBlock.NumberU64() == 0 { - if !bc.HasState(newHeadBlock.Root()) && bc.stateRecoverable(newHeadBlock.Root()) { - // Rewind to a block with recoverable state. If the state is - // missing, run the state recovery here. - if err := bc.triedb.Recover(newHeadBlock.Root()); err != nil { - log.Crit("Failed to rollback state", "err", err) // Shouldn't happen - } - log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) - } - break - } - log.Debug("Skipping block with threshold state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "root", newHeadBlock.Root()) - newHeadBlock = bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) // Keep rewinding - } - } + var newHeadBlock *types.Header + newHeadBlock, rootNumber = bc.rewindHead(header, root) rawdb.WriteHeadBlockHash(db, newHeadBlock.Hash()) // Degrade the chain markers if they are explicitly reverted. // In theory we should update all in-memory markers in the // last step, however the direction of SetHead is from high // to low, so it's safe to update in-memory markers directly. - bc.currentBlock.Store(newHeadBlock.Header()) - headBlockGauge.Update(int64(newHeadBlock.NumberU64())) + bc.currentBlock.Store(newHeadBlock) + headBlockGauge.Update(int64(newHeadBlock.Number.Uint64())) // The head state is missing, which is only possible in the path-based // scheme. This situation occurs when the chain head is rewound below // the pivot point. In this scenario, there is no possible recovery // approach except for rerunning a snap sync. Do nothing here until the // state syncer picks it up. - if !bc.HasState(newHeadBlock.Root()) { - log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number(), "hash", newHeadBlock.Hash()) + if !bc.HasState(newHeadBlock.Root) { + if newHeadBlock.Number.Uint64() != 0 { + log.Crit("Chain is stateless at a non-genesis block") + } + log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number, "hash", newHeadBlock.Hash()) } } // Rewind the snap block in a simpleton way to the target head @@ -783,6 +881,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha // intent afterwards is full block importing, delete the chain segment // between the stateful-block and the sethead target. var wipe bool + frozen, _ := bc.db.Ancients() if headNumber+1 < frozen { wipe = pivot == nil || headNumber >= *pivot } @@ -832,7 +931,6 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha bc.receiptsCache.Purge() bc.blockCache.Purge() bc.txLookupCache.Purge() - bc.futureBlocks.Purge() // Clear safe block, finalized block if needed if safe := bc.CurrentSafeBlock(); safe != nil && head < safe.Number.Uint64() { @@ -1086,24 +1184,6 @@ func (bc *BlockChain) insertStopped() bool { return bc.procInterrupt.Load() } -func (bc *BlockChain) procFutureBlocks() { - blocks := make([]*types.Block, 0, bc.futureBlocks.Len()) - for _, hash := range bc.futureBlocks.Keys() { - if block, exist := bc.futureBlocks.Peek(hash); exist { - blocks = append(blocks, block) - } - } - if len(blocks) > 0 { - slices.SortFunc(blocks, func(a, b *types.Block) int { - return a.Number().Cmp(b.Number()) - }) - // Insert one by one as chain insertion needs contiguous ancestry between blocks - for i := range blocks { - bc.InsertChain(blocks[i : i+1]) - } - } -} - // WriteStatus status of write type WriteStatus byte @@ -1504,8 +1584,6 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types if status == CanonStatTy { bc.writeHeadBlock(block) } - bc.futureBlocks.Remove(block.Hash()) - if status == CanonStatTy { bc.chainFeed.Send(ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) if len(logs) > 0 { @@ -1525,25 +1603,6 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types return status, nil } -// addFutureBlock checks if the block is within the max allowed window to get -// accepted for future processing, and returns an error if the block is too far -// ahead and was not added. -// -// TODO after the transition, the future block shouldn't be kept. Because -// it's not checked in the Geth side anymore. -func (bc *BlockChain) addFutureBlock(block *types.Block) error { - max := uint64(time.Now().Unix() + maxTimeFutureBlocks) - if block.Time() > max { - return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max) - } - if block.Difficulty().Cmp(common.Big0) == 0 { - // Never add PoS blocks into the future queue - return nil - } - bc.futureBlocks.Add(block.Hash(), block) - return nil -} - // InsertChain attempts to insert the given batch of blocks in to the canonical // chain or, otherwise, create a fork. If an error is returned it will return // the index number of the failing block as well an error describing what went @@ -1681,26 +1740,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) _, err := bc.recoverAncestors(block) return it.index, err } - // First block is future, shove it (and all children) to the future queue (unknown ancestor) - case errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash())): - for block != nil && (it.index == 0 || errors.Is(err, consensus.ErrUnknownAncestor)) { - log.Debug("Future block, postponing import", "number", block.Number(), "hash", block.Hash()) - if err := bc.addFutureBlock(block); err != nil { - return it.index, err - } - block, err = it.next() - } - stats.queued += it.processed() - stats.ignored += it.remaining() - - // If there are any still remaining, mark as ignored - return it.index, err - // Some other error(except ErrKnownBlock) occurred, abort. // ErrKnownBlock is allowed here since some known blocks // still need re-execution to generate snapshots that are missing case err != nil && !errors.Is(err, ErrKnownBlock): - bc.futureBlocks.Remove(block.Hash()) stats.ignored += len(it.chain) bc.reportBlock(block, nil, err) return it.index, err @@ -1711,7 +1754,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) // The chain importer is starting and stopping trie prefetchers. If a bad // block or other error is hit however, an early return may not properly // terminate the background threads. This defer ensures that we clean up - // and dangling prefetcher, without defering each and holding on live refs. + // and dangling prefetcher, without deferring each and holding on live refs. if activeState != nil { activeState.StopPrefetcher() } @@ -1761,9 +1804,13 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) return it.index, err } stats.processed++ - if bc.logger != nil { - bc.logger.OnBlockStart(block, bc.GetTd(block.ParentHash(), block.NumberU64()-1), bc.CurrentFinalBlock(), bc.CurrentSafeBlock(), bc.chainConfig) - bc.logger.OnBlockEnd(nil) + if bc.logger != nil && bc.logger.OnSkippedBlock != nil { + bc.logger.OnSkippedBlock(tracing.BlockEvent{ + Block: block, + TD: bc.GetTd(block.ParentHash(), block.NumberU64()-1), + Finalized: bc.CurrentFinalBlock(), + Safe: bc.CurrentSafeBlock(), + }) } // We can assume that logs are empty here, since the only way for consecutive @@ -1858,23 +1905,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) "root", block.Root()) } } - - // Any blocks remaining here? The only ones we care about are the future ones - if block != nil && errors.Is(err, consensus.ErrFutureBlock) { - if err := bc.addFutureBlock(block); err != nil { - return it.index, err - } - block, err = it.next() - - for ; block != nil && errors.Is(err, consensus.ErrUnknownAncestor); block, err = it.next() { - if err := bc.addFutureBlock(block); err != nil { - return it.index, err - } - stats.queued++ - } - } stats.ignored += it.remaining() - return it.index, err } @@ -1889,9 +1920,16 @@ type blockProcessingResult struct { // processBlock executes and validates the given block. If there was no error // it writes the block and associated state to database. func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, start time.Time, setHead bool) (_ *blockProcessingResult, blockEndErr error) { - if bc.logger != nil { + if bc.logger != nil && bc.logger.OnBlockStart != nil { td := bc.GetTd(block.ParentHash(), block.NumberU64()-1) - bc.logger.OnBlockStart(block, td, bc.CurrentFinalBlock(), bc.CurrentSafeBlock(), bc.chainConfig) + bc.logger.OnBlockStart(tracing.BlockEvent{ + Block: block, + TD: td, + Finalized: bc.CurrentFinalBlock(), + Safe: bc.CurrentSafeBlock(), + }) + } + if bc.logger != nil && bc.logger.OnBlockEnd != nil { defer func() { bc.logger.OnBlockEnd(blockEndErr) }() @@ -2257,6 +2295,12 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Block) error { // rewind the canonical chain to a lower point. log.Error("Impossible reorg, please file an issue", "oldnum", oldBlock.Number(), "oldhash", oldBlock.Hash(), "oldblocks", len(oldChain), "newnum", newBlock.Number(), "newhash", newBlock.Hash(), "newblocks", len(newChain)) } + // Reset the tx lookup cache in case to clear stale txlookups. + // This is done before writing any new chain data to avoid the + // weird scenario that canonical chain is changed while the + // stale lookups are still cached. + bc.txLookupCache.Purge() + // Insert the new chain(except the head block(reverse order)), // taking care of the proper incremental order. for i := len(newChain) - 1; i >= 1; i-- { @@ -2271,11 +2315,13 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Block) error { // Delete useless indexes right now which includes the non-canonical // transaction indexes, canonical chain indexes which above the head. - indexesBatch := bc.db.NewBatch() - for _, tx := range types.HashDifference(deletedTxs, addedTxs) { + var ( + indexesBatch = bc.db.NewBatch() + diffs = types.HashDifference(deletedTxs, addedTxs) + ) + for _, tx := range diffs { rawdb.DeleteTxLookupEntry(indexesBatch, tx) } - // Delete all hash markers that are not part of the new canonical chain. // Because the reorg function does not handle new chain head, all hash // markers greater than or equal to new chain head should be deleted. @@ -2395,20 +2441,6 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) { return head.Hash(), nil } -func (bc *BlockChain) updateFutureBlocks() { - futureTimer := time.NewTicker(5 * time.Second) - defer futureTimer.Stop() - defer bc.wg.Done() - for { - select { - case <-futureTimer.C: - bc.procFutureBlocks() - case <-bc.quit: - return - } - } -} - // skipBlock returns 'true', if the block being imported can be skipped over, meaning // that the block does not need to be processed but can be considered already fully 'done'. func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool { @@ -2516,7 +2548,7 @@ func (bc *BlockChain) SetTrieFlushInterval(interval time.Duration) { bc.flushInterval.Store(int64(interval)) } -// GetTrieFlushInterval gets the in-memory tries flush interval +// GetTrieFlushInterval gets the in-memory tries flushAlloc interval func (bc *BlockChain) GetTrieFlushInterval() time.Duration { return time.Duration(bc.flushInterval.Load()) } diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index 9bf662b6b7..c7c4c4bfea 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -179,8 +179,3 @@ func (it *insertIterator) first() *types.Block { func (it *insertIterator) remaining() int { return len(it.chain) - it.index } - -// processed returns the number of processed blocks. -func (it *insertIterator) processed() int { - return it.index + 1 -} diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 6fb09abacc..9e8e3bd419 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) // CurrentHeader retrieves the current head header of the canonical chain. The @@ -406,10 +406,15 @@ func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) { } // TrieDB retrieves the low level trie database used for data storage. -func (bc *BlockChain) TrieDB() *trie.Database { +func (bc *BlockChain) TrieDB() *triedb.Database { return bc.triedb } +// HeaderChain returns the underlying header chain. +func (bc *BlockChain) HeaderChain() *HeaderChain { + return bc.hc +} + // SubscribeRemovedLogsEvent registers a subscription of RemovedLogsEvent. func (bc *BlockChain) SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription { return bc.scope.Track(bc.rmLogsFeed.Subscribe(ch)) diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index b2df39d17b..a4761f337b 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -22,7 +22,7 @@ package core import ( "math/big" - "path" + "path/filepath" "testing" "time" @@ -1757,12 +1757,12 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) { // It's hard to follow the test case, visualize the input - //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // fmt.Println(tt.dump(true)) // Create a temporary persistent database datadir := t.TempDir() - ancient := path.Join(datadir, "ancient") + ancient := filepath.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, @@ -1830,10 +1830,14 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s } // Force run a freeze cycle type freezer interface { - Freeze(threshold uint64) error + Freeze() error Ancients() (uint64, error) } - db.(freezer).Freeze(tt.freezeThreshold) + if tt.freezeThreshold < uint64(tt.canonicalBlocks) { + final := uint64(tt.canonicalBlocks) - tt.freezeThreshold + chain.SetFinalized(canonblocks[int(final)-1].Header()) + } + db.(freezer).Freeze() // Set the simulated pivot block if tt.pivotBlock != nil { @@ -1908,7 +1912,7 @@ func testIssue23496(t *testing.T, scheme string) { // Create a temporary persistent database datadir := t.TempDir() - ancient := path.Join(datadir, "ancient") + ancient := filepath.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index fa739f924f..b96ee12c99 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -34,9 +34,9 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) // rewindTest is a test case for chain rollback upon user request. @@ -2033,21 +2033,25 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme } // Reopen the trie database without persisting in-memory dirty nodes. chain.triedb.Close() - dbconfig := &trie.Config{} + dbconfig := &triedb.Config{} if scheme == rawdb.PathScheme { dbconfig.PathDB = pathdb.Defaults } else { dbconfig.HashDB = hashdb.Defaults } - chain.triedb = trie.NewDatabase(chain.db, dbconfig) + chain.triedb = triedb.NewDatabase(chain.db, dbconfig) chain.stateCache = state.NewDatabaseWithNodeDB(chain.db, chain.triedb) // Force run a freeze cycle type freezer interface { - Freeze(threshold uint64) error + Freeze() error Ancients() (uint64, error) } - db.(freezer).Freeze(tt.freezeThreshold) + if tt.freezeThreshold < uint64(tt.canonicalBlocks) { + final := uint64(tt.canonicalBlocks) - tt.freezeThreshold + chain.SetFinalized(canonblocks[int(final)-1].Header()) + } + db.(freezer).Freeze() // Set the simulated pivot block if tt.pivotBlock != nil { diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index dd012c430c..80f8035df1 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -24,7 +24,7 @@ import ( "fmt" "math/big" "os" - "path" + "path/filepath" "strings" "testing" "time" @@ -63,7 +63,7 @@ type snapshotTestBasic struct { func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) { // Create a temporary persistent database datadir := t.TempDir() - ancient := path.Join(datadir, "ancient") + ancient := filepath.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, diff --git a/core/blockchain_test.go b/core/blockchain_test.go index d16bb39cc3..b41398f023 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -21,6 +21,7 @@ import ( "fmt" "math/big" "math/rand" + "os" "sync" "testing" "time" @@ -35,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" @@ -837,7 +839,7 @@ func testFastVsFullChains(t *testing.T, scheme string) { funds = big.NewInt(1000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{address: {Balance: funds}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } signer = types.LatestSigner(gspec.Config) @@ -970,7 +972,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { funds = big.NewInt(1000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{address: {Balance: funds}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } ) @@ -1090,7 +1092,7 @@ func testChainTxReorgs(t *testing.T, scheme string) { gspec = &Genesis{ Config: params.TestChainConfig, GasLimit: 3141592, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr1: {Balance: big.NewInt(1000000000000000)}, addr2: {Balance: big.NewInt(1000000000000000)}, addr3: {Balance: big.NewInt(1000000000000000)}, @@ -1205,7 +1207,7 @@ func testLogReorgs(t *testing.T, scheme string) { // this code generates a log code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} + gspec = &Genesis{Config: params.TestChainConfig, Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} signer = types.LatestSigner(gspec.Config) ) @@ -1262,7 +1264,7 @@ func testLogRebirth(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} + gspec = &Genesis{Config: params.TestChainConfig, Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} signer = types.LatestSigner(gspec.Config) engine = ethash.NewFaker() blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) @@ -1344,7 +1346,7 @@ func testSideLogRebirth(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} + gspec = &Genesis{Config: params.TestChainConfig, Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} signer = types.LatestSigner(gspec.Config) blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) @@ -1441,7 +1443,7 @@ func testReorgSideEvent(t *testing.T, scheme string) { addr1 = crypto.PubkeyToAddress(key1.PublicKey) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}, + Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}, } signer = types.LatestSigner(gspec.Config) ) @@ -1584,7 +1586,7 @@ func testEIP155Transition(t *testing.T, scheme string) { EIP155Block: big.NewInt(2), HomesteadBlock: new(big.Int), }, - Alloc: GenesisAlloc{address: {Balance: funds}, deleteAddr: {Balance: new(big.Int)}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}, deleteAddr: {Balance: new(big.Int)}}, } ) genDb, blocks, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 4, func(i int, block *BlockGen) { @@ -1699,7 +1701,7 @@ func testEIP161AccountRemoval(t *testing.T, scheme string) { EIP150Block: new(big.Int), EIP158Block: big.NewInt(2), }, - Alloc: GenesisAlloc{address: {Balance: funds}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, } ) _, blocks, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 3, func(i int, block *BlockGen) { @@ -1930,7 +1932,7 @@ func testBlockchainRecovery(t *testing.T, scheme string) { key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} + gspec = &Genesis{Config: params.TestChainConfig, Alloc: types.GenesisAlloc{address: {Balance: funds}}} ) height := uint64(1024) _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(height), nil) @@ -2127,7 +2129,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Generate a canonical chain to act as the main dataset chainConfig := *params.TestChainConfig var ( - merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) engine = beacon.New(ethash.NewFaker()) key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key.PublicKey) @@ -2135,7 +2136,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon gspec = &Genesis{ Config: &chainConfig, - Alloc: GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, + Alloc: types.GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, BaseFee: big.NewInt(params.InitialBaseFee), } signer = types.LatestSigner(gspec.Config) @@ -2151,8 +2152,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Activate the transition since genesis if required if mergePoint == 0 { mergeBlock = 0 - merger.ReachTTD() - merger.FinalizePoS() // Set the terminal total difficulty in the config gspec.Config.TerminalTotalDifficulty = big.NewInt(0) @@ -2187,8 +2186,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Activate the transition in the middle of the chain if mergePoint == 1 { - merger.ReachTTD() - merger.FinalizePoS() // Set the terminal total difficulty in the config ttd := big.NewInt(int64(len(blocks))) ttd.Mul(ttd, params.GenesisDifficulty) @@ -2730,7 +2727,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in bankFunds = big.NewInt(100000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ testBankAddress: {Balance: bankFunds}, common.HexToAddress("0xc0de"): { Code: []byte{0x60, 0x01, 0x50}, @@ -2908,7 +2905,7 @@ func testDeleteCreateRevert(t *testing.T, scheme string) { funds = big.NewInt(100000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xAAAAA selfdestructs if called aa: { @@ -3032,7 +3029,7 @@ func testDeleteRecreateSlots(t *testing.T, scheme string) { gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xAAAAA selfdestructs if called aa: { @@ -3062,7 +3059,7 @@ func testDeleteRecreateSlots(t *testing.T, scheme string) { }) // Import the canonical chain chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ - //Tracer: logger.NewJSONLogger(nil, os.Stdout), + Tracer: logger.NewJSONLogger(nil, os.Stdout).Hooks(), }, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) @@ -3118,7 +3115,7 @@ func testDeleteRecreateAccount(t *testing.T, scheme string) { gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xAAAAA selfdestructs if called aa: { @@ -3144,7 +3141,7 @@ func testDeleteRecreateAccount(t *testing.T, scheme string) { }) // Import the canonical chain chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ - //Tracer: logger.NewJSONLogger(nil, os.Stdout), + Tracer: logger.NewJSONLogger(nil, os.Stdout).Hooks(), }, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) @@ -3239,7 +3236,7 @@ func testDeleteRecreateSlotsAcrossManyBlocks(t *testing.T, scheme string) { t.Logf("Destination address: %x\n", aa) gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xAAAAA selfdestructs if called aa: { @@ -3434,7 +3431,7 @@ func testInitThenFailCreateContract(t *testing.T, scheme string) { gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address aa has some funds aa: {Balance: big.NewInt(100000)}, @@ -3509,7 +3506,7 @@ func testEIP2718Transition(t *testing.T, scheme string) { funds = big.NewInt(1000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xAAAA sloads 0x00 and 0x01 aa: { @@ -3594,7 +3591,7 @@ func testEIP1559Transition(t *testing.T, scheme string) { config = *params.AllEthashProtocolChanges gspec = &Genesis{ Config: &config, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr1: {Balance: funds}, addr2: {Balance: funds}, // The address 0xAAAA sloads 0x00 and 0x01 @@ -3735,7 +3732,7 @@ func testSetCanonical(t *testing.T, scheme string) { funds = big.NewInt(100000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{address: {Balance: funds}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } signer = types.LatestSigner(gspec.Config) @@ -3852,7 +3849,7 @@ func testCanonicalHashMarker(t *testing.T, scheme string) { var ( gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{}, + Alloc: types.GenesisAlloc{}, BaseFee: big.NewInt(params.InitialBaseFee), } engine = ethash.NewFaker() @@ -3965,7 +3962,7 @@ func testCreateThenDelete(t *testing.T, config *params.ChainConfig) { }...) gspec := &Genesis{ Config: config, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, }, } @@ -4051,7 +4048,7 @@ func TestDeleteThenCreate(t *testing.T) { gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, }, } @@ -4163,7 +4160,7 @@ func TestTransientStorageReset(t *testing.T) { }...) gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, }, } @@ -4231,7 +4228,7 @@ func TestEIP3651(t *testing.T) { config = *params.AllEthashProtocolChanges gspec = &Genesis{ Config: &config, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr1: {Balance: funds}, addr2: {Balance: funds}, // The address 0xAAAA sloads 0x00 and 0x01 @@ -4290,7 +4287,7 @@ func TestEIP3651(t *testing.T) { b.AddTx(tx) }) - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ /*Tracer: logger.NewMarkdownLogger(&logger.Config{}, os.Stderr)*/ }, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{Tracer: logger.NewMarkdownLogger(&logger.Config{}, os.Stderr).Hooks()}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } diff --git a/core/chain_makers.go b/core/chain_makers.go index 6814b24698..733030fd1c 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" "github.com/holiman/uint256" ) @@ -83,7 +83,7 @@ func (b *BlockGen) SetDifficulty(diff *big.Int) { b.header.Difficulty = diff } -// SetPos makes the header a PoS-header (0 difficulty) +// SetPoS makes the header a PoS-header (0 difficulty) func (b *BlockGen) SetPoS() { b.header.Difficulty = new(big.Int) } @@ -101,7 +101,7 @@ func (b *BlockGen) SetParentBeaconRoot(root common.Hash) { blockContext = NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase) vmenv = vm.NewEVM(blockContext, vm.TxContext{}, b.statedb, b.cm.config, vm.Config{}) ) - ProcessBeaconBlockRoot(root, vmenv, b.statedb, nil) + ProcessBeaconBlockRoot(root, vmenv, b.statedb) } // addTx adds a transaction to the generated block. If no coinbase has @@ -312,7 +312,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } cm := newChainMaker(parent, config, engine) - genblock := func(i int, parent *types.Block, triedb *trie.Database, statedb *state.StateDB) (*types.Block, types.Receipts) { + genblock := func(i int, parent *types.Block, triedb *triedb.Database, statedb *state.StateDB) (*types.Block, types.Receipts) { b := &BlockGen{i: i, cm: cm, parent: parent, statedb: statedb, engine: engine} b.header = cm.makeHeader(parent, statedb, b.engine) @@ -362,7 +362,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } // Forcibly use hash-based state scheme for retaining all nodes in disk. - triedb := trie.NewDatabase(db, trie.HashDefaults) + triedb := triedb.NewDatabase(db, triedb.HashDefaults) defer triedb.Close() for i := 0; i < n; i++ { @@ -407,8 +407,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse // then generate chain on top. func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) { db := rawdb.NewMemoryDatabase() - - triedb := trie.NewDatabase(db, trie.HashDefaults) + triedb := triedb.NewDatabase(db, triedb.HashDefaults) defer triedb.Close() _, err := genesis.Commit(db, triedb) if err != nil { diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 84148841f5..b46b898afb 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) func TestGeneratePOSChain(t *testing.T) { @@ -46,7 +46,7 @@ func TestGeneratePOSChain(t *testing.T) { asm4788 = common.Hex2Bytes("3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500") gspec = &Genesis{ Config: &config, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, params.BeaconRootsStorageAddress: {Balance: common.Big0, Code: asm4788}, }, @@ -69,19 +69,19 @@ func TestGeneratePOSChain(t *testing.T) { storage[common.Hash{0x01}] = common.Hash{0x01} storage[common.Hash{0x02}] = common.Hash{0x02} storage[common.Hash{0x03}] = common.HexToHash("0303") - gspec.Alloc[aa] = GenesisAccount{ + gspec.Alloc[aa] = types.Account{ Balance: common.Big1, Nonce: 1, Storage: storage, Code: common.Hex2Bytes("6042"), } - gspec.Alloc[bb] = GenesisAccount{ + gspec.Alloc[bb] = types.Account{ Balance: common.Big2, Nonce: 1, Storage: storage, Code: common.Hex2Bytes("600154600354"), } - genesis := gspec.MustCommit(gendb, trie.NewDatabase(gendb, trie.HashDefaults)) + genesis := gspec.MustCommit(gendb, triedb.NewDatabase(gendb, triedb.HashDefaults)) genchain, genreceipts := GenerateChain(gspec.Config, genesis, beacon.NewFaker(), gendb, 4, func(i int, gen *BlockGen) { gen.SetParentBeaconRoot(common.Hash{byte(i + 1)}) @@ -202,9 +202,9 @@ func ExampleGenerateChain() { // Ensure that key1 has some funds in the genesis block. gspec := &Genesis{ Config: ¶ms.ChainConfig{HomesteadBlock: new(big.Int)}, - Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, + Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, } - genesis := gspec.MustCommit(genDb, trie.NewDatabase(genDb, trie.HashDefaults)) + genesis := gspec.MustCommit(genDb, triedb.NewDatabase(genDb, triedb.HashDefaults)) // This call generates a chain of 5 blocks. The function runs for // each block and adds different features to gen based on the diff --git a/core/evm.go b/core/evm.go index 1479a55780..4c12e2aa02 100644 --- a/core/evm.go +++ b/core/evm.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" - "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/holiman/uint256" @@ -137,6 +137,6 @@ func CanTransfer(db vm.StateDB, addr common.Address, amount *uint256.Int) bool { // Transfer subtracts amount from sender and adds amount to recipient using the given Db func Transfer(db vm.StateDB, sender, recipient common.Address, amount *uint256.Int) { - db.SubBalance(sender, amount, state.BalanceChangeTransfer) - db.AddBalance(recipient, amount, state.BalanceChangeTransfer) + db.SubBalance(sender, amount, tracing.BalanceChangeTransfer) + db.AddBalance(recipient, amount, tracing.BalanceChangeTransfer) } diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index 776c428f75..b9d346bd90 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -74,8 +74,10 @@ func TestCreation(t *testing.T) { {15049999, 0, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // Last Arrow Glacier block {15050000, 0, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 1681338455}}, // First Gray Glacier block {20000000, 1681338454, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 1681338455}}, // Last Gray Glacier block - {20000000, 1681338455, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}}, // First Shanghai block - {30000000, 2000000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}}, // Future Shanghai block + {20000000, 1681338455, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // First Shanghai block + {30000000, 1710338134, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // Last Shanghai block + {40000000, 1710338135, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // First Cancun block + {50000000, 2000000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // Future Cancun block }, }, // Goerli test cases @@ -141,6 +143,7 @@ func TestValidation(t *testing.T) { // Config that has not timestamp enabled legacyConfig := *params.MainnetChainConfig legacyConfig.ShanghaiTime = nil + legacyConfig.CancunTime = nil tests := []struct { config *params.ChainConfig @@ -213,14 +216,10 @@ func TestValidation(t *testing.T) { // at some future block 88888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - // - // TODO(karalabe): This testcase will fail once mainnet gets timestamped forks, make legacy chain config {&legacyConfig, 88888888, 0, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 88888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Byzantium. Remote is also in Byzantium, but announces Gopherium (non existing // fork) at block 7279999, before Petersburg. Local is incompatible. - // - // TODO(karalabe): This testcase will fail once mainnet gets timestamped forks, make legacy chain config {&legacyConfig, 7279999, 0, ID{Hash: checksumToBytes(0xa00bc324), Next: 7279999}, ErrLocalIncompatibleOrStale}, //------------------------------------ @@ -297,34 +296,25 @@ func TestValidation(t *testing.T) { // Local is mainnet currently in Shanghai only (so it's aware of Cancun), remote announces // also Shanghai, but it's not yet aware of Cancun (e.g. non updated node before the fork). // In this case we don't know if Cancun passed yet or not. - // - // TODO(karalabe): Enable this when Cancun is specced - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0x71147644), Next: 0}, nil}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}, nil}, // Local is mainnet currently in Shanghai only (so it's aware of Cancun), remote announces // also Shanghai, and it's also aware of Cancun (e.g. updated node before the fork). We // don't know if Cancun passed yet (will pass) or not. - // - // TODO(karalabe): Enable this when Cancun is specced and update next timestamp - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, nil}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}, nil}, // Local is mainnet currently in Shanghai only (so it's aware of Cancun), remote announces // also Shanghai, and it's also aware of some random fork (e.g. misconfigured Cancun). As // neither forks passed at neither nodes, they may mismatch, but we still connect for now. - // - // TODO(karalabe): Enable this when Cancun is specced - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0x71147644), Next: math.MaxUint64}, nil}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: math.MaxUint64}, nil}, // Local is mainnet exactly on Cancun, remote announces Shanghai + knowledge about Cancun. Remote // is simply out of sync, accept. - // - // TODO(karalabe): Enable this when Cancun is specced, update local head and time, next timestamp - // {params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, nil}, + {params.MainnetChainConfig, 21000000, 1710338135, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}, nil}, // Local is mainnet Cancun, remote announces Shanghai + knowledge about Cancun. Remote // is simply out of sync, accept. - // TODO(karalabe): Enable this when Cancun is specced, update local head and time, next timestamp - //{params.MainnetChainConfig, 21123456, 1678123456, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, nil}, + {params.MainnetChainConfig, 21123456, 1710338136, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}, nil}, // Local is mainnet Prague, remote announces Shanghai + knowledge about Cancun. Remote // is definitely out of sync. It may or may not need the Prague update, we don't know yet. @@ -333,9 +323,7 @@ func TestValidation(t *testing.T) { //{params.MainnetChainConfig, 0, 0, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}, nil}, // Local is mainnet Shanghai, remote announces Cancun. Local is out of sync, accept. - // - // TODO(karalabe): Enable this when Cancun is specced, update remote checksum - //{params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x00000000), Next: 0}, nil}, + {params.MainnetChainConfig, 21000000, 1700000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}, nil}, // Local is mainnet Shanghai, remote announces Cancun, but is not aware of Prague. Local // out of sync. Local also knows about a future fork, but that is uncertain yet. @@ -345,9 +333,7 @@ func TestValidation(t *testing.T) { // Local is mainnet Cancun. remote announces Shanghai but is not aware of further forks. // Remote needs software update. - // - // TODO(karalabe): Enable this when Cancun is specced, update local head and time - //{params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x71147644), Next: 0}, ErrRemoteStale}, + {params.MainnetChainConfig, 21000000, 1710338135, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}, ErrRemoteStale}, // Local is mainnet Shanghai, and isn't aware of more forks. Remote announces Shanghai + // 0xffffffff. Local needs software update, reject. @@ -355,24 +341,20 @@ func TestValidation(t *testing.T) { // Local is mainnet Shanghai, and is aware of Cancun. Remote announces Cancun + // 0xffffffff. Local needs software update, reject. - // - // TODO(karalabe): Enable this when Cancun is specced, update remote checksum - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(checksumUpdate(0x00000000, math.MaxUint64)), Next: 0}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(checksumUpdate(0x9f3d2254, math.MaxUint64)), Next: 0}, ErrLocalIncompatibleOrStale}, // Local is mainnet Shanghai, remote is random Shanghai. {params.MainnetChainConfig, 20000000, 1681338455, ID{Hash: checksumToBytes(0x12345678), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Shanghai, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Cancun, far in the future. Remote announces Gopherium (non existing fork) // at some future timestamp 8888888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0xdce96c2d), Next: 8888888888}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0x9f3d2254), Next: 8888888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Shanghai. Remote is also in Shanghai, but announces Gopherium (non existing // fork) at timestamp 1668000000, before Cancun. Local is incompatible. - // - // TODO(karalabe): Enable this when Cancun is specced - //{params.MainnetChainConfig, 20999999, 1677999999, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 20999999, 1699999999, ID{Hash: checksumToBytes(0x71147644), Next: 1700000000}, ErrLocalIncompatibleOrStale}, } genesis := core.DefaultGenesisBlock().ToBlock() for i, tt := range tests { diff --git a/core/gen_genesis.go b/core/gen_genesis.go index 38614252a3..b8acf9df7c 100644 --- a/core/gen_genesis.go +++ b/core/gen_genesis.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" ) @@ -18,21 +19,21 @@ var _ = (*genesisSpecMarshaling)(nil) // MarshalJSON marshals as JSON. func (g Genesis) MarshalJSON() ([]byte, error) { type Genesis struct { - Config *params.ChainConfig `json:"config"` - Nonce math.HexOrDecimal64 `json:"nonce"` - Timestamp math.HexOrDecimal64 `json:"timestamp"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` - Mixhash common.Hash `json:"mixHash"` - Coinbase common.Address `json:"coinbase"` - Alloc map[common.UnprefixedAddress]GenesisAccount `json:"alloc" gencodec:"required"` - Number math.HexOrDecimal64 `json:"number"` - GasUsed math.HexOrDecimal64 `json:"gasUsed"` - ParentHash common.Hash `json:"parentHash"` - BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` - ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas"` - BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed"` + Config *params.ChainConfig `json:"config"` + Nonce math.HexOrDecimal64 `json:"nonce"` + Timestamp math.HexOrDecimal64 `json:"timestamp"` + ExtraData hexutil.Bytes `json:"extraData"` + GasLimit math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` + Mixhash common.Hash `json:"mixHash"` + Coinbase common.Address `json:"coinbase"` + Alloc map[common.UnprefixedAddress]types.Account `json:"alloc" gencodec:"required"` + Number math.HexOrDecimal64 `json:"number"` + GasUsed math.HexOrDecimal64 `json:"gasUsed"` + ParentHash common.Hash `json:"parentHash"` + BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` + ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas"` + BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed"` } var enc Genesis enc.Config = g.Config @@ -44,7 +45,7 @@ func (g Genesis) MarshalJSON() ([]byte, error) { enc.Mixhash = g.Mixhash enc.Coinbase = g.Coinbase if g.Alloc != nil { - enc.Alloc = make(map[common.UnprefixedAddress]GenesisAccount, len(g.Alloc)) + enc.Alloc = make(map[common.UnprefixedAddress]types.Account, len(g.Alloc)) for k, v := range g.Alloc { enc.Alloc[common.UnprefixedAddress(k)] = v } @@ -61,21 +62,21 @@ func (g Genesis) MarshalJSON() ([]byte, error) { // UnmarshalJSON unmarshals from JSON. func (g *Genesis) UnmarshalJSON(input []byte) error { type Genesis struct { - Config *params.ChainConfig `json:"config"` - Nonce *math.HexOrDecimal64 `json:"nonce"` - Timestamp *math.HexOrDecimal64 `json:"timestamp"` - ExtraData *hexutil.Bytes `json:"extraData"` - GasLimit *math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` - Mixhash *common.Hash `json:"mixHash"` - Coinbase *common.Address `json:"coinbase"` - Alloc map[common.UnprefixedAddress]GenesisAccount `json:"alloc" gencodec:"required"` - Number *math.HexOrDecimal64 `json:"number"` - GasUsed *math.HexOrDecimal64 `json:"gasUsed"` - ParentHash *common.Hash `json:"parentHash"` - BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` - ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas"` - BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed"` + Config *params.ChainConfig `json:"config"` + Nonce *math.HexOrDecimal64 `json:"nonce"` + Timestamp *math.HexOrDecimal64 `json:"timestamp"` + ExtraData *hexutil.Bytes `json:"extraData"` + GasLimit *math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` + Mixhash *common.Hash `json:"mixHash"` + Coinbase *common.Address `json:"coinbase"` + Alloc map[common.UnprefixedAddress]types.Account `json:"alloc" gencodec:"required"` + Number *math.HexOrDecimal64 `json:"number"` + GasUsed *math.HexOrDecimal64 `json:"gasUsed"` + ParentHash *common.Hash `json:"parentHash"` + BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` + ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas"` + BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed"` } var dec Genesis if err := json.Unmarshal(input, &dec); err != nil { @@ -110,7 +111,7 @@ func (g *Genesis) UnmarshalJSON(input []byte) error { if dec.Alloc == nil { return errors.New("missing required field 'alloc' for Genesis") } - g.Alloc = make(GenesisAlloc, len(dec.Alloc)) + g.Alloc = make(types.GenesisAlloc, len(dec.Alloc)) for k, v := range dec.Alloc { g.Alloc[common.Address(k)] = v } diff --git a/core/genesis.go b/core/genesis.go index 504251b445..ee0e322f80 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -18,7 +18,6 @@ package core import ( "bytes" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -30,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -37,15 +37,21 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" "github.com/holiman/uint256" ) //go:generate go run github.com/fjl/gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go -//go:generate go run github.com/fjl/gencodec -type GenesisAccount -field-override genesisAccountMarshaling -out gen_genesis_account.go var errGenesisNoConfig = errors.New("genesis has no chain configuration") +// Deprecated: use types.Account instead. +type GenesisAccount = types.Account + +// Deprecated: use types.GenesisAlloc instead. +type GenesisAlloc = types.GenesisAlloc + // Genesis specifies the header fields, state of a genesis block. It also defines hard // fork switch-over blocks through the chain configuration. type Genesis struct { @@ -57,7 +63,7 @@ type Genesis struct { Difficulty *big.Int `json:"difficulty" gencodec:"required"` Mixhash common.Hash `json:"mixHash"` Coinbase common.Address `json:"coinbase"` - Alloc GenesisAlloc `json:"alloc" gencodec:"required"` + Alloc types.GenesisAlloc `json:"alloc" gencodec:"required"` // These fields are used for consensus tests. Please don't use them // in actual genesis blocks. @@ -107,29 +113,14 @@ func ReadGenesis(db ethdb.Database) (*Genesis, error) { return &genesis, nil } -// GenesisAlloc specifies the initial state that is part of the genesis block. -type GenesisAlloc map[common.Address]GenesisAccount - -func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { - m := make(map[common.UnprefixedAddress]GenesisAccount) - if err := json.Unmarshal(data, &m); err != nil { - return err - } - *ga = make(GenesisAlloc) - for addr, a := range m { - (*ga)[common.Address(addr)] = a - } - return nil -} - -// hash computes the state root according to the genesis specification. -func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) { +// hashAlloc computes the state root according to the genesis specification. +func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) { // If a genesis-time verkle trie is requested, create a trie config // with the verkle trie enabled so that the tree can be initialized // as such. - var config *trie.Config + var config *triedb.Config if isVerkle { - config = &trie.Config{ + config = &triedb.Config{ PathDB: pathdb.Defaults, IsVerkle: true, } @@ -143,7 +134,7 @@ func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) { } for addr, account := range *ga { if account.Balance != nil { - statedb.AddBalance(addr, uint256.MustFromBig(account.Balance), state.BalanceIncreaseGenesisBalance) + statedb.AddBalance(addr, uint256.MustFromBig(account.Balance), tracing.BalanceIncreaseGenesisBalance) } statedb.SetCode(addr, account.Code) statedb.SetNonce(addr, account.Nonce) @@ -154,10 +145,10 @@ func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) { return statedb.Commit(0, false) } -// flush is very similar with hash, but the main difference is all the generated +// flushAlloc is very similar with hash, but the main difference is all the generated // states will be persisted into the given database. Also, the genesis state // specification will be flushed as well. -func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash) error { +func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Database, blockhash common.Hash) error { statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil) if err != nil { return err @@ -166,7 +157,7 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhas if account.Balance != nil { // This is not actually logged via tracer because OnGenesisBlock // already captures the allocations. - statedb.AddBalance(addr, uint256.MustFromBig(account.Balance), state.BalanceIncreaseGenesisBalance) + statedb.AddBalance(addr, uint256.MustFromBig(account.Balance), tracing.BalanceIncreaseGenesisBalance) } statedb.SetCode(addr, account.Code) statedb.SetNonce(addr, account.Nonce) @@ -193,7 +184,7 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhas return nil } -func getGenesisState(db ethdb.Database, blockhash common.Hash) (alloc GenesisAlloc, err error) { +func getGenesisState(db ethdb.Database, blockhash common.Hash) (alloc types.GenesisAlloc, err error) { blob := rawdb.ReadGenesisStateSpec(db, blockhash) if len(blob) != 0 { if err := alloc.UnmarshalJSON(blob); err != nil { @@ -226,15 +217,6 @@ func getGenesisState(db ethdb.Database, blockhash common.Hash) (alloc GenesisAll return nil, nil } -// GenesisAccount is an account in the state of the genesis block. -type GenesisAccount struct { - Code []byte `json:"code,omitempty"` - Storage map[common.Hash]common.Hash `json:"storage,omitempty"` - Balance *big.Int `json:"balance" gencodec:"required"` - Nonce uint64 `json:"nonce,omitempty"` - PrivateKey []byte `json:"secretKey,omitempty"` // for tests -} - // field type overrides for gencodec type genesisSpecMarshaling struct { Nonce math.HexOrDecimal64 @@ -244,40 +226,12 @@ type genesisSpecMarshaling struct { GasUsed math.HexOrDecimal64 Number math.HexOrDecimal64 Difficulty *math.HexOrDecimal256 - Alloc map[common.UnprefixedAddress]GenesisAccount + Alloc map[common.UnprefixedAddress]types.Account BaseFee *math.HexOrDecimal256 ExcessBlobGas *math.HexOrDecimal64 BlobGasUsed *math.HexOrDecimal64 } -type genesisAccountMarshaling struct { - Code hexutil.Bytes - Balance *math.HexOrDecimal256 - Nonce math.HexOrDecimal64 - Storage map[storageJSON]storageJSON - PrivateKey hexutil.Bytes -} - -// storageJSON represents a 256 bit byte array, but allows less than 256 bits when -// unmarshaling from hex. -type storageJSON common.Hash - -func (h *storageJSON) UnmarshalText(text []byte) error { - text = bytes.TrimPrefix(text, []byte("0x")) - if len(text) > 64 { - return fmt.Errorf("too many hex characters in storage key/value %q", text) - } - offset := len(h) - len(text)/2 // pad on the left - if _, err := hex.Decode(h[offset:], text); err != nil { - return fmt.Errorf("invalid hex storage key/value %q", text) - } - return nil -} - -func (h storageJSON) MarshalText() ([]byte, error) { - return hexutil.Bytes(h[:]).MarshalText() -} - // GenesisMismatchError is raised when trying to overwrite an existing // genesis block with an incompatible one. type GenesisMismatchError struct { @@ -307,11 +261,11 @@ type ChainOverrides struct { // error is a *params.ConfigCompatError and the new, unwritten config is returned. // // The returned chain configuration is never nil. -func SetupGenesisBlock(db ethdb.Database, triedb *trie.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlock(db ethdb.Database, triedb *triedb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { return SetupGenesisBlockWithOverride(db, triedb, genesis, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -449,6 +403,8 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { return g.Config case ghash == params.MainnetGenesisHash: return params.MainnetChainConfig + case ghash == params.HoleskyGenesisHash: + return params.HoleskyChainConfig case ghash == params.SepoliaGenesisHash: return params.SepoliaChainConfig case ghash == params.GoerliGenesisHash: @@ -466,7 +422,7 @@ func (g *Genesis) IsVerkle() bool { // ToBlock returns the genesis block according to genesis specification. func (g *Genesis) ToBlock() *types.Block { - root, err := g.Alloc.hash(g.IsVerkle()) + root, err := hashAlloc(&g.Alloc, g.IsVerkle()) if err != nil { panic(err) } @@ -525,7 +481,7 @@ func (g *Genesis) ToBlock() *types.Block { // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. -func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block, error) { +func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Block, error) { block := g.ToBlock() if block.Number().Sign() != 0 { return nil, errors.New("can't commit genesis block with number > 0") @@ -540,10 +496,10 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block if config.Clique != nil && len(block.Extra()) < 32+crypto.SignatureLength { return nil, errors.New("can't start clique chain without signers") } - // All the checks has passed, flush the states derived from the genesis + // All the checks has passed, flushAlloc the states derived from the genesis // specification as well as the specification itself into the provided // database. - if err := g.Alloc.flush(db, triedb, block.Hash()); err != nil { + if err := flushAlloc(&g.Alloc, db, triedb, block.Hash()); err != nil { return nil, err } rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty()) @@ -559,9 +515,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block // MustCommit writes the genesis block and state to db, panicking on error. // The block is committed as the canonical head block. -// Note the state changes will be committed in hash-based scheme, use Commit -// if path-scheme is preferred. -func (g *Genesis) MustCommit(db ethdb.Database, triedb *trie.Database) *types.Block { +func (g *Genesis) MustCommit(db ethdb.Database, triedb *triedb.Database) *types.Block { block, err := g.Commit(db, triedb) if err != nil { panic(err) @@ -629,7 +583,7 @@ func DeveloperGenesisBlock(gasLimit uint64, faucet *common.Address) *Genesis { GasLimit: gasLimit, BaseFee: big.NewInt(params.InitialBaseFee), Difficulty: big.NewInt(1), - Alloc: map[common.Address]GenesisAccount{ + Alloc: map[common.Address]types.Account{ common.BytesToAddress([]byte{1}): {Balance: big.NewInt(1)}, // ECRecover common.BytesToAddress([]byte{2}): {Balance: big.NewInt(1)}, // SHA256 common.BytesToAddress([]byte{3}): {Balance: big.NewInt(1)}, // RIPEMD @@ -642,12 +596,12 @@ func DeveloperGenesisBlock(gasLimit uint64, faucet *common.Address) *Genesis { }, } if faucet != nil { - genesis.Alloc[*faucet] = GenesisAccount{Balance: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(9))} + genesis.Alloc[*faucet] = types.Account{Balance: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(9))} } return genesis } -func decodePrealloc(data string) GenesisAlloc { +func decodePrealloc(data string) types.GenesisAlloc { var p []struct { Addr *big.Int Balance *big.Int @@ -663,9 +617,9 @@ func decodePrealloc(data string) GenesisAlloc { if err := rlp.NewStream(strings.NewReader(data), 0).Decode(&p); err != nil { panic(err) } - ga := make(GenesisAlloc, len(p)) + ga := make(types.GenesisAlloc, len(p)) for _, account := range p { - acc := GenesisAccount{Balance: account.Balance} + acc := types.Account{Balance: account.Balance} if account.Misc != nil { acc.Nonce = account.Misc.Nonce acc.Code = account.Misc.Code diff --git a/core/genesis_test.go b/core/genesis_test.go index 1d85b510ca..61be0bd252 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -27,18 +27,19 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) func TestInvalidCliqueConfig(t *testing.T) { block := DefaultGoerliGenesisBlock() block.ExtraData = []byte{} db := rawdb.NewMemoryDatabase() - if _, err := block.Commit(db, trie.NewDatabase(db, nil)); err == nil { + if _, err := block.Commit(db, triedb.NewDatabase(db, nil)); err == nil { t.Fatal("Expected error on invalid clique config") } } @@ -53,7 +54,7 @@ func testSetupGenesis(t *testing.T, scheme string) { customghash = common.HexToHash("0x89c99d90b79719238d2645c7642f2c9295246e80775b38cfd162b696817fbd50") customg = Genesis{ Config: ¶ms.ChainConfig{HomesteadBlock: big.NewInt(3)}, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, }, } @@ -71,7 +72,7 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "genesis without ChainConfig", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), new(Genesis)) + return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), new(Genesis)) }, wantErr: errGenesisNoConfig, wantConfig: params.AllEthashProtocolChanges, @@ -79,7 +80,7 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "no block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil) + return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), nil) }, wantHash: params.MainnetGenesisHash, wantConfig: params.MainnetChainConfig, @@ -87,8 +88,8 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "mainnet block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - DefaultGenesisBlock().MustCommit(db, trie.NewDatabase(db, newDbConfig(scheme))) - return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil) + DefaultGenesisBlock().MustCommit(db, triedb.NewDatabase(db, newDbConfig(scheme))) + return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), nil) }, wantHash: params.MainnetGenesisHash, wantConfig: params.MainnetChainConfig, @@ -96,7 +97,7 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "custom block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - tdb := trie.NewDatabase(db, newDbConfig(scheme)) + tdb := triedb.NewDatabase(db, newDbConfig(scheme)) customg.Commit(db, tdb) return SetupGenesisBlock(db, tdb, nil) }, @@ -106,7 +107,7 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "custom block in DB, genesis == goerli", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - tdb := trie.NewDatabase(db, newDbConfig(scheme)) + tdb := triedb.NewDatabase(db, newDbConfig(scheme)) customg.Commit(db, tdb) return SetupGenesisBlock(db, tdb, DefaultGoerliGenesisBlock()) }, @@ -117,7 +118,7 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "compatible config in DB", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - tdb := trie.NewDatabase(db, newDbConfig(scheme)) + tdb := triedb.NewDatabase(db, newDbConfig(scheme)) oldcustomg.Commit(db, tdb) return SetupGenesisBlock(db, tdb, &customg) }, @@ -129,7 +130,7 @@ func testSetupGenesis(t *testing.T, scheme string) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { // Commit the 'old' genesis block with Homestead transition at #2. // Advance to block #4, past the homestead transition block of customg. - tdb := trie.NewDatabase(db, newDbConfig(scheme)) + tdb := triedb.NewDatabase(db, newDbConfig(scheme)) oldcustomg.Commit(db, tdb) bc, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) @@ -188,7 +189,7 @@ func TestGenesisHashes(t *testing.T) { } { // Test via MustCommit db := rawdb.NewMemoryDatabase() - if have := c.genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)).Hash(); have != c.want { + if have := c.genesis.MustCommit(db, triedb.NewDatabase(db, triedb.HashDefaults)).Hash(); have != c.want { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } // Test via ToBlock @@ -206,7 +207,7 @@ func TestGenesis_Commit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - genesisBlock := genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) + genesisBlock := genesis.MustCommit(db, triedb.NewDatabase(db, triedb.HashDefaults)) if genesis.Difficulty != nil { t.Fatalf("assumption wrong") @@ -228,16 +229,16 @@ func TestGenesis_Commit(t *testing.T) { func TestReadWriteGenesisAlloc(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() - alloc = &GenesisAlloc{ + alloc = &types.GenesisAlloc{ {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, {2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}}, } - hash, _ = alloc.hash(false) + hash, _ = hashAlloc(alloc, false) ) blob, _ := json.Marshal(alloc) rawdb.WriteGenesisStateSpec(db, hash, blob) - var reload GenesisAlloc + var reload types.GenesisAlloc err := reload.UnmarshalJSON(rawdb.ReadGenesisStateSpec(db, hash)) if err != nil { t.Fatalf("Failed to load genesis state %v", err) @@ -256,11 +257,11 @@ func TestReadWriteGenesisAlloc(t *testing.T) { } } -func newDbConfig(scheme string) *trie.Config { +func newDbConfig(scheme string) *triedb.Config { if scheme == rawdb.HashScheme { - return trie.HashDefaults + return triedb.HashDefaults } - return &trie.Config{PathDB: pathdb.Defaults} + return &triedb.Config{PathDB: pathdb.Defaults} } func TestVerkleGenesisCommit(t *testing.T) { @@ -298,7 +299,7 @@ func TestVerkleGenesisCommit(t *testing.T) { Config: verkleConfig, Timestamp: verkleTime, Difficulty: big.NewInt(0), - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, }, } @@ -310,7 +311,7 @@ func TestVerkleGenesisCommit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - triedb := trie.NewDatabase(db, &trie.Config{IsVerkle: true, PathDB: pathdb.Defaults}) + triedb := triedb.NewDatabase(db, &triedb.Config{IsVerkle: true, PathDB: pathdb.Defaults}) block := genesis.MustCommit(db, triedb) if !bytes.Equal(block.Root().Bytes(), expected) { t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) diff --git a/core/headerchain_test.go b/core/headerchain_test.go index 2c0323e6f7..25d9bfffcb 100644 --- a/core/headerchain_test.go +++ b/core/headerchain_test.go @@ -28,7 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) func verifyUnbrokenCanonchain(hc *HeaderChain) error { @@ -73,7 +73,7 @@ func TestHeaderInsertion(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges} ) - gspec.Commit(db, trie.NewDatabase(db, nil)) + gspec.Commit(db, triedb.NewDatabase(db, nil)) hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false }) if err != nil { t.Fatal(err) diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index ea3367db36..e34b24fd76 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -315,7 +315,7 @@ func ReadStateScheme(db ethdb.Reader) string { // the stored state. // // - If the provided scheme is none, use the scheme consistent with persistent -// state, or fallback to hash-based scheme if state is empty. +// state, or fallback to path-based scheme if state is empty. // // - If the provided scheme is hash, use hash-based scheme or error out if not // compatible with persistent state scheme. @@ -329,10 +329,8 @@ func ParseStateScheme(provided string, disk ethdb.Database) (string, error) { stored := ReadStateScheme(disk) if provided == "" { if stored == "" { - // use default scheme for empty database, flip it when - // path mode is chosen as default - log.Info("State schema set to default", "scheme", "hash") - return HashScheme, nil + log.Info("State schema set to default", "scheme", "path") + return PathScheme, nil // use default scheme for empty database } log.Info("State scheme set to already existing", "scheme", stored) return stored, nil // reuse scheme of persistent scheme diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index bb2c409dbb..d8214874bd 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -17,9 +17,9 @@ package rawdb import ( + "errors" "fmt" "sync" - "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -43,8 +43,6 @@ const ( // The background thread will keep moving ancient chain segments from key-value // database to flat files for saving space on live database. type chainFreezer struct { - threshold atomic.Uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests) - *Freezer quit chan struct{} wg sync.WaitGroup @@ -57,13 +55,11 @@ func newChainFreezer(datadir string, namespace string, readonly bool) (*chainFre if err != nil { return nil, err } - cf := chainFreezer{ + return &chainFreezer{ Freezer: freezer, quit: make(chan struct{}), trigger: make(chan chan struct{}), - } - cf.threshold.Store(params.FullImmutabilityThreshold) - return &cf, nil + }, nil } // Close closes the chain freezer instance and terminates the background thread. @@ -77,6 +73,57 @@ func (f *chainFreezer) Close() error { return f.Freezer.Close() } +// readHeadNumber returns the number of chain head block. 0 is returned if the +// block is unknown or not available yet. +func (f *chainFreezer) readHeadNumber(db ethdb.KeyValueReader) uint64 { + hash := ReadHeadBlockHash(db) + if hash == (common.Hash{}) { + log.Error("Head block is not reachable") + return 0 + } + number := ReadHeaderNumber(db, hash) + if number == nil { + log.Error("Number of head block is missing") + return 0 + } + return *number +} + +// readFinalizedNumber returns the number of finalized block. 0 is returned +// if the block is unknown or not available yet. +func (f *chainFreezer) readFinalizedNumber(db ethdb.KeyValueReader) uint64 { + hash := ReadFinalizedBlockHash(db) + if hash == (common.Hash{}) { + return 0 + } + number := ReadHeaderNumber(db, hash) + if number == nil { + log.Error("Number of finalized block is missing") + return 0 + } + return *number +} + +// freezeThreshold returns the threshold for chain freezing. It's determined +// by formula: max(finality, HEAD-params.FullImmutabilityThreshold). +func (f *chainFreezer) freezeThreshold(db ethdb.KeyValueReader) (uint64, error) { + var ( + head = f.readHeadNumber(db) + final = f.readFinalizedNumber(db) + headLimit uint64 + ) + if head > params.FullImmutabilityThreshold { + headLimit = head - params.FullImmutabilityThreshold + } + if final == 0 && headLimit == 0 { + return 0, errors.New("freezing threshold is not available") + } + if final > headLimit { + return final, nil + } + return headLimit, nil +} + // freeze is a background thread that periodically checks the blockchain for any // import progress and moves ancient data from the fast database into the freezer. // @@ -114,60 +161,39 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { return } } - // Retrieve the freezing threshold. - hash := ReadHeadBlockHash(nfdb) - if hash == (common.Hash{}) { - log.Debug("Current full block hash unavailable") // new chain, empty database + threshold, err := f.freezeThreshold(nfdb) + if err != nil { backoff = true + log.Debug("Current full block not old enough to freeze", "err", err) continue } - number := ReadHeaderNumber(nfdb, hash) - threshold := f.threshold.Load() frozen := f.frozen.Load() - switch { - case number == nil: - log.Error("Current full block number unavailable", "hash", hash) - backoff = true - continue - case *number < threshold: - log.Debug("Current full block not old enough to freeze", "number", *number, "hash", hash, "delay", threshold) - backoff = true - continue - - case *number-threshold <= frozen: - log.Debug("Ancient blocks frozen already", "number", *number, "hash", hash, "frozen", frozen) + // Short circuit if the blocks below threshold are already frozen. + if frozen != 0 && frozen-1 >= threshold { backoff = true + log.Debug("Ancient blocks frozen already", "threshold", threshold, "frozen", frozen) continue } - head := ReadHeader(nfdb, hash, *number) - if head == nil { - log.Error("Current full block unavailable", "number", *number, "hash", hash) - backoff = true - continue - } - // Seems we have data ready to be frozen, process in usable batches var ( - start = time.Now() - first, _ = f.Ancients() - limit = *number - threshold + start = time.Now() + first = frozen // the first block to freeze + last = threshold // the last block to freeze ) - if limit-first > freezerBatchLimit { - limit = first + freezerBatchLimit + if last-first+1 > freezerBatchLimit { + last = freezerBatchLimit + first - 1 } - ancients, err := f.freezeRange(nfdb, first, limit) + ancients, err := f.freezeRange(nfdb, first, last) if err != nil { log.Error("Error in block freeze operation", "err", err) backoff = true continue } - // Batch of blocks have been frozen, flush them before wiping from leveldb if err := f.Sync(); err != nil { log.Crit("Failed to flush frozen tables", "err", err) } - // Wipe out all data from the active database batch := db.NewBatch() for i := 0; i < len(ancients); i++ { @@ -250,8 +276,11 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { } } +// freezeRange moves a batch of chain segments from the fast database to the freezer. +// The parameters (number, limit) specify the relevant block range, both of which +// are included. func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes []common.Hash, err error) { - hashes = make([]common.Hash, 0, limit-number) + hashes = make([]common.Hash, 0, limit-number+1) _, err = f.ModifyAncients(func(op ethdb.AncientWriteOp) error { for ; number <= limit; number++ { @@ -293,11 +322,9 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash if err := op.AppendRaw(ChainFreezerDifficultyTable, number, td); err != nil { return fmt.Errorf("can't write td to Freezer: %v", err) } - hashes = append(hashes, hash) } return nil }) - return hashes, err } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 056b915dbd..ed5368eff0 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -66,16 +66,10 @@ func (frdb *freezerdb) Close() error { // Freeze is a helper method used for external testing to trigger and block until // a freeze cycle completes, without having to sleep for a minute to trigger the // automatic background run. -func (frdb *freezerdb) Freeze(threshold uint64) error { +func (frdb *freezerdb) Freeze() error { if frdb.AncientStore.(*chainFreezer).readonly { return errReadOnly } - // Set the freezer threshold to a temporary value - defer func(old uint64) { - frdb.AncientStore.(*chainFreezer).threshold.Store(old) - }(frdb.AncientStore.(*chainFreezer).threshold.Load()) - frdb.AncientStore.(*chainFreezer).threshold.Store(threshold) - // Trigger a freeze cycle and block until it's done trigger := make(chan struct{}, 1) frdb.AncientStore.(*chainFreezer).trigger <- trigger diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 4471463932..91b4943e59 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -894,7 +894,7 @@ func getChunk(size int, b int) []byte { } // TODO (?) -// - test that if we remove several head-files, aswell as data last data-file, +// - test that if we remove several head-files, as well as data last data-file, // the index is truncated accordingly // Right now, the freezer would fail on these conditions: // 1. have data files d0, d1, d2, d3 diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index b4bd6a382a..2a15663890 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -24,6 +24,7 @@ import ( "math/rand" "os" "path" + "path/filepath" "sync" "testing" @@ -393,7 +394,7 @@ func TestRenameWindows(t *testing.T) { dir2 := t.TempDir() // Create file in dir1 and fill with data - f, err := os.Create(path.Join(dir1, fname)) + f, err := os.Create(filepath.Join(dir1, fname)) if err != nil { t.Fatal(err) } diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 11cf5b40fe..dbf010be0c 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -113,8 +113,8 @@ var ( skeletonHeaderPrefix = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header // Path-based storage scheme of merkle patricia trie. - trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node - trieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node + TrieNodeAccountPrefix = []byte("A") // TrieNodeAccountPrefix + hexPath -> trie node + TrieNodeStoragePrefix = []byte("O") // TrieNodeStoragePrefix + accountHash + hexPath -> trie node stateIDPrefix = []byte("L") // stateIDPrefix + state root -> state id PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage @@ -265,15 +265,15 @@ func stateIDKey(root common.Hash) []byte { return append(stateIDPrefix, root.Bytes()...) } -// accountTrieNodeKey = trieNodeAccountPrefix + nodePath. +// accountTrieNodeKey = TrieNodeAccountPrefix + nodePath. func accountTrieNodeKey(path []byte) []byte { - return append(trieNodeAccountPrefix, path...) + return append(TrieNodeAccountPrefix, path...) } -// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath. +// storageTrieNodeKey = TrieNodeStoragePrefix + accountHash + nodePath. func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte { - buf := make([]byte, len(trieNodeStoragePrefix)+common.HashLength+len(path)) - n := copy(buf, trieNodeStoragePrefix) + buf := make([]byte, len(TrieNodeStoragePrefix)+common.HashLength+len(path)) + n := copy(buf, TrieNodeStoragePrefix) n += copy(buf[n:], accountHash.Bytes()) copy(buf[n:], path) return buf @@ -294,16 +294,16 @@ func IsLegacyTrieNode(key []byte, val []byte) bool { // account trie node in path-based state scheme, and returns the resolved // node path if so. func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) { - if !bytes.HasPrefix(key, trieNodeAccountPrefix) { + if !bytes.HasPrefix(key, TrieNodeAccountPrefix) { return false, nil } // The remaining key should only consist a hex node path // whose length is in the range 0 to 64 (64 is excluded // since leaves are always wrapped with shortNode). - if len(key) >= len(trieNodeAccountPrefix)+common.HashLength*2 { + if len(key) >= len(TrieNodeAccountPrefix)+common.HashLength*2 { return false, nil } - return true, key[len(trieNodeAccountPrefix):] + return true, key[len(TrieNodeAccountPrefix):] } // IsAccountTrieNode reports whether a provided database entry is an account @@ -317,20 +317,20 @@ func IsAccountTrieNode(key []byte) bool { // trie node in path-based state scheme, and returns the resolved account hash // and node path if so. func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) { - if !bytes.HasPrefix(key, trieNodeStoragePrefix) { + if !bytes.HasPrefix(key, TrieNodeStoragePrefix) { return false, common.Hash{}, nil } // The remaining key consists of 2 parts: // - 32 bytes account hash // - hex node path whose length is in the range 0 to 64 - if len(key) < len(trieNodeStoragePrefix)+common.HashLength { + if len(key) < len(TrieNodeStoragePrefix)+common.HashLength { return false, common.Hash{}, nil } - if len(key) >= len(trieNodeStoragePrefix)+common.HashLength+common.HashLength*2 { + if len(key) >= len(TrieNodeStoragePrefix)+common.HashLength+common.HashLength*2 { return false, common.Hash{}, nil } - accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength]) - return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:] + accountHash := common.BytesToHash(key[len(TrieNodeStoragePrefix) : len(TrieNodeStoragePrefix)+common.HashLength]) + return true, accountHash, key[len(TrieNodeStoragePrefix)+common.HashLength:] } // IsStorageTrieNode reports whether a provided database entry is a storage diff --git a/core/rlp_test.go b/core/rlp_test.go index a2fb4937f8..bc37408537 100644 --- a/core/rlp_test.go +++ b/core/rlp_test.go @@ -41,7 +41,7 @@ func getBlock(transactions int, uncles int, dataSize int) *types.Block { funds = big.NewInt(1_000_000_000_000_000_000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{address: {Balance: funds}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, } ) // We need to generate as many blocks +1 as uncles diff --git a/core/state/database.go b/core/state/database.go index b55f870d90..7520923eef 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/triedb" ) const ( @@ -67,7 +68,7 @@ type Database interface { DiskDB() ethdb.KeyValueStore // TrieDB returns the underlying trie database for managing trie nodes. - TrieDB() *trie.Database + TrieDB() *triedb.Database } // Trie is a Ethereum Merkle Patricia trie. @@ -150,17 +151,17 @@ func NewDatabase(db ethdb.Database) Database { // NewDatabaseWithConfig creates a backing store for state. The returned database // is safe for concurrent use and retains a lot of collapsed RLP trie nodes in a // large memory cache. -func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { +func NewDatabaseWithConfig(db ethdb.Database, config *triedb.Config) Database { return &cachingDB{ disk: db, codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - triedb: trie.NewDatabase(db, config), + triedb: triedb.NewDatabase(db, config), } } // NewDatabaseWithNodeDB creates a state database with an already initialized node database. -func NewDatabaseWithNodeDB(db ethdb.Database, triedb *trie.Database) Database { +func NewDatabaseWithNodeDB(db ethdb.Database, triedb *triedb.Database) Database { return &cachingDB{ disk: db, codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), @@ -173,7 +174,7 @@ type cachingDB struct { disk ethdb.KeyValueStore codeSizeCache *lru.Cache[common.Hash, int] codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - triedb *trie.Database + triedb *triedb.Database } // OpenTrie opens the main account trie at a specific root hash. @@ -260,6 +261,6 @@ func (db *cachingDB) DiskDB() ethdb.KeyValueStore { } // TrieDB retrieves any intermediate trie-node caching layer. -func (db *cachingDB) TrieDB() *trie.Database { +func (db *cachingDB) TrieDB() *triedb.Database { return db.triedb } diff --git a/core/state/dump.go b/core/state/dump.go index 55abb50f1c..c9aad4f8e2 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -57,7 +57,6 @@ type DumpAccount struct { Storage map[common.Hash]string `json:"storage,omitempty"` Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode AddressHash hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key - } // Dump represents the full dump in a collected format, as one large map. diff --git a/core/state/metadata.go b/core/state/metadata.go deleted file mode 100644 index 6978e67075..0000000000 --- a/core/state/metadata.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package state - -// BalanceChangeReason is used to indicate the reason for a balance change, useful -// for tracing and reporting. -type BalanceChangeReason byte - -const ( - BalanceChangeUnspecified BalanceChangeReason = 0 - - // Issuance - // BalanceIncreaseRewardMineUncle is a reward for mining an uncle block. - BalanceIncreaseRewardMineUncle BalanceChangeReason = 1 - // BalanceIncreaseRewardMineBlock is a reward for mining a block. - BalanceIncreaseRewardMineBlock BalanceChangeReason = 2 - // BalanceIncreaseWithdrawal is ether withdrawn from the beacon chain. - BalanceIncreaseWithdrawal BalanceChangeReason = 3 - // BalanceIncreaseGenesisBalance is ether allocated at the genesis block. - BalanceIncreaseGenesisBalance BalanceChangeReason = 4 - - // Transaction fees - // BalanceIncreaseRewardTransactionFee is the transaction tip increasing block builder's balance. - BalanceIncreaseRewardTransactionFee BalanceChangeReason = 5 - // BalanceDecreaseGasBuy is spent to purchase gas for execution a transaction. - // Part of this gas will be burnt as per EIP-1559 rules. - BalanceDecreaseGasBuy BalanceChangeReason = 6 - // BalanceIncreaseGasReturn is ether returned for unused gas at the end of execution. - BalanceIncreaseGasReturn BalanceChangeReason = 7 - - // DAO fork - // BalanceIncreaseDaoContract is ether sent to the DAO refund contract. - BalanceIncreaseDaoContract BalanceChangeReason = 8 - // BalanceDecreaseDaoAccount is ether taken from a DAO account to be moved to the refund contract. - BalanceDecreaseDaoAccount BalanceChangeReason = 9 - - // BalanceChangeTransfer is ether transferred via a call. - // it is a decrease for the sender and an increase for the recipient. - BalanceChangeTransfer BalanceChangeReason = 10 - // BalanceChangeTouchAccount is a transfer of zero value. It is only there to - // touch-create an account. - BalanceChangeTouchAccount BalanceChangeReason = 11 - - // BalanceIncreaseSelfdestruct is added to the recipient as indicated by a selfdestructing account. - BalanceIncreaseSelfdestruct BalanceChangeReason = 12 - // BalanceDecreaseSelfdestruct is deducted from a contract due to self-destruct. - BalanceDecreaseSelfdestruct BalanceChangeReason = 13 - // BalanceDecreaseSelfdestructBurn is ether that is sent to an already self-destructed - // account within the same tx (captured at end of tx). - // Note it doesn't account for a self-destruct which appoints itself as recipient. - BalanceDecreaseSelfdestructBurn BalanceChangeReason = 14 -) diff --git a/core/state/metrics.go b/core/state/metrics.go index 64c651461e..7447e44dfd 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -33,5 +33,4 @@ var ( slotDeletionTimer = metrics.NewRegisteredResettingTimer("state/delete/storage/timer", nil) slotDeletionCount = metrics.NewRegisteredMeter("state/delete/storage/slot", nil) slotDeletionSize = metrics.NewRegisteredMeter("state/delete/storage/size", nil) - slotDeletionSkip = metrics.NewRegisteredGauge("state/delete/storage/skip", nil) ) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index a0f95078d0..59c580daca 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) const ( @@ -86,7 +87,7 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) { return nil, errors.New("failed to load head block") } // Offline pruning is only supported in legacy hash based scheme. - triedb := trie.NewDatabase(db, trie.HashDefaults) + triedb := triedb.NewDatabase(db, triedb.HashDefaults) snapconfig := snapshot.Config{ CacheSize: 256, @@ -121,7 +122,7 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta // the trie nodes(and codes) belong to the active state will be filtered // out. A very small part of stale tries will also be filtered because of // the false-positive rate of bloom filter. But the assumption is held here - // that the false-positive is low enough(~0.05%). The probablity of the + // that the false-positive is low enough(~0.05%). The probability of the // dangling node is the state root is super low. So the dangling nodes in // theory will never ever be visited again. var ( @@ -366,7 +367,7 @@ func RecoverPruning(datadir string, db ethdb.Database) error { AsyncBuild: false, } // Offline pruning is only supported in legacy hash based scheme. - triedb := trie.NewDatabase(db, trie.HashDefaults) + triedb := triedb.NewDatabase(db, triedb.HashDefaults) snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root()) if err != nil { return err // The relevant snapshot(s) might not exist @@ -409,7 +410,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db, trie.HashDefaults)) + t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), triedb.NewDatabase(db, triedb.HashDefaults)) if err != nil { return err } @@ -433,7 +434,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { } if acc.Root != types.EmptyRootHash { id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root) - storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db, trie.HashDefaults)) + storageTrie, err := trie.NewStateTrie(id, triedb.NewDatabase(db, triedb.HashDefaults)) if err != nil { return err } diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 1377d0fa3f..70c9f44189 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -43,7 +43,7 @@ var ( aggregatorMemoryLimit = uint64(4 * 1024 * 1024) // aggregatorItemLimit is an approximate number of items that will end up - // in the agregator layer before it's flushed out to disk. A plain account + // in the aggregator layer before it's flushed out to disk. A plain account // weighs around 14B (+hash), a storage slot 32B (+hash), a deleted slot // 0B (+hash). Slots are mostly set/unset in lockstep, so that average at // 16B (+hash). All in all, the average entry seems to be 15+32=47B. Use a diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go index d563b67ca4..f5518a204c 100644 --- a/core/state/snapshot/disklayer.go +++ b/core/state/snapshot/disklayer.go @@ -26,13 +26,13 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) // diskLayer is a low level persistent snapshot built on top of a key-value store. type diskLayer struct { diskdb ethdb.KeyValueStore // Key-value store containing the base snapshot - triedb *trie.Database // Trie node cache for reconstruction purposes + triedb *triedb.Database // Trie node cache for reconstruction purposes cache *fastcache.Cache // Cache to avoid hitting the disk for direct access root common.Hash // Root hash of the base snapshot diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index f95b798515..168458c405 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -139,7 +139,7 @@ func TestDiskMerge(t *testing.T) { // Retrieve all the data through the disk layer and validate it base = snaps.Snapshot(diffRoot) if _, ok := base.(*diskLayer); !ok { - t.Fatalf("update not flattend into the disk layer") + t.Fatalf("update not flattened into the disk layer") } // assertAccount ensures that an account matches the given blob. @@ -362,7 +362,7 @@ func TestDiskPartialMerge(t *testing.T) { // Retrieve all the data through the disk layer and validate it base = snaps.Snapshot(diffRoot) if _, ok := base.(*diskLayer); !ok { - t.Fatalf("test %d: update not flattend into the disk layer", i) + t.Fatalf("test %d: update not flattened into the disk layer", i) } assertAccount(accNoModNoCache, accNoModNoCache[:]) assertAccount(accNoModCache, accNoModCache[:]) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index f455a6db3f..8de4b134d3 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" ) var ( @@ -55,7 +56,7 @@ var ( // generateSnapshot regenerates a brand new snapshot based on an existing state // database and head block asynchronously. The snapshot is returned immediately // and generation is continued in the background until done. -func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash) *diskLayer { +func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *triedb.Database, cache int, root common.Hash) *diskLayer { // Create a new disk layer with an initialized state marker at zero var ( stats = &generatorStats{start: time.Now()} @@ -353,7 +354,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi var resolver trie.NodeResolver if len(result.keys) > 0 { mdb := rawdb.NewMemoryDatabase() - tdb := trie.NewDatabase(mdb, trie.HashDefaults) + tdb := triedb.NewDatabase(mdb, triedb.HashDefaults) defer tdb.Close() snapTrie := trie.NewEmpty(tdb) for i, key := range result.keys { diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 7d941f6285..da93ebc875 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -29,9 +29,10 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) @@ -155,20 +156,20 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { type testHelper struct { diskdb ethdb.Database - triedb *trie.Database + triedb *triedb.Database accTrie *trie.StateTrie nodes *trienode.MergedNodeSet } func newHelper(scheme string) *testHelper { diskdb := rawdb.NewMemoryDatabase() - config := &trie.Config{} + config := &triedb.Config{} if scheme == rawdb.PathScheme { config.PathDB = &pathdb.Config{} // disable caching } else { config.HashDB = &hashdb.Config{} // disable caching } - triedb := trie.NewDatabase(diskdb, config) + triedb := triedb.NewDatabase(diskdb, config) accTrie, _ := trie.NewStateTrie(trie.StateTrieID(types.EmptyRootHash), triedb) return &testHelper{ diskdb: diskdb, diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 4d070208f5..8513e73dd0 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) const journalVersion uint64 = 0 @@ -120,7 +120,7 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. -func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash, cache int, recovery bool, noBuild bool) (snapshot, bool, error) { +func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *triedb.Database, root common.Hash, cache int, recovery bool, noBuild bool) (snapshot, bool, error) { // If snapshotting is disabled (initial sync in progress), don't do anything, // wait for the chain to permit us to do something meaningful if rawdb.ReadSnapshotDisabled(diskdb) { diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 6389842382..5c38cb7252 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) var ( @@ -168,7 +168,7 @@ type Config struct { type Tree struct { config Config // Snapshots configurations diskdb ethdb.KeyValueStore // Persistent database to store the snapshot - triedb *trie.Database // In-memory cache to access the trie through + triedb *triedb.Database // In-memory cache to access the trie through layers map[common.Hash]snapshot // Collection of all known layers lock sync.RWMutex @@ -192,7 +192,7 @@ type Tree struct { // state trie. // - otherwise, the entire snapshot is considered invalid and will be recreated on // a background thread. -func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash) (*Tree, error) { +func New(config Config, diskdb ethdb.KeyValueStore, triedb *triedb.Database, root common.Hash) (*Tree, error) { // Create a new, empty snapshot tree snap := &Tree{ config: config, @@ -258,6 +258,14 @@ func (t *Tree) Disable() { for _, layer := range t.layers { switch layer := layer.(type) { case *diskLayer: + + layer.lock.RLock() + generating := layer.genMarker != nil + layer.lock.RUnlock() + if !generating { + // Generator is already aborted or finished + break + } // If the base layer is generating, abort it if layer.genAbort != nil { abort := make(chan *generatorStats) diff --git a/core/state/state_object.go b/core/state/state_object.go index 3b6715ebe8..910f496341 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -23,9 +23,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/holiman/uint256" @@ -93,7 +93,7 @@ type stateObject struct { // empty returns whether the account is considered empty. func (s *stateObject) empty() bool { - return s.data.Nonce == 0 && s.data.Balance.Sign() == 0 && bytes.Equal(s.data.CodeHash, types.EmptyCodeHash.Bytes()) + return s.data.Nonce == 0 && s.data.Balance.IsZero() && bytes.Equal(s.data.CodeHash, types.EmptyCodeHash.Bytes()) } // newObject creates a state object. @@ -197,9 +197,8 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { if s.db.snap != nil { start := time.Now() enc, err = s.db.snap.Storage(s.addrHash, crypto.Keccak256Hash(key.Bytes())) - if metrics.EnabledExpensive { - s.db.SnapshotStorageReads += time.Since(start) - } + s.db.SnapshotStorageReads += time.Since(start) + if len(enc) > 0 { _, content, _, err := rlp.Split(enc) if err != nil { @@ -217,9 +216,8 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { return common.Hash{} } val, err := tr.GetStorage(s.address, key.Bytes()) - if metrics.EnabledExpensive { - s.db.StorageReads += time.Since(start) - } + s.db.StorageReads += time.Since(start) + if err != nil { s.db.setError(err) return common.Hash{} @@ -243,7 +241,7 @@ func (s *stateObject) SetState(key, value common.Hash) { key: key, prevalue: prev, }) - if s.db.logger != nil { + if s.db.logger != nil && s.db.logger.OnStorageChange != nil { s.db.logger.OnStorageChange(s.address, key, prev, value) } s.setState(key, value) @@ -286,9 +284,8 @@ func (s *stateObject) updateTrie() (Trie, error) { return s.trie, nil } // Track the amount of time wasted on updating the storage trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now()) + // The snapshot storage map for the object var ( storage map[common.Hash][]byte @@ -373,9 +370,8 @@ func (s *stateObject) updateRoot() { return } // Track the amount of time wasted on hashing the storage trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now()) + s.data.Root = tr.Hash() } @@ -389,9 +385,8 @@ func (s *stateObject) commit() (*trienode.NodeSet, error) { return nil, nil } // Track the amount of time wasted on committing the storage trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) + // The trie is currently in an open state and could potentially contain // cached mutations. Call commit to acquire a set of nodes that have been // modified, the set can be nil if nothing to commit. @@ -408,10 +403,10 @@ func (s *stateObject) commit() (*trienode.NodeSet, error) { // AddBalance adds amount to s's balance. // It is used to add funds to the destination account of a transfer. -func (s *stateObject) AddBalance(amount *uint256.Int, reason BalanceChangeReason) { +func (s *stateObject) AddBalance(amount *uint256.Int, reason tracing.BalanceChangeReason) { // EIP161: We must check emptiness for the objects such that the account // clearing (0,0,0 objects) can take effect. - if amount.Sign() == 0 { + if amount.IsZero() { if s.empty() { s.touch() } @@ -422,19 +417,19 @@ func (s *stateObject) AddBalance(amount *uint256.Int, reason BalanceChangeReason // SubBalance removes amount from s's balance. // It is used to remove funds from the origin account of a transfer. -func (s *stateObject) SubBalance(amount *uint256.Int, reason BalanceChangeReason) { - if amount.Sign() == 0 { +func (s *stateObject) SubBalance(amount *uint256.Int, reason tracing.BalanceChangeReason) { + if amount.IsZero() { return } s.SetBalance(new(uint256.Int).Sub(s.Balance(), amount), reason) } -func (s *stateObject) SetBalance(amount *uint256.Int, reason BalanceChangeReason) { +func (s *stateObject) SetBalance(amount *uint256.Int, reason tracing.BalanceChangeReason) { s.db.journal.append(balanceChange{ account: &s.address, prev: new(uint256.Int).Set(s.data.Balance), }) - if s.db.logger != nil { + if s.db.logger != nil && s.db.logger.OnBalanceChange != nil { s.db.logger.OnBalanceChange(s.address, s.Balance().ToBig(), amount.ToBig(), reason) } s.setBalance(amount) @@ -514,7 +509,7 @@ func (s *stateObject) SetCode(codeHash common.Hash, code []byte) { prevhash: s.CodeHash(), prevcode: prevcode, }) - if s.db.logger != nil { + if s.db.logger != nil && s.db.logger.OnCodeChange != nil { s.db.logger.OnCodeChange(s.address, common.BytesToHash(s.CodeHash()), prevcode, codeHash, code) } s.setCode(codeHash, code) @@ -531,7 +526,7 @@ func (s *stateObject) SetNonce(nonce uint64) { account: &s.address, prev: s.data.Nonce, }) - if s.db.logger != nil { + if s.db.logger != nil && s.db.logger.OnNonceChange != nil { s.db.logger.OnNonceChange(s.address, s.data.Nonce, nonce) } s.setNonce(nonce) diff --git a/core/state/state_test.go b/core/state/state_test.go index 10628d400a..c6e6db906e 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -23,10 +23,11 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" "github.com/holiman/uint256" ) @@ -43,17 +44,17 @@ func newStateEnv() *stateEnv { func TestDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - tdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) + tdb := NewDatabaseWithConfig(db, &triedb.Config{Preimages: true}) sdb, _ := New(types.EmptyRootHash, tdb, nil) s := &stateEnv{db: db, state: sdb} // generate a few entries obj1 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01})) - obj1.AddBalance(uint256.NewInt(22), BalanceChangeUnspecified) + obj1.AddBalance(uint256.NewInt(22), tracing.BalanceChangeUnspecified) obj2 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01, 0x02})) obj2.SetCode(crypto.Keccak256Hash([]byte{3, 3, 3, 3, 3, 3, 3}), []byte{3, 3, 3, 3, 3, 3, 3}) obj3 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x02})) - obj3.SetBalance(uint256.NewInt(44), BalanceChangeUnspecified) + obj3.SetBalance(uint256.NewInt(44), tracing.BalanceChangeUnspecified) // write some of them to the trie s.state.updateStateObject(obj1) @@ -100,19 +101,19 @@ func TestDump(t *testing.T) { func TestIterativeDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - tdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) + tdb := NewDatabaseWithConfig(db, &triedb.Config{Preimages: true}) sdb, _ := New(types.EmptyRootHash, tdb, nil) s := &stateEnv{db: db, state: sdb} // generate a few entries obj1 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01})) - obj1.AddBalance(uint256.NewInt(22), BalanceChangeUnspecified) + obj1.AddBalance(uint256.NewInt(22), tracing.BalanceChangeUnspecified) obj2 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01, 0x02})) obj2.SetCode(crypto.Keccak256Hash([]byte{3, 3, 3, 3, 3, 3, 3}), []byte{3, 3, 3, 3, 3, 3, 3}) obj3 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x02})) - obj3.SetBalance(uint256.NewInt(44), BalanceChangeUnspecified) + obj3.SetBalance(uint256.NewInt(44), tracing.BalanceChangeUnspecified) obj4 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x00})) - obj4.AddBalance(uint256.NewInt(1337), BalanceChangeUnspecified) + obj4.AddBalance(uint256.NewInt(1337), tracing.BalanceChangeUnspecified) // write some of them to the trie s.state.updateStateObject(obj1) @@ -208,7 +209,7 @@ func TestSnapshot2(t *testing.T) { // db, trie are already non-empty values so0 := state.getStateObject(stateobjaddr0) - so0.SetBalance(uint256.NewInt(42), BalanceChangeUnspecified) + so0.SetBalance(uint256.NewInt(42), tracing.BalanceChangeUnspecified) so0.SetNonce(43) so0.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e'}), []byte{'c', 'a', 'f', 'e'}) so0.selfDestructed = false @@ -220,7 +221,7 @@ func TestSnapshot2(t *testing.T) { // and one with deleted == true so1 := state.getStateObject(stateobjaddr1) - so1.SetBalance(uint256.NewInt(52), BalanceChangeUnspecified) + so1.SetBalance(uint256.NewInt(52), tracing.BalanceChangeUnspecified) so1.SetNonce(53) so1.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e', '2'}), []byte{'c', 'a', 'f', 'e', '2'}) so1.selfDestructed = true diff --git a/core/state/statedb.go b/core/state/statedb.go index 587a4bd477..6fdf341c61 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -26,10 +26,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" @@ -37,35 +37,11 @@ import ( "github.com/holiman/uint256" ) -const ( - // storageDeleteLimit denotes the highest permissible memory allocation - // employed for contract storage deletion. - storageDeleteLimit = 512 * 1024 * 1024 -) - type revision struct { id int journalIndex int } -// StateLogger is used to collect state update traces from EVM transaction -// execution. -// The following hooks are invoked post execution. I.e. looking up state -// after the hook should reflect the new value. -// Note that reference types are actual VM data structures; make copies -// if you need to retain them beyond the current call. -type StateLogger interface { - OnBalanceChange(addr common.Address, prev, new *big.Int, reason BalanceChangeReason) - OnNonceChange(addr common.Address, prev, new uint64) - OnCodeChange(addr common.Address, prevCodeHash common.Hash, prevCode []byte, codeHash common.Hash, code []byte) - OnStorageChange(addr common.Address, slot common.Hash, prev, new common.Hash) - OnLog(log *types.Log) - // OnNewAccount is called when a new account is created. - // Reset indicates an account existed at that address - // which will be replaced. - OnNewAccount(addr common.Address, reset bool) -} - // StateDB structs within the ethereum protocol are used to store anything // within the merkle trie. StateDBs take care of caching and storing // nested states. It's the general query interface to retrieve: @@ -82,7 +58,7 @@ type StateDB struct { prefetcher *triePrefetcher trie Trie hasher crypto.KeccakState - logger StateLogger + logger *tracing.Hooks snaps *snapshot.Tree // Nil if snapshot is not available snap snapshot.Snapshot // Nil if snapshot is not available @@ -125,9 +101,6 @@ type StateDB struct { // Preimages occurred seen by VM in the scope of block. preimages map[common.Hash][]byte - // Enabled precompile contracts - precompiles map[common.Address]struct{} - // Per-transaction access list accessList *accessList @@ -184,7 +157,6 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) stateObjectsDestruct: make(map[common.Address]*types.StateAccount), logs: make(map[common.Hash][]*types.Log), preimages: make(map[common.Hash][]byte), - precompiles: make(map[common.Address]struct{}), journal: newJournal(), accessList: newAccessList(), transientStorage: newTransientStorage(), @@ -197,7 +169,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) } // SetLogger sets the logger for account update hooks. -func (s *StateDB) SetLogger(l StateLogger) { +func (s *StateDB) SetLogger(l *tracing.Hooks) { s.logger = l } @@ -241,7 +213,7 @@ func (s *StateDB) AddLog(log *types.Log) { log.TxHash = s.thash log.TxIndex = uint(s.txIndex) log.Index = s.logSize - if s.logger != nil { + if s.logger != nil && s.logger.OnLog != nil { s.logger.OnLog(log) } s.logs[s.thash] = append(s.logs[s.thash], log) @@ -405,7 +377,7 @@ func (s *StateDB) HasSelfDestructed(addr common.Address) bool { */ // AddBalance adds amount to the account associated with addr. -func (s *StateDB) AddBalance(addr common.Address, amount *uint256.Int, reason BalanceChangeReason) { +func (s *StateDB) AddBalance(addr common.Address, amount *uint256.Int, reason tracing.BalanceChangeReason) { stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.AddBalance(amount, reason) @@ -413,14 +385,14 @@ func (s *StateDB) AddBalance(addr common.Address, amount *uint256.Int, reason Ba } // SubBalance subtracts amount from the account associated with addr. -func (s *StateDB) SubBalance(addr common.Address, amount *uint256.Int, reason BalanceChangeReason) { +func (s *StateDB) SubBalance(addr common.Address, amount *uint256.Int, reason tracing.BalanceChangeReason) { stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.SubBalance(amount, reason) } } -func (s *StateDB) SetBalance(addr common.Address, amount *uint256.Int, reason BalanceChangeReason) { +func (s *StateDB) SetBalance(addr common.Address, amount *uint256.Int, reason tracing.BalanceChangeReason) { stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.SetBalance(amount, reason) @@ -488,8 +460,8 @@ func (s *StateDB) SelfDestruct(addr common.Address) { prev: stateObject.selfDestructed, prevbalance: prev, }) - if s.logger != nil && prev.Sign() > 0 { - s.logger.OnBalanceChange(addr, prev.ToBig(), n.ToBig(), BalanceDecreaseSelfdestruct) + if s.logger != nil && s.logger.OnBalanceChange != nil && prev.Sign() > 0 { + s.logger.OnBalanceChange(addr, prev.ToBig(), n.ToBig(), tracing.BalanceDecreaseSelfdestruct) } stateObject.markSelfdestructed() stateObject.data.Balance = n @@ -540,9 +512,8 @@ func (s *StateDB) GetTransientState(addr common.Address, key common.Hash) common // updateStateObject writes the given object to the trie. func (s *StateDB) updateStateObject(obj *stateObject) { // Track the amount of time wasted on updating the account from the trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) + // Encode the account and update the account trie addr := obj.Address() if err := s.trie.UpdateAccount(addr, &obj.data); err != nil { @@ -572,9 +543,8 @@ func (s *StateDB) updateStateObject(obj *stateObject) { // deleteStateObject removes the given object from the state trie. func (s *StateDB) deleteStateObject(obj *stateObject) { // Track the amount of time wasted on deleting the account from the trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) + // Delete the account from the trie addr := obj.Address() if err := s.trie.DeleteAccount(addr); err != nil { @@ -606,9 +576,8 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { if s.snap != nil { start := time.Now() acc, err := s.snap.Account(crypto.HashData(s.hasher, addr.Bytes())) - if metrics.EnabledExpensive { - s.SnapshotAccountReads += time.Since(start) - } + s.SnapshotAccountReads += time.Since(start) + if err == nil { if acc == nil { return nil @@ -632,9 +601,8 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { start := time.Now() var err error data, err = s.trie.GetAccount(addr) - if metrics.EnabledExpensive { - s.AccountReads += time.Since(start) - } + s.AccountReads += time.Since(start) + if err != nil { s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %w", addr.Bytes(), err)) return nil @@ -667,9 +635,6 @@ func (s *StateDB) getOrNewStateObject(addr common.Address) *stateObject { func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) { prev = s.getDeletedStateObject(addr) // Note, prev might have been deleted, we need that! newobj = newObject(s, addr, nil) - if s.logger != nil { - s.logger.OnNewAccount(addr, prev != nil) - } if prev == nil { s.journal.append(createObjectChange{account: &addr}) } else { @@ -878,8 +843,8 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { obj.deleted = true // If ether was sent to account post-selfdestruct it is burnt. - if bal := obj.Balance(); bal.Sign() != 0 && s.logger != nil { - s.logger.OnBalanceChange(obj.address, bal.ToBig(), new(big.Int), BalanceDecreaseSelfdestructBurn) + if bal := obj.Balance(); s.logger != nil && s.logger.OnBalanceChange != nil && obj.selfDestructed && bal.Sign() != 0 { + s.logger.OnBalanceChange(obj.address, bal.ToBig(), new(big.Int), tracing.BalanceDecreaseSelfdestructBurn) } // We need to maintain account deletions explicitly (will remain // set indefinitely). Note only the first occurred self-destruct @@ -970,9 +935,8 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { s.stateObjectsPending = make(map[common.Address]struct{}) } // Track the amount of time wasted on hashing the account trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) + return s.trie.Hash() } @@ -996,10 +960,10 @@ func (s *StateDB) clearJournalAndRefund() { // of a specific account. It leverages the associated state snapshot for fast // storage iteration and constructs trie node deletion markers by creating // stack trie with iterated slots. -func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { iter, err := s.snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{}) if err != nil { - return false, 0, nil, nil, err + return 0, nil, nil, err } defer iter.Release() @@ -1015,40 +979,37 @@ func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (boo }) stack := trie.NewStackTrie(options) for iter.Next() { - if size > storageDeleteLimit { - return true, size, nil, nil, nil - } slot := common.CopyBytes(iter.Slot()) if err := iter.Error(); err != nil { // error might occur after Slot function - return false, 0, nil, nil, err + return 0, nil, nil, err } size += common.StorageSize(common.HashLength + len(slot)) slots[iter.Hash()] = slot if err := stack.Update(iter.Hash().Bytes(), slot); err != nil { - return false, 0, nil, nil, err + return 0, nil, nil, err } } if err := iter.Error(); err != nil { // error might occur during iteration - return false, 0, nil, nil, err + return 0, nil, nil, err } if stack.Hash() != root { - return false, 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) + return 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) } - return false, size, slots, nodes, nil + return size, slots, nodes, nil } // slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage," // employed when the associated state snapshot is not available. It iterates the // storage slots along with all internal trie nodes via trie directly. -func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie) if err != nil { - return false, 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) + return 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) } it, err := tr.NodeIterator(nil) if err != nil { - return false, 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) + return 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) } var ( size common.StorageSize @@ -1056,9 +1017,6 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r slots = make(map[common.Hash][]byte) ) for it.Next(true) { - if size > storageDeleteLimit { - return true, size, nil, nil, nil - } if it.Leaf() { slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob()) size += common.StorageSize(common.HashLength + len(it.LeafBlob())) @@ -1071,9 +1029,9 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r nodes.AddNode(it.Path(), trienode.NewDeleted()) } if err := it.Error(); err != nil { - return false, 0, nil, nil, err + return 0, nil, nil, err } - return false, size, slots, nodes, nil + return size, slots, nodes, nil } // deleteStorage is designed to delete the storage trie of a designated account. @@ -1081,41 +1039,37 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r // potentially leading to an out-of-memory panic. The function will make an attempt // to utilize an efficient strategy if the associated state snapshot is reachable; // otherwise, it will resort to a less-efficient approach. -func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { var ( - start = time.Now() - err error - aborted bool - size common.StorageSize - slots map[common.Hash][]byte - nodes *trienode.NodeSet + start = time.Now() + err error + size common.StorageSize + slots map[common.Hash][]byte + nodes *trienode.NodeSet ) // The fast approach can be failed if the snapshot is not fully // generated, or it's internally corrupted. Fallback to the slow // one just in case. if s.snap != nil { - aborted, size, slots, nodes, err = s.fastDeleteStorage(addrHash, root) + size, slots, nodes, err = s.fastDeleteStorage(addrHash, root) } if s.snap == nil || err != nil { - aborted, size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root) + size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root) } if err != nil { - return false, nil, nil, err + return nil, nil, err } - if metrics.EnabledExpensive { - if aborted { - slotDeletionSkip.Inc(1) - } - n := int64(len(slots)) + // Report the metrics + n := int64(len(slots)) - slotDeletionMaxCount.UpdateIfGt(int64(len(slots))) - slotDeletionMaxSize.UpdateIfGt(int64(size)) + slotDeletionMaxCount.UpdateIfGt(int64(len(slots))) + slotDeletionMaxSize.UpdateIfGt(int64(size)) - slotDeletionTimer.UpdateSince(start) - slotDeletionCount.Mark(n) - slotDeletionSize.Mark(int64(size)) - } - return aborted, slots, nodes, nil + slotDeletionTimer.UpdateSince(start) + slotDeletionCount.Mark(n) + slotDeletionSize.Mark(int64(size)) + + return slots, nodes, nil } // handleDestruction processes all destruction markers and deletes the account @@ -1142,13 +1096,12 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root // // In case (d), **original** account along with its storages should be deleted, // with their values be tracked as original value. -func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.Address]struct{}, error) { +func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) error { // Short circuit if geth is running with hash mode. This procedure can consume // considerable time and storage deletion isn't supported in hash mode, thus // preemptively avoiding unnecessary expenses. - incomplete := make(map[common.Address]struct{}) if s.db.TrieDB().Scheme() == rawdb.HashScheme { - return incomplete, nil + return nil } for addr, prev := range s.stateObjectsDestruct { // The original account was non-existing, and it's marked as destructed @@ -1171,18 +1124,9 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A continue } // Remove storage slots belong to the account. - aborted, slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) + slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) if err != nil { - return nil, fmt.Errorf("failed to delete storage, err: %w", err) - } - // The storage is too huge to handle, skip it but mark as incomplete. - // For case (d), the account is resurrected might with a few slots - // created. In this case, wipe the entire storage state diff because - // of aborted deletion. - if aborted { - incomplete[addr] = struct{}{} - delete(s.storagesOrigin, addr) - continue + return fmt.Errorf("failed to delete storage, err: %w", err) } if s.storagesOrigin[addr] == nil { s.storagesOrigin[addr] = slots @@ -1194,10 +1138,10 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A } } if err := nodes.Merge(set); err != nil { - return nil, err + return err } } - return incomplete, nil + return nil } // Commit writes the state to the underlying in-memory trie database. @@ -1226,8 +1170,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er codeWriter = s.db.DiskDB().NewBatch() ) // Handle all state deletions first - incomplete, err := s.handleDestruction(nodes) - if err != nil { + if err := s.handleDestruction(nodes); err != nil { return common.Hash{}, err } // Handle all state updates afterwards @@ -1264,10 +1207,8 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er } } // Write the account trie changes, measuring the amount of wasted time - var start time.Time - if metrics.EnabledExpensive { - start = time.Now() - } + start := time.Now() + root, set, err := s.trie.Commit(true) if err != nil { return common.Hash{}, err @@ -1279,23 +1220,23 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er } accountTrieNodesUpdated, accountTrieNodesDeleted = set.Size() } - if metrics.EnabledExpensive { - s.AccountCommits += time.Since(start) + // Report the commit metrics + s.AccountCommits += time.Since(start) + + accountUpdatedMeter.Mark(int64(s.AccountUpdated)) + storageUpdatedMeter.Mark(int64(s.StorageUpdated)) + accountDeletedMeter.Mark(int64(s.AccountDeleted)) + storageDeletedMeter.Mark(int64(s.StorageDeleted)) + accountTrieUpdatedMeter.Mark(int64(accountTrieNodesUpdated)) + accountTrieDeletedMeter.Mark(int64(accountTrieNodesDeleted)) + storageTriesUpdatedMeter.Mark(int64(storageTrieNodesUpdated)) + storageTriesDeletedMeter.Mark(int64(storageTrieNodesDeleted)) + s.AccountUpdated, s.AccountDeleted = 0, 0 + s.StorageUpdated, s.StorageDeleted = 0, 0 - accountUpdatedMeter.Mark(int64(s.AccountUpdated)) - storageUpdatedMeter.Mark(int64(s.StorageUpdated)) - accountDeletedMeter.Mark(int64(s.AccountDeleted)) - storageDeletedMeter.Mark(int64(s.StorageDeleted)) - accountTrieUpdatedMeter.Mark(int64(accountTrieNodesUpdated)) - accountTrieDeletedMeter.Mark(int64(accountTrieNodesDeleted)) - storageTriesUpdatedMeter.Mark(int64(storageTrieNodesUpdated)) - storageTriesDeletedMeter.Mark(int64(storageTrieNodesDeleted)) - s.AccountUpdated, s.AccountDeleted = 0, 0 - s.StorageUpdated, s.StorageDeleted = 0, 0 - } // If snapshotting is enabled, update the snapshot tree with this new version if s.snap != nil { - start := time.Now() + start = time.Now() // Only update if there's a state transition (skip empty Clique blocks) if parent := s.snap.Root(); parent != root { if err := s.snaps.Update(root, parent, s.convertAccountSet(s.stateObjectsDestruct), s.accounts, s.storages); err != nil { @@ -1309,9 +1250,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er log.Warn("Failed to cap snapshot tree", "root", root, "layers", 128, "err", err) } } - if metrics.EnabledExpensive { - s.SnapshotCommits += time.Since(start) - } + s.SnapshotCommits += time.Since(start) s.snap = nil } if root == (common.Hash{}) { @@ -1322,15 +1261,14 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er origin = types.EmptyRootHash } if root != origin { - start := time.Now() - set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete) + start = time.Now() + set := triestate.New(s.accountsOrigin, s.storagesOrigin) if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil { return common.Hash{}, err } s.originalRoot = root - if metrics.EnabledExpensive { - s.TrieDBCommits += time.Since(start) - } + s.TrieDBCommits += time.Since(start) + if s.onCommit != nil { s.onCommit(set) } @@ -1386,14 +1324,6 @@ func (s *StateDB) Prepare(rules params.Rules, sender, coinbase common.Address, d s.transientStorage = newTransientStorage() } -// PrepareBlock prepares the statedb for execution of a block. It tracks -// the addresses of enabled precompiles for debugging purposes. -func (s *StateDB) PrepareBlock(precompiles []common.Address) { - for _, addr := range precompiles { - s.precompiles[addr] = struct{}{} - } -} - // AddAddressToAccessList adds the given address to the access list func (s *StateDB) AddAddressToAccessList(addr common.Address) { if s.accessList.AddAddress(addr) { diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index e4c08cb55b..65cf278108 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -31,12 +31,14 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" "github.com/holiman/uint256" ) @@ -60,7 +62,7 @@ func newStateTestAction(addr common.Address, r *rand.Rand, index int) testAction { name: "SetBalance", fn: func(a testAction, s *StateDB) { - s.SetBalance(addr, uint256.NewInt(uint64(a.args[0])), BalanceChangeUnspecified) + s.SetBalance(addr, uint256.NewInt(uint64(a.args[0])), tracing.BalanceChangeUnspecified) }, args: make([]int64, 1), }, @@ -181,7 +183,7 @@ func (test *stateTest) run() bool { storageList = append(storageList, copy2DSet(states.Storages)) } disk = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabase(disk, &trie.Config{PathDB: pathdb.Defaults}) + tdb = triedb.NewDatabase(disk, &triedb.Config{PathDB: pathdb.Defaults}) sdb = NewDatabaseWithNodeDB(disk, tdb) byzantium = rand.Intn(2) == 0 ) @@ -252,7 +254,7 @@ func (test *stateTest) run() bool { // - the account was indeed not present in trie // - the account is present in new trie, nil->nil is regarded as invalid // - the slots transition is correct -func (test *stateTest) verifyAccountCreation(next common.Hash, db *trie.Database, otr, ntr *trie.Trie, addr common.Address, slots map[common.Hash][]byte) error { +func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, slots map[common.Hash][]byte) error { // Verify account change addrHash := crypto.Keccak256Hash(addr.Bytes()) oBlob, err := otr.Get(addrHash.Bytes()) @@ -303,7 +305,7 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *trie.Database // - the account was indeed present in trie // - the account in old trie matches the provided value // - the slots transition is correct -func (test *stateTest) verifyAccountUpdate(next common.Hash, db *trie.Database, otr, ntr *trie.Trie, addr common.Address, origin []byte, slots map[common.Hash][]byte) error { +func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, origin []byte, slots map[common.Hash][]byte) error { // Verify account change addrHash := crypto.Keccak256Hash(addr.Bytes()) oBlob, err := otr.Get(addrHash.Bytes()) @@ -357,7 +359,7 @@ func (test *stateTest) verifyAccountUpdate(next common.Hash, db *trie.Database, return nil } -func (test *stateTest) verify(root common.Hash, next common.Hash, db *trie.Database, accountsOrigin map[common.Address][]byte, storagesOrigin map[common.Address]map[common.Hash][]byte) error { +func (test *stateTest) verify(root common.Hash, next common.Hash, db *triedb.Database, accountsOrigin map[common.Address][]byte, storagesOrigin map[common.Address]map[common.Hash][]byte) error { otr, err := trie.New(trie.StateTrieID(root), db) if err != nil { return err diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 2bb24acec8..bc8c634479 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -32,13 +32,15 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" "github.com/holiman/uint256" ) @@ -48,14 +50,14 @@ func TestUpdateLeaks(t *testing.T) { // Create an empty state database var ( db = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabase(db, nil) + tdb = triedb.NewDatabase(db, nil) ) state, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(db, tdb), nil) // Update it with some accounts for i := byte(0); i < 255; i++ { addr := common.BytesToAddress([]byte{i}) - state.AddBalance(addr, uint256.NewInt(uint64(11*i)), BalanceChangeUnspecified) + state.AddBalance(addr, uint256.NewInt(uint64(11*i)), tracing.BalanceChangeUnspecified) state.SetNonce(addr, uint64(42*i)) if i%2 == 0 { state.SetState(addr, common.BytesToHash([]byte{i, i, i}), common.BytesToHash([]byte{i, i, i, i})) @@ -84,13 +86,13 @@ func TestIntermediateLeaks(t *testing.T) { // Create two state databases, one transitioning to the final state, the other final from the beginning transDb := rawdb.NewMemoryDatabase() finalDb := rawdb.NewMemoryDatabase() - transNdb := trie.NewDatabase(transDb, nil) - finalNdb := trie.NewDatabase(finalDb, nil) + transNdb := triedb.NewDatabase(transDb, nil) + finalNdb := triedb.NewDatabase(finalDb, nil) transState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(transDb, transNdb), nil) finalState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(finalDb, finalNdb), nil) modify := func(state *StateDB, addr common.Address, i, tweak byte) { - state.SetBalance(addr, uint256.NewInt(uint64(11*i)+uint64(tweak)), BalanceChangeUnspecified) + state.SetBalance(addr, uint256.NewInt(uint64(11*i)+uint64(tweak)), tracing.BalanceChangeUnspecified) state.SetNonce(addr, uint64(42*i+tweak)) if i%2 == 0 { state.SetState(addr, common.Hash{i, i, i, 0}, common.Hash{}) @@ -166,7 +168,7 @@ func TestCopy(t *testing.T) { for i := byte(0); i < 255; i++ { obj := orig.getOrNewStateObject(common.BytesToAddress([]byte{i})) - obj.AddBalance(uint256.NewInt(uint64(i)), BalanceChangeUnspecified) + obj.AddBalance(uint256.NewInt(uint64(i)), tracing.BalanceChangeUnspecified) orig.updateStateObject(obj) } orig.Finalise(false) @@ -183,9 +185,9 @@ func TestCopy(t *testing.T) { copyObj := copy.getOrNewStateObject(common.BytesToAddress([]byte{i})) ccopyObj := ccopy.getOrNewStateObject(common.BytesToAddress([]byte{i})) - origObj.AddBalance(uint256.NewInt(2*uint64(i)), BalanceChangeUnspecified) - copyObj.AddBalance(uint256.NewInt(3*uint64(i)), BalanceChangeUnspecified) - ccopyObj.AddBalance(uint256.NewInt(4*uint64(i)), BalanceChangeUnspecified) + origObj.AddBalance(uint256.NewInt(2*uint64(i)), tracing.BalanceChangeUnspecified) + copyObj.AddBalance(uint256.NewInt(3*uint64(i)), tracing.BalanceChangeUnspecified) + ccopyObj.AddBalance(uint256.NewInt(4*uint64(i)), tracing.BalanceChangeUnspecified) orig.updateStateObject(origObj) copy.updateStateObject(copyObj) @@ -265,14 +267,14 @@ func newTestAction(addr common.Address, r *rand.Rand) testAction { { name: "SetBalance", fn: func(a testAction, s *StateDB) { - s.SetBalance(addr, uint256.NewInt(uint64(a.args[0])), BalanceChangeUnspecified) + s.SetBalance(addr, uint256.NewInt(uint64(a.args[0])), tracing.BalanceChangeUnspecified) }, args: make([]int64, 1), }, { name: "AddBalance", fn: func(a testAction, s *StateDB) { - s.AddBalance(addr, uint256.NewInt(uint64(a.args[0])), BalanceChangeUnspecified) + s.AddBalance(addr, uint256.NewInt(uint64(a.args[0])), tracing.BalanceChangeUnspecified) }, args: make([]int64, 1), }, @@ -535,7 +537,7 @@ func TestTouchDelete(t *testing.T) { s.state, _ = New(root, s.state.db, s.state.snaps) snapshot := s.state.Snapshot() - s.state.AddBalance(common.Address{}, new(uint256.Int), BalanceChangeUnspecified) + s.state.AddBalance(common.Address{}, new(uint256.Int), tracing.BalanceChangeUnspecified) if len(s.state.journal.dirties) != 1 { t.Fatal("expected one dirty state object") @@ -551,7 +553,7 @@ func TestTouchDelete(t *testing.T) { func TestCopyOfCopy(t *testing.T) { state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) addr := common.HexToAddress("aaaa") - state.SetBalance(addr, uint256.NewInt(42), BalanceChangeUnspecified) + state.SetBalance(addr, uint256.NewInt(42), tracing.BalanceChangeUnspecified) if got := state.Copy().GetBalance(addr).Uint64(); got != 42 { t.Fatalf("1st copy fail, expected 42, got %v", got) @@ -574,9 +576,9 @@ func TestCopyCommitCopy(t *testing.T) { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, uint256.NewInt(42), BalanceChangeUnspecified) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42), tracing.BalanceChangeUnspecified) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) @@ -647,9 +649,9 @@ func TestCopyCopyCommitCopy(t *testing.T) { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, uint256.NewInt(42), BalanceChangeUnspecified) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42), tracing.BalanceChangeUnspecified) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) @@ -716,9 +718,9 @@ func TestCommitCopy(t *testing.T) { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, uint256.NewInt(42), BalanceChangeUnspecified) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42), tracing.BalanceChangeUnspecified) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) @@ -765,7 +767,7 @@ func TestDeleteCreateRevert(t *testing.T) { state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) addr := common.BytesToAddress([]byte("so")) - state.SetBalance(addr, uint256.NewInt(1), BalanceChangeUnspecified) + state.SetBalance(addr, uint256.NewInt(1), tracing.BalanceChangeUnspecified) root, _ := state.Commit(0, false) state, _ = New(root, state.db, state.snaps) @@ -775,7 +777,7 @@ func TestDeleteCreateRevert(t *testing.T) { state.Finalise(true) id := state.Snapshot() - state.SetBalance(addr, uint256.NewInt(2), BalanceChangeUnspecified) + state.SetBalance(addr, uint256.NewInt(2), tracing.BalanceChangeUnspecified) state.RevertToSnapshot(id) // Commit the entire state and make sure we don't crash and have the correct state @@ -798,34 +800,34 @@ func TestMissingTrieNodes(t *testing.T) { func testMissingTrieNodes(t *testing.T, scheme string) { // Create an initial state with a few accounts var ( - triedb *trie.Database - memDb = rawdb.NewMemoryDatabase() + tdb *triedb.Database + memDb = rawdb.NewMemoryDatabase() ) if scheme == rawdb.PathScheme { - triedb = trie.NewDatabase(memDb, &trie.Config{PathDB: &pathdb.Config{ + tdb = triedb.NewDatabase(memDb, &triedb.Config{PathDB: &pathdb.Config{ CleanCacheSize: 0, DirtyCacheSize: 0, }}) // disable caching } else { - triedb = trie.NewDatabase(memDb, &trie.Config{HashDB: &hashdb.Config{ + tdb = triedb.NewDatabase(memDb, &triedb.Config{HashDB: &hashdb.Config{ CleanCacheSize: 0, }}) // disable caching } - db := NewDatabaseWithNodeDB(memDb, triedb) + db := NewDatabaseWithNodeDB(memDb, tdb) var root common.Hash state, _ := New(types.EmptyRootHash, db, nil) addr := common.BytesToAddress([]byte("so")) { - state.SetBalance(addr, uint256.NewInt(1), BalanceChangeUnspecified) + state.SetBalance(addr, uint256.NewInt(1), tracing.BalanceChangeUnspecified) state.SetCode(addr, []byte{1, 2, 3}) a2 := common.BytesToAddress([]byte("another")) - state.SetBalance(a2, uint256.NewInt(100), BalanceChangeUnspecified) + state.SetBalance(a2, uint256.NewInt(100), tracing.BalanceChangeUnspecified) state.SetCode(a2, []byte{1, 2, 4}) root, _ = state.Commit(0, false) t.Logf("root: %x", root) // force-flush - triedb.Commit(root, false) + tdb.Commit(root, false) } // Create a new state on the old root state, _ = New(root, db, nil) @@ -845,7 +847,7 @@ func testMissingTrieNodes(t *testing.T, scheme string) { t.Errorf("expected %d, got %d", exp, got) } // Modify the state - state.SetBalance(addr, uint256.NewInt(2), BalanceChangeUnspecified) + state.SetBalance(addr, uint256.NewInt(2), tracing.BalanceChangeUnspecified) root, err := state.Commit(0, false) if err == nil { t.Fatalf("expected error, got root :%x", root) @@ -1032,7 +1034,7 @@ func TestFlushOrderDataLoss(t *testing.T) { // Create a state trie with many accounts and slots var ( memdb = rawdb.NewMemoryDatabase() - triedb = trie.NewDatabase(memdb, nil) + triedb = triedb.NewDatabase(memdb, nil) statedb = NewDatabaseWithNodeDB(memdb, triedb) state, _ = New(types.EmptyRootHash, statedb, nil) ) @@ -1104,7 +1106,7 @@ func TestStateDBTransientStorage(t *testing.T) { func TestResetObject(t *testing.T) { var ( disk = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabase(disk, nil) + tdb = triedb.NewDatabase(disk, nil) db = NewDatabaseWithNodeDB(disk, tdb) snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash) state, _ = New(types.EmptyRootHash, db, snaps) @@ -1113,13 +1115,13 @@ func TestResetObject(t *testing.T) { slotB = common.HexToHash("0x2") ) // Initialize account with balance and storage in first transaction. - state.SetBalance(addr, uint256.NewInt(1), BalanceChangeUnspecified) + state.SetBalance(addr, uint256.NewInt(1), tracing.BalanceChangeUnspecified) state.SetState(addr, slotA, common.BytesToHash([]byte{0x1})) state.IntermediateRoot(true) // Reset account and mutate balance and storages state.CreateAccount(addr) - state.SetBalance(addr, uint256.NewInt(2), BalanceChangeUnspecified) + state.SetBalance(addr, uint256.NewInt(2), tracing.BalanceChangeUnspecified) state.SetState(addr, slotB, common.BytesToHash([]byte{0x2})) root, _ := state.Commit(0, true) @@ -1138,14 +1140,14 @@ func TestResetObject(t *testing.T) { func TestDeleteStorage(t *testing.T) { var ( disk = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabase(disk, nil) + tdb = triedb.NewDatabase(disk, nil) db = NewDatabaseWithNodeDB(disk, tdb) snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash) state, _ = New(types.EmptyRootHash, db, snaps) addr = common.HexToAddress("0x1") ) // Initialize account and populate storage - state.SetBalance(addr, uint256.NewInt(1), BalanceChangeUnspecified) + state.SetBalance(addr, uint256.NewInt(1), tracing.BalanceChangeUnspecified) state.CreateAccount(addr) for i := 0; i < 1000; i++ { slot := common.Hash(uint256.NewInt(uint64(i)).Bytes32()) @@ -1160,12 +1162,12 @@ func TestDeleteStorage(t *testing.T) { obj := fastState.getOrNewStateObject(addr) storageRoot := obj.data.Root - _, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } - _, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } diff --git a/core/state/sync.go b/core/state/sync.go index d6775e8896..411b54eab0 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// NewStateSync create a new state trie download scheduler. +// NewStateSync creates a new state trie download scheduler. func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync { // Register the storage slot callback if the external callback is specified. var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 172cc9b6a7..b7039c9e1c 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -22,13 +22,15 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" "github.com/holiman/uint256" ) @@ -41,16 +43,16 @@ type testAccount struct { } // makeTestState create a sample test state to test node-wise reconstruction. -func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, common.Hash, []*testAccount) { +func makeTestState(scheme string) (ethdb.Database, Database, *triedb.Database, common.Hash, []*testAccount) { // Create an empty state - config := &trie.Config{Preimages: true} + config := &triedb.Config{Preimages: true} if scheme == rawdb.PathScheme { config.PathDB = pathdb.Defaults } else { config.HashDB = hashdb.Defaults } db := rawdb.NewMemoryDatabase() - nodeDb := trie.NewDatabase(db, config) + nodeDb := triedb.NewDatabase(db, config) sdb := NewDatabaseWithNodeDB(db, nodeDb) state, _ := New(types.EmptyRootHash, sdb, nil) @@ -60,7 +62,7 @@ func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, com obj := state.getOrNewStateObject(common.BytesToAddress([]byte{i})) acc := &testAccount{address: common.BytesToAddress([]byte{i})} - obj.AddBalance(uint256.NewInt(uint64(11*i)), BalanceChangeUnspecified) + obj.AddBalance(uint256.NewInt(uint64(11*i)), tracing.BalanceChangeUnspecified) acc.balance = uint256.NewInt(uint64(11 * i)) obj.SetNonce(uint64(42 * i)) @@ -87,7 +89,7 @@ func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, com // checkStateAccounts cross references a reconstructed state with an expected // account array. func checkStateAccounts(t *testing.T, db ethdb.Database, scheme string, root common.Hash, accounts []*testAccount) { - var config trie.Config + var config triedb.Config if scheme == rawdb.PathScheme { config.PathDB = pathdb.Defaults } @@ -114,7 +116,7 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, scheme string, root com // checkStateConsistency checks that all data of a state root is present. func checkStateConsistency(db ethdb.Database, scheme string, root common.Hash) error { - config := &trie.Config{Preimages: true} + config := &triedb.Config{Preimages: true} if scheme == rawdb.PathScheme { config.PathDB = pathdb.Defaults } @@ -130,8 +132,8 @@ func checkStateConsistency(db ethdb.Database, scheme string, root common.Hash) e // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { - dbA := trie.NewDatabase(rawdb.NewMemoryDatabase(), nil) - dbB := trie.NewDatabase(rawdb.NewMemoryDatabase(), &trie.Config{PathDB: pathdb.Defaults}) + dbA := triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil) + dbB := triedb.NewDatabase(rawdb.NewMemoryDatabase(), &triedb.Config{PathDB: pathdb.Defaults}) sync := NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, dbA.Scheme()) if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { @@ -237,7 +239,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, s id := trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root) stTrie, err := trie.New(id, ndb) if err != nil { - t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err) + t.Fatalf("failed to retrieve storage trie for path %x: %v", node.syncPath[1], err) } data, _, err := stTrie.GetNode(node.syncPath[1]) if err != nil { diff --git a/core/state/trie_prefetcher_test.go b/core/state/trie_prefetcher_test.go index fc129cb874..a616adf98f 100644 --- a/core/state/trie_prefetcher_test.go +++ b/core/state/trie_prefetcher_test.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/holiman/uint256" ) @@ -35,9 +36,9 @@ func filledStateDB() *StateDB { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, uint256.NewInt(42), BalanceChangeUnspecified) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42), tracing.BalanceChangeUnspecified) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie for i := 0; i < 100; i++ { sk := common.BigToHash(big.NewInt(int64(i))) state.SetState(addr, sk, sk) // Change the storage trie diff --git a/core/state_processor.go b/core/state_processor.go index 6e46e7a08d..6dd5744c72 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -75,13 +76,11 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg var ( context = NewEVMBlockContext(header, p.bc, nil) vmenv = vm.NewEVM(context, vm.TxContext{}, statedb, p.config, cfg) - rules = vmenv.ChainConfig().Rules(context.BlockNumber, context.Random != nil, context.Time) signer = types.MakeSigner(p.config, header.Number, header.Time) ) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb, p.bc.logger) + ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) } - statedb.PrepareBlock(vm.ActivePrecompiles(rules)) // Iterate over and process the individual transactions for i, tx := range block.Transactions() { msg, err := TransactionToMessage(tx, signer, header.BaseFee) @@ -112,11 +111,20 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // and uses the input parameters for its environment similar to ApplyTransaction. However, // this method takes an already created EVM instance as input. func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (receipt *types.Receipt, err error) { - if evm.Config.Tracer != nil { - evm.Config.Tracer.CaptureTxStart(evm, tx, msg.From) - defer func() { - evm.Config.Tracer.CaptureTxEnd(receipt, err) - }() + if evm.Config.Tracer != nil && evm.Config.Tracer.OnTxStart != nil { + evm.Config.Tracer.OnTxStart(&tracing.VMContext{ + ChainConfig: evm.ChainConfig(), + StateDB: statedb, + BlockNumber: evm.Context.BlockNumber, + Time: evm.Context.Time, + Coinbase: evm.Context.Coinbase, + Random: evm.Context.Random, + }, tx, msg.From) + if evm.Config.Tracer.OnTxEnd != nil { + defer func() { + evm.Config.Tracer.OnTxEnd(receipt, err) + }() + } } // Create a new context to be used in the EVM environment. txContext := NewEVMTxContext(msg) @@ -185,13 +193,7 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo // ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root // contract. This method is exported to be used in tests. -func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb *state.StateDB, logger BlockchainLogger) { - if logger != nil { - logger.OnBeaconBlockRootStart(beaconRoot) - defer func() { - logger.OnBeaconBlockRootEnd() - }() - } +func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb *state.StateDB) { // If EIP-4788 is enabled, we need to invoke the beaconroot storage contract with // the new root msg := &Message{ diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 2f5f0dc02b..7718c0cde4 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -117,12 +117,12 @@ func TestStateProcessorErrors(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{ Config: config, - Alloc: GenesisAlloc{ - common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): GenesisAccount{ + Alloc: types.GenesisAlloc{ + common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): types.Account{ Balance: big.NewInt(1000000000000000000), // 1 ether Nonce: 0, }, - common.HexToAddress("0xfd0810DD14796680f72adf1a371963d0745BCc64"): GenesisAccount{ + common.HexToAddress("0xfd0810DD14796680f72adf1a371963d0745BCc64"): types.Account{ Balance: big.NewInt(1000000000000000000), // 1 ether Nonce: math.MaxUint64, }, @@ -281,8 +281,8 @@ func TestStateProcessorErrors(t *testing.T) { IstanbulBlock: big.NewInt(0), MuirGlacierBlock: big.NewInt(0), }, - Alloc: GenesisAlloc{ - common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): GenesisAccount{ + Alloc: types.GenesisAlloc{ + common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): types.Account{ Balance: big.NewInt(1000000000000000000), // 1 ether Nonce: 0, }, @@ -319,8 +319,8 @@ func TestStateProcessorErrors(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{ Config: config, - Alloc: GenesisAlloc{ - common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): GenesisAccount{ + Alloc: types.GenesisAlloc{ + common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): types.Account{ Balance: big.NewInt(1000000000000000000), // 1 ether Nonce: 0, Code: common.FromHex("0xB0B0FACE"), diff --git a/core/state_transition.go b/core/state_transition.go index 5e3b018f82..2a914f7f10 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -23,7 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" cmath "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto/kzg4844" @@ -68,7 +68,7 @@ func (result *ExecutionResult) Revert() []byte { } // IntrinsicGas computes the 'intrinsic gas' for a message with the given data. -func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028 bool, isEIP3860 bool) (uint64, error) { +func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028, isEIP3860 bool) (uint64, error) { // Set the starting gas for the raw transaction var gas uint64 if isContractCreation && isHomestead { @@ -238,7 +238,7 @@ func (st *StateTransition) to() common.Address { func (st *StateTransition) buyGas() error { mgval := new(big.Int).SetUint64(st.msg.GasLimit) - mgval = mgval.Mul(mgval, st.msg.GasPrice) + mgval.Mul(mgval, st.msg.GasPrice) balanceCheck := new(big.Int).Set(mgval) if st.msg.GasFeeCap != nil { balanceCheck.SetUint64(st.msg.GasLimit) @@ -268,15 +268,14 @@ func (st *StateTransition) buyGas() error { return err } - if st.evm.Config.Tracer != nil { - st.evm.Config.Tracer.OnGasChange(0, st.msg.GasLimit, vm.GasChangeTxInitialBalance) + if st.evm.Config.Tracer != nil && st.evm.Config.Tracer.OnGasChange != nil { + st.evm.Config.Tracer.OnGasChange(0, st.msg.GasLimit, tracing.GasChangeTxInitialBalance) } - - st.gasRemaining += st.msg.GasLimit + st.gasRemaining = st.msg.GasLimit st.initialGas = st.msg.GasLimit mgvalU256, _ := uint256.FromBig(mgval) - st.state.SubBalance(st.msg.From, mgvalU256, state.BalanceDecreaseGasBuy) + st.state.SubBalance(st.msg.From, mgvalU256, tracing.BalanceDecreaseGasBuy) return nil } @@ -404,8 +403,8 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { if st.gasRemaining < gas { return nil, fmt.Errorf("%w: have %d, want %d", ErrIntrinsicGas, st.gasRemaining, gas) } - if t := st.evm.Config.Tracer; t != nil { - t.OnGasChange(st.gasRemaining, st.gasRemaining-gas, vm.GasChangeTxIntrinsicGas) + if t := st.evm.Config.Tracer; t != nil && t.OnGasChange != nil { + t.OnGasChange(st.gasRemaining, st.gasRemaining-gas, tracing.GasChangeTxIntrinsicGas) } st.gasRemaining -= gas @@ -461,7 +460,7 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { } else { fee := new(uint256.Int).SetUint64(st.gasUsed()) fee.Mul(fee, effectiveTipU256) - st.state.AddBalance(st.evm.Context.Coinbase, fee, state.BalanceIncreaseRewardTransactionFee) + st.state.AddBalance(st.evm.Context.Coinbase, fee, tracing.BalanceIncreaseRewardTransactionFee) } return &ExecutionResult{ @@ -479,19 +478,19 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { refund = st.state.GetRefund() } - if st.evm.Config.Tracer != nil && refund > 0 { - st.evm.Config.Tracer.OnGasChange(st.gasRemaining, st.gasRemaining+refund, vm.GasChangeTxRefunds) + if st.evm.Config.Tracer != nil && st.evm.Config.Tracer.OnGasChange != nil && refund > 0 { + st.evm.Config.Tracer.OnGasChange(st.gasRemaining, st.gasRemaining+refund, tracing.GasChangeTxRefunds) } st.gasRemaining += refund // Return ETH for remaining gas, exchanged at the original rate. remaining := uint256.NewInt(st.gasRemaining) - remaining = remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice)) - st.state.AddBalance(st.msg.From, remaining, state.BalanceIncreaseGasReturn) + remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice)) + st.state.AddBalance(st.msg.From, remaining, tracing.BalanceIncreaseGasReturn) - if st.evm.Config.Tracer != nil && st.gasRemaining > 0 { - st.evm.Config.Tracer.OnGasChange(st.gasRemaining, 0, vm.GasChangeTxLeftOverReturned) + if st.evm.Config.Tracer != nil && st.evm.Config.Tracer.OnGasChange != nil && st.gasRemaining > 0 { + st.evm.Config.Tracer.OnGasChange(st.gasRemaining, 0, tracing.GasChangeTxLeftOverReturned) } // Also return remaining gas to the block gas counter so it is diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go new file mode 100644 index 0000000000..48cb4d2027 --- /dev/null +++ b/core/tracing/hooks.go @@ -0,0 +1,275 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tracing + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" +) + +// OpContext provides the context at which the opcode is being +// executed in, including the memory, stack and various contract-level information. +type OpContext interface { + MemoryData() []byte + StackData() []uint256.Int + Caller() common.Address + Address() common.Address + CallValue() *uint256.Int + CallInput() []byte +} + +// StateDB gives tracers access to the whole state. +type StateDB interface { + GetBalance(common.Address) *uint256.Int + GetNonce(common.Address) uint64 + GetCode(common.Address) []byte + GetState(common.Address, common.Hash) common.Hash + Exist(common.Address) bool + GetRefund() uint64 +} + +// VMContext provides the context for the EVM execution. +type VMContext struct { + Coinbase common.Address + BlockNumber *big.Int + Time uint64 + Random *common.Hash + // Effective tx gas price + GasPrice *big.Int + ChainConfig *params.ChainConfig + StateDB StateDB +} + +// BlockEvent is emitted upon tracing an incoming block. +// It contains the block as well as consensus related information. +type BlockEvent struct { + Block *types.Block + TD *big.Int + Finalized *types.Header + Safe *types.Header +} + +type ( + /* + - VM events - + */ + + // TxStartHook is called before the execution of a transaction starts. + // Call simulations don't come with a valid signature. `from` field + // to be used for address of the caller. + TxStartHook = func(vm *VMContext, tx *types.Transaction, from common.Address) + + // TxEndHook is called after the execution of a transaction ends. + TxEndHook = func(receipt *types.Receipt, err error) + + // EnterHook is invoked when the processing of a message starts. + EnterHook = func(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) + + // ExitHook is invoked when the processing of a message ends. + // `revert` is true when there was an error during the execution. + // Exceptionally, before the homestead hardfork a contract creation that + // ran out of gas when attempting to persist the code to database did not + // count as a call failure and did not cause a revert of the call. This will + // be indicated by `reverted == false` and `err == ErrCodeStoreOutOfGas`. + ExitHook = func(depth int, output []byte, gasUsed uint64, err error, reverted bool) + + // OpcodeHook is invoked just prior to the execution of an opcode. + OpcodeHook = func(pc uint64, op byte, gas, cost uint64, scope OpContext, rData []byte, depth int, err error) + + // FaultHook is invoked when an error occurs during the execution of an opcode. + FaultHook = func(pc uint64, op byte, gas, cost uint64, scope OpContext, depth int, err error) + + // GasChangeHook is invoked when the gas changes. + GasChangeHook = func(old, new uint64, reason GasChangeReason) + + /* + - Chain events - + */ + + // BlockchainInitHook is called when the blockchain is initialized. + BlockchainInitHook = func(chainConfig *params.ChainConfig) + + // BlockStartHook is called before executing `block`. + // `td` is the total difficulty prior to `block`. + BlockStartHook = func(event BlockEvent) + + // BlockEndHook is called after executing a block. + BlockEndHook = func(err error) + + // SkippedBlockHook indicates a block was skipped during processing + // due to it being known previously. This can happen e.g. when recovering + // from a crash. + SkippedBlockHook = func(event BlockEvent) + + // GenesisBlockHook is called when the genesis block is being processed. + GenesisBlockHook = func(genesis *types.Block, alloc types.GenesisAlloc) + + /* + - State events - + */ + + // BalanceChangeHook is called when the balance of an account changes. + BalanceChangeHook = func(addr common.Address, prev, new *big.Int, reason BalanceChangeReason) + + // NonceChangeHook is called when the nonce of an account changes. + NonceChangeHook = func(addr common.Address, prev, new uint64) + + // CodeChangeHook is called when the code of an account changes. + CodeChangeHook = func(addr common.Address, prevCodeHash common.Hash, prevCode []byte, codeHash common.Hash, code []byte) + + // StorageChangeHook is called when the storage of an account changes. + StorageChangeHook = func(addr common.Address, slot common.Hash, prev, new common.Hash) + + // LogHook is called when a log is emitted. + LogHook = func(log *types.Log) +) + +type Hooks struct { + // VM events + OnTxStart TxStartHook + OnTxEnd TxEndHook + OnEnter EnterHook + OnExit ExitHook + OnOpcode OpcodeHook + OnFault FaultHook + OnGasChange GasChangeHook + // Chain events + OnBlockchainInit BlockchainInitHook + OnBlockStart BlockStartHook + OnBlockEnd BlockEndHook + OnSkippedBlock SkippedBlockHook + OnGenesisBlock GenesisBlockHook + // State events + OnBalanceChange BalanceChangeHook + OnNonceChange NonceChangeHook + OnCodeChange CodeChangeHook + OnStorageChange StorageChangeHook + OnLog LogHook +} + +// BalanceChangeReason is used to indicate the reason for a balance change, useful +// for tracing and reporting. +type BalanceChangeReason byte + +const ( + BalanceChangeUnspecified BalanceChangeReason = 0 + + // Issuance + // BalanceIncreaseRewardMineUncle is a reward for mining an uncle block. + BalanceIncreaseRewardMineUncle BalanceChangeReason = 1 + // BalanceIncreaseRewardMineBlock is a reward for mining a block. + BalanceIncreaseRewardMineBlock BalanceChangeReason = 2 + // BalanceIncreaseWithdrawal is ether withdrawn from the beacon chain. + BalanceIncreaseWithdrawal BalanceChangeReason = 3 + // BalanceIncreaseGenesisBalance is ether allocated at the genesis block. + BalanceIncreaseGenesisBalance BalanceChangeReason = 4 + + // Transaction fees + // BalanceIncreaseRewardTransactionFee is the transaction tip increasing block builder's balance. + BalanceIncreaseRewardTransactionFee BalanceChangeReason = 5 + // BalanceDecreaseGasBuy is spent to purchase gas for execution a transaction. + // Part of this gas will be burnt as per EIP-1559 rules. + BalanceDecreaseGasBuy BalanceChangeReason = 6 + // BalanceIncreaseGasReturn is ether returned for unused gas at the end of execution. + BalanceIncreaseGasReturn BalanceChangeReason = 7 + + // DAO fork + // BalanceIncreaseDaoContract is ether sent to the DAO refund contract. + BalanceIncreaseDaoContract BalanceChangeReason = 8 + // BalanceDecreaseDaoAccount is ether taken from a DAO account to be moved to the refund contract. + BalanceDecreaseDaoAccount BalanceChangeReason = 9 + + // BalanceChangeTransfer is ether transferred via a call. + // it is a decrease for the sender and an increase for the recipient. + BalanceChangeTransfer BalanceChangeReason = 10 + // BalanceChangeTouchAccount is a transfer of zero value. It is only there to + // touch-create an account. + BalanceChangeTouchAccount BalanceChangeReason = 11 + + // BalanceIncreaseSelfdestruct is added to the recipient as indicated by a selfdestructing account. + BalanceIncreaseSelfdestruct BalanceChangeReason = 12 + // BalanceDecreaseSelfdestruct is deducted from a contract due to self-destruct. + BalanceDecreaseSelfdestruct BalanceChangeReason = 13 + // BalanceDecreaseSelfdestructBurn is ether that is sent to an already self-destructed + // account within the same tx (captured at end of tx). + // Note it doesn't account for a self-destruct which appoints itself as recipient. + BalanceDecreaseSelfdestructBurn BalanceChangeReason = 14 +) + +// GasChangeReason is used to indicate the reason for a gas change, useful +// for tracing and reporting. +// +// There is essentially two types of gas changes, those that can be emitted once per transaction +// and those that can be emitted on a call basis, so possibly multiple times per transaction. +// +// They can be recognized easily by their name, those that start with `GasChangeTx` are emitted +// once per transaction, while those that start with `GasChangeCall` are emitted on a call basis. +type GasChangeReason byte + +const ( + GasChangeUnspecified GasChangeReason = 0 + + // GasChangeTxInitialBalance is the initial balance for the call which will be equal to the gasLimit of the call. There is only + // one such gas change per transaction. + GasChangeTxInitialBalance GasChangeReason = 1 + // GasChangeTxIntrinsicGas is the amount of gas that will be charged for the intrinsic cost of the transaction, there is + // always exactly one of those per transaction. + GasChangeTxIntrinsicGas GasChangeReason = 2 + // GasChangeTxRefunds is the sum of all refunds which happened during the tx execution (e.g. storage slot being cleared) + // this generates an increase in gas. There is at most one of such gas change per transaction. + GasChangeTxRefunds GasChangeReason = 3 + // GasChangeTxLeftOverReturned is the amount of gas left over at the end of transaction's execution that will be returned + // to the chain. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas + // left at the end of execution, no such even will be emitted. The returned gas's value in Wei is returned to caller. + // There is at most one of such gas change per transaction. + GasChangeTxLeftOverReturned GasChangeReason = 4 + + // GasChangeCallInitialBalance is the initial balance for the call which will be equal to the gasLimit of the call. There is only + // one such gas change per call. + GasChangeCallInitialBalance GasChangeReason = 5 + // GasChangeCallLeftOverReturned is the amount of gas left over that will be returned to the caller, this change will always + // be a negative change as we "drain" left over gas towards 0. If there was no gas left at the end of execution, no such even + // will be emitted. + GasChangeCallLeftOverReturned GasChangeReason = 6 + // GasChangeCallLeftOverRefunded is the amount of gas that will be refunded to the call after the child call execution it + // executed completed. This value is always positive as we are giving gas back to the you, the left over gas of the child. + // If there was no gas left to be refunded, no such even will be emitted. + GasChangeCallLeftOverRefunded GasChangeReason = 7 + // GasChangeCallContractCreation is the amount of gas that will be burned for a CREATE. + GasChangeCallContractCreation GasChangeReason = 8 + // GasChangeContractCreation is the amount of gas that will be burned for a CREATE2. + GasChangeCallContractCreation2 GasChangeReason = 9 + // GasChangeCallCodeStorage is the amount of gas that will be charged for code storage. + GasChangeCallCodeStorage GasChangeReason = 10 + // GasChangeCallOpCode is the amount of gas that will be charged for an opcode executed by the EVM, exact opcode that was + // performed can be check by `OnOpcode` handling. + GasChangeCallOpCode GasChangeReason = 11 + // GasChangeCallPrecompiledContract is the amount of gas that will be charged for a precompiled contract execution. + GasChangeCallPrecompiledContract GasChangeReason = 12 + // GasChangeCallStorageColdAccess is the amount of gas that will be charged for a cold storage access as controlled by EIP2929 rules. + GasChangeCallStorageColdAccess GasChangeReason = 13 + // GasChangeCallFailedExecution is the burning of the remaining gas when the execution failed without a revert. + GasChangeCallFailedExecution GasChangeReason = 14 + + // GasChangeIgnored is a special value that can be used to indicate that the gas change should be ignored as + // it will be "manually" tracked by a direct emit of the gas change event. + GasChangeIgnored GasChangeReason = 0xFF +) diff --git a/core/txindexer.go b/core/txindexer.go index 61de41947c..70fe5f3322 100644 --- a/core/txindexer.go +++ b/core/txindexer.go @@ -127,9 +127,10 @@ func (indexer *txIndexer) loop(chain *BlockChain) { // Listening to chain events and manipulate the transaction indexes. var ( - stop chan struct{} // Non-nil if background routine is active. - done chan struct{} // Non-nil if background routine is active. - lastHead uint64 // The latest announced chain head (whose tx indexes are assumed created) + stop chan struct{} // Non-nil if background routine is active. + done chan struct{} // Non-nil if background routine is active. + lastHead uint64 // The latest announced chain head (whose tx indexes are assumed created) + lastTail = rawdb.ReadTxIndexTail(indexer.db) // The oldest indexed block, nil means nothing indexed headCh = make(chan ChainHeadEvent) sub = chain.SubscribeChainHeadEvent(headCh) @@ -156,8 +157,9 @@ func (indexer *txIndexer) loop(chain *BlockChain) { case <-done: stop = nil done = nil + lastTail = rawdb.ReadTxIndexTail(indexer.db) case ch := <-indexer.progress: - ch <- indexer.report(lastHead) + ch <- indexer.report(lastHead, lastTail) case ch := <-indexer.term: if stop != nil { close(stop) @@ -173,11 +175,7 @@ func (indexer *txIndexer) loop(chain *BlockChain) { } // report returns the tx indexing progress. -func (indexer *txIndexer) report(head uint64) TxIndexProgress { - var ( - remaining uint64 - tail = rawdb.ReadTxIndexTail(indexer.db) - ) +func (indexer *txIndexer) report(head uint64, tail *uint64) TxIndexProgress { total := indexer.limit if indexer.limit == 0 || total > head { total = head + 1 // genesis included @@ -188,6 +186,7 @@ func (indexer *txIndexer) report(head uint64) TxIndexProgress { } // The value of indexed might be larger than total if some blocks need // to be unindexed, avoiding a negative remaining. + var remaining uint64 if indexed < total { remaining = total - indexed } diff --git a/core/txindexer_test.go b/core/txindexer_test.go index 66f26edaeb..7b5ff1f206 100644 --- a/core/txindexer_test.go +++ b/core/txindexer_test.go @@ -39,7 +39,7 @@ func TestTxIndexer(t *testing.T) { gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, BaseFee: big.NewInt(params.InitialBaseFee), } engine = ethash.NewFaker() @@ -85,7 +85,7 @@ func TestTxIndexer(t *testing.T) { for number := *tail; number <= chainHead; number += 1 { verifyIndexes(db, number, true) } - progress := indexer.report(chainHead) + progress := indexer.report(chainHead, tail) if !progress.Done() { t.Fatalf("Expect fully indexed") } diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index f4162acac3..6dbcc9dadc 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -268,7 +268,7 @@ func newBlobTxMeta(id uint64, size uint32, tx *types.Transaction) *blobTxMeta { // going up, crossing the smaller positive jump counter). As such, the pool // cares only about the min of the two delta values for eviction priority. // -// priority = min(delta-basefee, delta-blobfee) +// priority = min(deltaBasefee, deltaBlobfee) // // - The above very aggressive dimensionality and noise reduction should result // in transaction being grouped into a small number of buckets, the further @@ -280,7 +280,7 @@ func newBlobTxMeta(id uint64, size uint32, tx *types.Transaction) *blobTxMeta { // with high fee caps since it could enable pool wars. As such, any positive // priority will be grouped together. // -// priority = min(delta-basefee, delta-blobfee, 0) +// priority = min(deltaBasefee, deltaBlobfee, 0) // // Optimisation tradeoffs: // @@ -342,7 +342,7 @@ func (p *BlobPool) Filter(tx *types.Transaction) bool { // Init sets the gas price needed to keep a transaction in the pool and the chain // head to allow balance / nonce checks. The transaction journal will be loaded // from disk and filtered based on the provided starting settings. -func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.AddressReserver) error { +func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.AddressReserver) error { p.reserve = reserve var ( @@ -360,7 +360,7 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr } } // Initialize the state with head block, or fallback to empty one in - // case the head state is not available(might occur when node is not + // case the head state is not available (might occur when node is not // fully synced). state, err := p.chain.StateAt(head.Root) if err != nil { @@ -371,14 +371,14 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr } p.head, p.state = head, state - // Index all transactions on disk and delete anything inprocessable + // Index all transactions on disk and delete anything unprocessable var fails []uint64 index := func(id uint64, size uint32, blob []byte) { if p.parseTransaction(id, size, blob) != nil { fails = append(fails, id) } } - store, err := billy.Open(billy.Options{Path: queuedir}, newSlotter(), index) + store, err := billy.Open(billy.Options{Path: queuedir, Repair: true}, newSlotter(), index) if err != nil { return err } @@ -386,6 +386,8 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr if len(fails) > 0 { log.Warn("Dropping invalidated blob transactions", "ids", fails) + dropInvalidMeter.Mark(int64(len(fails))) + for _, id := range fails { if err := p.store.Delete(id); err != nil { p.Close() @@ -400,7 +402,7 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr } var ( basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head)) - blobfee = uint256.MustFromBig(big.NewInt(params.BlobTxMinBlobGasprice)) + blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice) ) if p.head.ExcessBlobGas != nil { blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(*p.head.ExcessBlobGas)) @@ -418,7 +420,7 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr basefeeGauge.Update(int64(basefee.Uint64())) blobfeeGauge.Update(int64(blobfee.Uint64())) - p.SetGasTip(gasTip) + p.SetGasTip(new(big.Int).SetUint64(gasTip)) // Since the user might have modified their pool's capacity, evict anything // above the current allowance @@ -434,8 +436,10 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr // Close closes down the underlying persistent store. func (p *BlobPool) Close() error { var errs []error - if err := p.limbo.Close(); err != nil { - errs = append(errs, err) + if p.limbo != nil { // Close might be invoked due to error in constructor, before p,limbo is set + if err := p.limbo.Close(); err != nil { + errs = append(errs, err) + } } if err := p.store.Close(); err != nil { errs = append(errs, err) @@ -456,7 +460,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { tx := new(types.Transaction) if err := rlp.DecodeBytes(blob, tx); err != nil { // This path is impossible unless the disk data representation changes - // across restarts. For that ever unprobable case, recover gracefully + // across restarts. For that ever improbable case, recover gracefully // by ignoring this data entry. log.Error("Failed to decode blob pool entry", "id", id, "err", err) return err @@ -467,11 +471,17 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { } meta := newBlobTxMeta(id, size, tx) - + if _, exists := p.lookup[meta.hash]; exists { + // This path is only possible after a crash, where deleted items are not + // removed via the normal shutdown-startup procedure and thus may get + // partially resurrected. + log.Error("Rejecting duplicate blob pool entry", "id", id, "hash", tx.Hash()) + return errors.New("duplicate blob entry") + } sender, err := p.signer.Sender(tx) if err != nil { // This path is impossible unless the signature validity changes across - // restarts. For that ever unprobable case, recover gracefully by ignoring + // restarts. For that ever improbable case, recover gracefully by ignoring // this data entry. log.Error("Failed to recover blob tx sender", "id", id, "hash", tx.Hash(), "err", err) return err @@ -530,15 +540,17 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 } delete(p.index, addr) delete(p.spent, addr) - if inclusions != nil { // only during reorgs will the heap will be initialized + if inclusions != nil { // only during reorgs will the heap be initialized heap.Remove(p.evict, p.evict.index[addr]) } p.reserve(addr, false) if gapped { log.Warn("Dropping dangling blob transactions", "from", addr, "missing", next, "drop", nonces, "ids", ids) + dropDanglingMeter.Mark(int64(len(ids))) } else { log.Trace("Dropping filled blob transactions", "from", addr, "filled", nonces, "ids", ids) + dropFilledMeter.Mark(int64(len(ids))) } for _, id := range ids { if err := p.store.Delete(id); err != nil { @@ -569,6 +581,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 txs = txs[1:] } log.Trace("Dropping overlapped blob transactions", "from", addr, "overlapped", nonces, "ids", ids, "left", len(txs)) + dropOverlappedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -600,10 +614,30 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 } continue } - // Sanity check that there's no double nonce. This case would be a coding - // error, but better know about it + // Sanity check that there's no double nonce. This case would generally + // be a coding error, so better know about it. + // + // Also, Billy behind the blobpool does not journal deletes. A process + // crash would result in previously deleted entities being resurrected. + // That could potentially cause a duplicate nonce to appear. if txs[i].nonce == txs[i-1].nonce { - log.Error("Duplicate nonce blob transaction", "from", addr, "nonce", txs[i].nonce) + id := p.lookup[txs[i].hash] + + log.Error("Dropping repeat nonce blob transaction", "from", addr, "nonce", txs[i].nonce, "id", id) + dropRepeatedMeter.Mark(1) + + p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[i].costCap) + p.stored -= uint64(txs[i].size) + delete(p.lookup, txs[i].hash) + + if err := p.store.Delete(id); err != nil { + log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) + } + txs = append(txs[:i], txs[i+1:]...) + p.index[addr] = txs + + i-- + continue } // Otherwise if there's a nonce gap evict all later transactions var ( @@ -621,6 +655,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 txs = txs[:i] log.Error("Dropping gapped blob transactions", "from", addr, "missing", txs[i-1].nonce+1, "drop", nonces, "ids", ids) + dropGappedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -657,7 +693,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 if len(txs) == 0 { delete(p.index, addr) delete(p.spent, addr) - if inclusions != nil { // only during reorgs will the heap will be initialized + if inclusions != nil { // only during reorgs will the heap be initialized heap.Remove(p.evict, p.evict.index[addr]) } p.reserve(addr, false) @@ -665,6 +701,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 p.index[addr] = txs } log.Warn("Dropping overdrafted blob transactions", "from", addr, "balance", balance, "spent", spent, "drop", nonces, "ids", ids) + dropOverdraftedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -695,6 +733,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 p.index[addr] = txs log.Warn("Dropping overcapped blob transactions", "from", addr, "kept", len(txs), "drop", nonces, "ids", ids) + dropOvercappedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -711,7 +751,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 // offload removes a tracked blob transaction from the pool and moves it into the // limbo for tracking until finality. // -// The method may log errors for various unexpcted scenarios but will not return +// The method may log errors for various unexpected scenarios but will not return // any of it since there's no clear error case. Some errors may be due to coding // issues, others caused by signers mining MEV stuff or swapping transactions. In // all cases, the pool needs to continue operating. @@ -769,7 +809,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { } } // Recheck the account's pooled transactions to drop included and - // invalidated one + // invalidated ones p.recheck(addr, inclusions) } if len(adds) > 0 { @@ -952,7 +992,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error { return err } - // Update the indixes and metrics + // Update the indices and metrics meta := newBlobTxMeta(id, p.store.Size(id), tx) if _, ok := p.index[addr]; !ok { if err := p.reserve(addr, true); err != nil { @@ -1019,6 +1059,8 @@ func (p *BlobPool) SetGasTip(tip *big.Int) { } // Clear out the transactions from the data store log.Warn("Dropping underpriced blob transaction", "from", addr, "rejected", tx.nonce, "tip", tx.execTipCap, "want", tip, "drop", nonces, "ids", ids) + dropUnderpricedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete dropped transaction", "id", id, "err", err) @@ -1089,8 +1131,12 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error { next = p.state.GetNonce(from) ) if uint64(len(p.index[from])) > tx.Nonce()-next { - // Account can support the replacement, but the price bump must also be met prev := p.index[from][int(tx.Nonce()-next)] + // Ensure the transaction is different than the one tracked locally + if prev.hash == tx.Hash() { + return txpool.ErrAlreadyKnown + } + // Account can support the replacement, but the price bump must also be met switch { case tx.GasFeeCapIntCmp(prev.execFeeCap.ToBig()) <= 0: return fmt.Errorf("%w: new tx gas fee cap %v <= %v queued", txpool.ErrReplaceUnderpriced, tx.GasFeeCap(), prev.execFeeCap) @@ -1161,7 +1207,7 @@ func (p *BlobPool) Get(hash common.Hash) *types.Transaction { } // Add inserts a set of blob transactions into the pool if they pass validation (both -// consensus validity and pool restictions). +// consensus validity and pool restrictions). func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error { var ( adds = make([]*types.Transaction, 0, len(txs)) @@ -1181,10 +1227,10 @@ func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error } // Add inserts a new blob transaction into the pool if it passes validation (both -// consensus validity and pool restictions). +// consensus validity and pool restrictions). func (p *BlobPool) add(tx *types.Transaction) (err error) { // The blob pool blocks on adding a transaction. This is because blob txs are - // only even pulled form the network, so this method will act as the overload + // only even pulled from the network, so this method will act as the overload // protection for fetches. waitStart := time.Now() p.lock.Lock() @@ -1198,6 +1244,22 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { // Ensure the transaction is valid from all perspectives if err := p.validateTx(tx); err != nil { log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err) + switch { + case errors.Is(err, txpool.ErrUnderpriced): + addUnderpricedMeter.Mark(1) + case errors.Is(err, core.ErrNonceTooLow): + addStaleMeter.Mark(1) + case errors.Is(err, core.ErrNonceTooHigh): + addGappedMeter.Mark(1) + case errors.Is(err, core.ErrInsufficientFunds): + addOverdraftedMeter.Mark(1) + case errors.Is(err, txpool.ErrAccountLimitExceeded): + addOvercappedMeter.Mark(1) + case errors.Is(err, txpool.ErrReplaceUnderpriced): + addNoreplaceMeter.Mark(1) + default: + addInvalidMeter.Mark(1) + } return err } // If the address is not yet known, request exclusivity to track the account @@ -1205,6 +1267,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { from, _ := types.Sender(p.signer, tx) // already validated above if _, ok := p.index[from]; !ok { if err := p.reserve(from, true); err != nil { + addNonExclusiveMeter.Mark(1) return err } defer func() { @@ -1244,6 +1307,8 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { } if len(p.index[from]) > offset { // Transaction replaces a previously queued one + dropReplacedMeter.Mark(1) + prev := p.index[from][offset] if err := p.store.Delete(prev.id); err != nil { // Shitty situation, but try to recover gracefully instead of going boom @@ -1322,6 +1387,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { } p.updateStorageMetrics() + addValidMeter.Mark(1) return nil } @@ -1371,7 +1437,9 @@ func (p *BlobPool) drop() { } } // Remove the transaction from the data store - log.Warn("Evicting overflown blob transaction", "from", from, "evicted", drop.nonce, "id", drop.id) + log.Debug("Evicting overflown blob transaction", "from", from, "evicted", drop.nonce, "id", drop.id) + dropOverflownMeter.Mark(1) + if err := p.store.Delete(drop.id); err != nil { log.Error("Failed to drop evicted transaction", "id", drop.id, "err", err) } @@ -1379,7 +1447,15 @@ func (p *BlobPool) drop() { // Pending retrieves all currently processable transactions, grouped by origin // account and sorted by nonce. -func (p *BlobPool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTransaction { +// +// The transactions can also be pre-filtered by the dynamic fee components to +// reduce allocations and load on downstream subsystems. +func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { + // If only plain transactions are requested, this pool is unsuitable as it + // contains none, don't even bother. + if filter.OnlyPlainTxs { + return nil + } // Track the amount of time waiting to retrieve the list of pending blob txs // from the pool and the amount of time actually spent on assembling the data. // The latter will be pretty much moot, but we've kept it to have symmetric @@ -1389,20 +1465,40 @@ func (p *BlobPool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTr pendwaitHist.Update(time.Since(pendStart).Nanoseconds()) defer p.lock.RUnlock() - defer func(start time.Time) { - pendtimeHist.Update(time.Since(start).Nanoseconds()) - }(time.Now()) + execStart := time.Now() + defer func() { + pendtimeHist.Update(time.Since(execStart).Nanoseconds()) + }() - pending := make(map[common.Address][]*txpool.LazyTransaction) + pending := make(map[common.Address][]*txpool.LazyTransaction, len(p.index)) for addr, txs := range p.index { - var lazies []*txpool.LazyTransaction + lazies := make([]*txpool.LazyTransaction, 0, len(txs)) for _, tx := range txs { + // If transaction filtering was requested, discard badly priced ones + if filter.MinTip != nil && filter.BaseFee != nil { + if tx.execFeeCap.Lt(filter.BaseFee) { + break // basefee too low, cannot be included, discard rest of txs from the account + } + tip := new(uint256.Int).Sub(tx.execFeeCap, filter.BaseFee) + if tip.Gt(tx.execTipCap) { + tip = tx.execTipCap + } + if tip.Lt(filter.MinTip) { + break // allowed or remaining tip too low, cannot be included, discard rest of txs from the account + } + } + if filter.BlobFee != nil { + if tx.blobFeeCap.Lt(filter.BlobFee) { + break // blobfee too low, cannot be included, discard rest of txs from the account + } + } + // Transaction was accepted according to the filter, append to the pending list lazies = append(lazies, &txpool.LazyTransaction{ Pool: p, Hash: tx.hash, - Time: time.Now(), // TODO(karalabe): Maybe save these and use that? - GasFeeCap: tx.execFeeCap.ToBig(), - GasTipCap: tx.execTipCap.ToBig(), + Time: execStart, // TODO(karalabe): Maybe save these and use that? + GasFeeCap: tx.execFeeCap, + GasTipCap: tx.execTipCap, Gas: tx.execGas, BlobGas: tx.blobGas, }) @@ -1462,7 +1558,7 @@ func (p *BlobPool) updateStorageMetrics() { } // updateLimboMetrics retrieves a bunch of stats from the limbo store and pushes -// // them out as metrics. +// them out as metrics. func (p *BlobPool) updateLimboMetrics() { stats := p.limbo.store.Infos() diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index c340a8c7ba..85e13980be 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -48,7 +49,7 @@ import ( ) var ( - emptyBlob = kzg4844.Blob{} + emptyBlob = new(kzg4844.Blob) emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) emptyBlobVHash = kzg4844.CalcBlobHashV1(sha256.New(), &emptyBlobCommit) @@ -185,7 +186,7 @@ func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64, return types.MustSignNewTx(key, types.LatestSigner(testChainConfig), blobtx) } -// makeUnsignedTx is a utility method to construct a random blob tranasaction +// makeUnsignedTx is a utility method to construct a random blob transaction // without signing it. func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64) *types.BlobTx { return &types.BlobTx{ @@ -198,7 +199,7 @@ func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap BlobHashes: []common.Hash{emptyBlobVHash}, Value: uint256.NewInt(100), Sidecar: &types.BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: []kzg4844.Blob{*emptyBlob}, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, @@ -305,7 +306,16 @@ func verifyPoolInternals(t *testing.T, pool *BlobPool) { // - 1. A transaction that cannot be decoded must be dropped // - 2. A transaction that cannot be recovered (bad signature) must be dropped // - 3. All transactions after a nonce gap must be dropped -// - 4. All transactions after an underpriced one (including it) must be dropped +// - 4. All transactions after an already included nonce must be dropped +// - 5. All transactions after an underpriced one (including it) must be dropped +// - 6. All transactions after an overdrafting sequence must be dropped +// - 7. All transactions exceeding the per-account limit must be dropped +// +// Furthermore, some strange corner-cases can also occur after a crash, as Billy's +// simplicity also allows it to resurrect past deleted entities: +// +// - 8. Fully duplicate transactions (matching hash) must be dropped +// - 9. Duplicate nonces from the same account must be dropped func TestOpenDrops(t *testing.T) { log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) @@ -338,7 +348,7 @@ func TestOpenDrops(t *testing.T) { badsig, _ := store.Put(blob) // Insert a sequence of transactions with a nonce gap in between to verify - // that anything gapped will get evicted (case 3) + // that anything gapped will get evicted (case 3). var ( gapper, _ = crypto.GenerateKey() @@ -357,7 +367,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions with a gapped starting nonce to verify - // that the entire set will get dropped. + // that the entire set will get dropped (case 3). var ( dangler, _ = crypto.GenerateKey() dangling = make(map[uint64]struct{}) @@ -370,7 +380,7 @@ func TestOpenDrops(t *testing.T) { dangling[id] = struct{}{} } // Insert a sequence of transactions with already passed nonces to veirfy - // that the entire set will get dropped. + // that the entire set will get dropped (case 4). var ( filler, _ = crypto.GenerateKey() filled = make(map[uint64]struct{}) @@ -382,8 +392,8 @@ func TestOpenDrops(t *testing.T) { id, _ := store.Put(blob) filled[id] = struct{}{} } - // Insert a sequence of transactions with partially passed nonces to veirfy - // that the included part of the set will get dropped + // Insert a sequence of transactions with partially passed nonces to verify + // that the included part of the set will get dropped (case 4). var ( overlapper, _ = crypto.GenerateKey() overlapped = make(map[uint64]struct{}) @@ -400,7 +410,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions with an underpriced first to verify that - // the entire set will get dropped (case 4). + // the entire set will get dropped (case 5). var ( underpayer, _ = crypto.GenerateKey() underpaid = make(map[uint64]struct{}) @@ -419,7 +429,7 @@ func TestOpenDrops(t *testing.T) { } // Insert a sequence of transactions with an underpriced in between to verify - // that it and anything newly gapped will get evicted (case 4). + // that it and anything newly gapped will get evicted (case 5). var ( outpricer, _ = crypto.GenerateKey() outpriced = make(map[uint64]struct{}) @@ -441,7 +451,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions fully overdrafted to verify that the - // entire set will get invalidated. + // entire set will get invalidated (case 6). var ( exceeder, _ = crypto.GenerateKey() exceeded = make(map[uint64]struct{}) @@ -459,7 +469,7 @@ func TestOpenDrops(t *testing.T) { exceeded[id] = struct{}{} } // Insert a sequence of transactions partially overdrafted to verify that part - // of the set will get invalidated. + // of the set will get invalidated (case 6). var ( overdrafter, _ = crypto.GenerateKey() overdrafted = make(map[uint64]struct{}) @@ -481,7 +491,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions overflowing the account cap to verify - // that part of the set will get invalidated. + // that part of the set will get invalidated (case 7). var ( overcapper, _ = crypto.GenerateKey() overcapped = make(map[uint64]struct{}) @@ -496,21 +506,59 @@ func TestOpenDrops(t *testing.T) { overcapped[id] = struct{}{} } } + // Insert a batch of duplicated transactions to verify that only one of each + // version will remain (case 8). + var ( + duplicater, _ = crypto.GenerateKey() + duplicated = make(map[uint64]struct{}) + ) + for _, nonce := range []uint64{0, 1, 2} { + blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, 1, 1, duplicater)) + + for i := 0; i < int(nonce)+1; i++ { + id, _ := store.Put(blob) + if i == 0 { + valids[id] = struct{}{} + } else { + duplicated[id] = struct{}{} + } + } + } + // Insert a batch of duplicated nonces to verify that only one of each will + // remain (case 9). + var ( + repeater, _ = crypto.GenerateKey() + repeated = make(map[uint64]struct{}) + ) + for _, nonce := range []uint64{0, 1, 2} { + for i := 0; i < int(nonce)+1; i++ { + blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater)) + + id, _ := store.Put(blob) + if i == 0 { + valids[id] = struct{}{} + } else { + repeated[id] = struct{}{} + } + } + } store.Close() // Create a blob pool out of the pre-seeded data statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(crypto.PubkeyToAddress(gapper.PublicKey), uint256.NewInt(1000000), state.BalanceChangeUnspecified) - statedb.AddBalance(crypto.PubkeyToAddress(dangler.PublicKey), uint256.NewInt(1000000), state.BalanceChangeUnspecified) - statedb.AddBalance(crypto.PubkeyToAddress(filler.PublicKey), uint256.NewInt(1000000), state.BalanceChangeUnspecified) + statedb.AddBalance(crypto.PubkeyToAddress(gapper.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(crypto.PubkeyToAddress(dangler.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(crypto.PubkeyToAddress(filler.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) statedb.SetNonce(crypto.PubkeyToAddress(filler.PublicKey), 3) - statedb.AddBalance(crypto.PubkeyToAddress(overlapper.PublicKey), uint256.NewInt(1000000), state.BalanceChangeUnspecified) + statedb.AddBalance(crypto.PubkeyToAddress(overlapper.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) statedb.SetNonce(crypto.PubkeyToAddress(overlapper.PublicKey), 2) - statedb.AddBalance(crypto.PubkeyToAddress(underpayer.PublicKey), uint256.NewInt(1000000), state.BalanceChangeUnspecified) - statedb.AddBalance(crypto.PubkeyToAddress(outpricer.PublicKey), uint256.NewInt(1000000), state.BalanceChangeUnspecified) - statedb.AddBalance(crypto.PubkeyToAddress(exceeder.PublicKey), uint256.NewInt(1000000), state.BalanceChangeUnspecified) - statedb.AddBalance(crypto.PubkeyToAddress(overdrafter.PublicKey), uint256.NewInt(1000000), state.BalanceChangeUnspecified) - statedb.AddBalance(crypto.PubkeyToAddress(overcapper.PublicKey), uint256.NewInt(10000000), state.BalanceChangeUnspecified) + statedb.AddBalance(crypto.PubkeyToAddress(underpayer.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(crypto.PubkeyToAddress(outpricer.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(crypto.PubkeyToAddress(exceeder.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(crypto.PubkeyToAddress(overdrafter.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(crypto.PubkeyToAddress(overcapper.PublicKey), uint256.NewInt(10000000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(crypto.PubkeyToAddress(duplicater.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(crypto.PubkeyToAddress(repeater.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) statedb.Commit(0, true) chain := &testBlockChain{ @@ -520,7 +568,7 @@ func TestOpenDrops(t *testing.T) { statedb: statedb, } pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(big.NewInt(1), chain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } defer pool.Close() @@ -554,6 +602,10 @@ func TestOpenDrops(t *testing.T) { t.Errorf("partially overdrafted transaction remained in storage: %d", tx.id) } else if _, ok := overcapped[tx.id]; ok { t.Errorf("overcapped transaction remained in storage: %d", tx.id) + } else if _, ok := duplicated[tx.id]; ok { + t.Errorf("duplicated transaction remained in storage: %d", tx.id) + } else if _, ok := repeated[tx.id]; ok { + t.Errorf("repeated nonce transaction remained in storage: %d", tx.id) } else { alive[tx.id] = struct{}{} } @@ -584,7 +636,7 @@ func TestOpenDrops(t *testing.T) { // Tests that transactions loaded from disk are indexed correctly. // -// - 1. Transactions must be groupped by sender, sorted by nonce +// - 1. Transactions must be grouped by sender, sorted by nonce // - 2. Eviction thresholds are calculated correctly for the sequences // - 3. Balance usage of an account is totals across all transactions func TestOpenIndex(t *testing.T) { @@ -598,7 +650,7 @@ func TestOpenIndex(t *testing.T) { store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(), nil) // Insert a sequence of transactions with varying price points to check that - // the cumulative minimumw will be maintained. + // the cumulative minimum will be maintained. var ( key, _ = crypto.GenerateKey() addr = crypto.PubkeyToAddress(key.PublicKey) @@ -625,7 +677,7 @@ func TestOpenIndex(t *testing.T) { // Create a blob pool out of the pre-seeded data statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(addr, uint256.NewInt(1_000_000_000), state.BalanceChangeUnspecified) + statedb.AddBalance(addr, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) statedb.Commit(0, true) chain := &testBlockChain{ @@ -635,7 +687,7 @@ func TestOpenIndex(t *testing.T) { statedb: statedb, } pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(big.NewInt(1), chain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } defer pool.Close() @@ -725,9 +777,9 @@ func TestOpenHeap(t *testing.T) { // Create a blob pool out of the pre-seeded data statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), state.BalanceChangeUnspecified) - statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000), state.BalanceChangeUnspecified) - statedb.AddBalance(addr3, uint256.NewInt(1_000_000_000), state.BalanceChangeUnspecified) + statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(addr3, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) statedb.Commit(0, true) chain := &testBlockChain{ @@ -737,7 +789,7 @@ func TestOpenHeap(t *testing.T) { statedb: statedb, } pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(big.NewInt(1), chain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } defer pool.Close() @@ -805,9 +857,9 @@ func TestOpenCap(t *testing.T) { for _, datacap := range []uint64{2 * (txAvgSize + blobSize), 100 * (txAvgSize + blobSize)} { // Create a blob pool out of the pre-seeded data, but cap it to 2 blob transaction statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), state.BalanceChangeUnspecified) - statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000), state.BalanceChangeUnspecified) - statedb.AddBalance(addr3, uint256.NewInt(1_000_000_000), state.BalanceChangeUnspecified) + statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(addr3, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) statedb.Commit(0, true) chain := &testBlockChain{ @@ -817,7 +869,7 @@ func TestOpenCap(t *testing.T) { statedb: statedb, } pool := New(Config{Datadir: storage, Datacap: datacap}, chain) - if err := pool.Init(big.NewInt(1), chain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } // Verify that enough transactions have been dropped to get the pool's size @@ -933,9 +985,14 @@ func TestAdd(t *testing.T) { }, }, adds: []addtx{ - { // New account, 1 tx pending: reject replacement nonce 0 (ignore price for now) + { // New account, 1 tx pending: reject duplicate nonce 0 from: "alice", tx: makeUnsignedTx(0, 1, 1, 1), + err: txpool.ErrAlreadyKnown, + }, + { // New account, 1 tx pending: reject replacement nonce 0 (ignore price for now) + from: "alice", + tx: makeUnsignedTx(0, 1, 1, 2), err: txpool.ErrReplaceUnderpriced, }, { // New account, 1 tx pending: accept nonce 1 @@ -958,10 +1015,10 @@ func TestAdd(t *testing.T) { tx: makeUnsignedTx(3, 1, 1, 1), err: nil, }, - { // Old account, 1 tx in chain, 1 tx pending: reject replacement nonce 1 (ignore price for now) + { // Old account, 1 tx in chain, 1 tx pending: reject duplicate nonce 1 from: "bob", tx: makeUnsignedTx(1, 1, 1, 1), - err: txpool.ErrReplaceUnderpriced, + err: txpool.ErrAlreadyKnown, }, { // Old account, 1 tx in chain, 1 tx pending: accept nonce 2 (ignore price for now) from: "bob", @@ -1177,6 +1234,24 @@ func TestAdd(t *testing.T) { }, }, }, + // Blob transactions that don't meet the min blob gas price should be rejected + { + seeds: map[string]seed{ + "alice": {balance: 10000000}, + }, + adds: []addtx{ + { // New account, no previous txs, nonce 0, but blob fee cap too low + from: "alice", + tx: makeUnsignedTx(0, 1, 1, 0), + err: txpool.ErrUnderpriced, + }, + { // Same as above but blob fee cap equals minimum, should be accepted + from: "alice", + tx: makeUnsignedTx(0, 1, 1, params.BlobTxMinBlobGasprice), + err: nil, + }, + }, + }, } for i, tt := range tests { // Create a temporary folder for the persistent backend @@ -1197,8 +1272,8 @@ func TestAdd(t *testing.T) { keys[acc], _ = crypto.GenerateKey() addrs[acc] = crypto.PubkeyToAddress(keys[acc].PublicKey) - // Seed the state database with this acocunt - statedb.AddBalance(addrs[acc], new(uint256.Int).SetUint64(seed.balance), state.BalanceChangeUnspecified) + // Seed the state database with this account + statedb.AddBalance(addrs[acc], new(uint256.Int).SetUint64(seed.balance), tracing.BalanceChangeUnspecified) statedb.SetNonce(addrs[acc], seed.nonce) // Sign the seed transactions and store them in the data store @@ -1219,7 +1294,7 @@ func TestAdd(t *testing.T) { statedb: statedb, } pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(big.NewInt(1), chain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { t.Fatalf("test %d: failed to create blob pool: %v", i, err) } verifyPoolInternals(t, pool) @@ -1237,3 +1312,65 @@ func TestAdd(t *testing.T) { pool.Close() } } + +// Benchmarks the time it takes to assemble the lazy pending transaction list +// from the pool contents. +func BenchmarkPoolPending100Mb(b *testing.B) { benchmarkPoolPending(b, 100_000_000) } +func BenchmarkPoolPending1GB(b *testing.B) { benchmarkPoolPending(b, 1_000_000_000) } +func BenchmarkPoolPending10GB(b *testing.B) { benchmarkPoolPending(b, 10_000_000_000) } + +func benchmarkPoolPending(b *testing.B, datacap uint64) { + // Calculate the maximum number of transaction that would fit into the pool + // and generate a set of random accounts to seed them with. + capacity := datacap / params.BlobTxBlobGasPerBlob + + var ( + basefee = uint64(1050) + blobfee = uint64(105) + signer = types.LatestSigner(testChainConfig) + statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) + chain = &testBlockChain{ + config: testChainConfig, + basefee: uint256.NewInt(basefee), + blobfee: uint256.NewInt(blobfee), + statedb: statedb, + } + pool = New(Config{Datadir: ""}, chain) + ) + + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + b.Fatalf("failed to create blob pool: %v", err) + } + // Fill the pool up with one random transaction from each account with the + // same price and everything to maximize the worst case scenario + for i := 0; i < int(capacity); i++ { + blobtx := makeUnsignedTx(0, 10, basefee+10, blobfee) + blobtx.R = uint256.NewInt(1) + blobtx.S = uint256.NewInt(uint64(100 + i)) + blobtx.V = uint256.NewInt(0) + tx := types.NewTx(blobtx) + addr, err := types.Sender(signer, tx) + if err != nil { + b.Fatal(err) + } + statedb.AddBalance(addr, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) + pool.add(tx) + } + statedb.Commit(0, true) + defer pool.Close() + + // Benchmark assembling the pending + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + p := pool.Pending(txpool.PendingFilter{ + MinTip: uint256.NewInt(1), + BaseFee: chain.basefee, + BlobFee: chain.blobfee, + }) + if len(p) != int(capacity) { + b.Fatalf("have %d want %d", len(p), capacity) + } + } +} diff --git a/core/txpool/blobpool/config.go b/core/txpool/blobpool/config.go index 99a2002a30..1d180739cd 100644 --- a/core/txpool/blobpool/config.go +++ b/core/txpool/blobpool/config.go @@ -30,8 +30,8 @@ type Config struct { // DefaultConfig contains the default configurations for the transaction pool. var DefaultConfig = Config{ Datadir: "blobpool", - Datacap: 10 * 1024 * 1024 * 1024, - PriceBump: 100, // either have patience or be aggressive, no mushy ground + Datacap: 10 * 1024 * 1024 * 1024 / 4, // TODO(karalabe): /4 handicap for rollout, gradually bump back up to 10GB + PriceBump: 100, // either have patience or be aggressive, no mushy ground } // sanitize checks the provided user configurations and changes anything that's diff --git a/core/txpool/blobpool/evictheap.go b/core/txpool/blobpool/evictheap.go index df594099f7..bc4543a352 100644 --- a/core/txpool/blobpool/evictheap.go +++ b/core/txpool/blobpool/evictheap.go @@ -30,7 +30,7 @@ import ( // transaction from each account to determine which account to evict from. // // The heap internally tracks a slice of cheapest transactions from each account -// and a mapping from addresses to indices for direct removals/udates. +// and a mapping from addresses to indices for direct removals/updates. // // The goal of the heap is to decide which account has the worst bottleneck to // evict transactions from. diff --git a/core/txpool/blobpool/limbo.go b/core/txpool/blobpool/limbo.go index d1fe9c7394..ec754f6894 100644 --- a/core/txpool/blobpool/limbo.go +++ b/core/txpool/blobpool/limbo.go @@ -53,7 +53,7 @@ func newLimbo(datadir string) (*limbo, error) { index: make(map[common.Hash]uint64), groups: make(map[uint64]map[uint64]common.Hash), } - // Index all limboed blobs on disk and delete anything inprocessable + // Index all limboed blobs on disk and delete anything unprocessable var fails []uint64 index := func(id uint64, size uint32, data []byte) { if l.parseBlob(id, data) != nil { @@ -89,7 +89,7 @@ func (l *limbo) parseBlob(id uint64, data []byte) error { item := new(limboBlob) if err := rlp.DecodeBytes(data, item); err != nil { // This path is impossible unless the disk data representation changes - // across restarts. For that ever unprobable case, recover gracefully + // across restarts. For that ever improbable case, recover gracefully // by ignoring this data entry. log.Error("Failed to decode blob limbo entry", "id", id, "err", err) return err @@ -172,7 +172,7 @@ func (l *limbo) pull(tx common.Hash) (*types.Transaction, error) { // update changes the block number under which a blob transaction is tracked. This // method should be used when a reorg changes a transaction's inclusion block. // -// The method may log errors for various unexpcted scenarios but will not return +// The method may log errors for various unexpected scenarios but will not return // any of it since there's no clear error case. Some errors may be due to coding // issues, others caused by signers mining MEV stuff or swapping transactions. In // all cases, the pool needs to continue operating. diff --git a/core/txpool/blobpool/metrics.go b/core/txpool/blobpool/metrics.go index 587804cc61..52419ade09 100644 --- a/core/txpool/blobpool/metrics.go +++ b/core/txpool/blobpool/metrics.go @@ -65,8 +65,8 @@ var ( pooltipGauge = metrics.NewRegisteredGauge("blobpool/pooltip", nil) // addwait/time, resetwait/time and getwait/time track the rough health of - // the pool and whether or not it's capable of keeping up with the load from - // the network. + // the pool and whether it's capable of keeping up with the load from the + // network. addwaitHist = metrics.NewRegisteredHistogram("blobpool/addwait", nil, metrics.NewExpDecaySample(1028, 0.015)) addtimeHist = metrics.NewRegisteredHistogram("blobpool/addtime", nil, metrics.NewExpDecaySample(1028, 0.015)) getwaitHist = metrics.NewRegisteredHistogram("blobpool/getwait", nil, metrics.NewExpDecaySample(1028, 0.015)) @@ -75,4 +75,31 @@ var ( pendtimeHist = metrics.NewRegisteredHistogram("blobpool/pendtime", nil, metrics.NewExpDecaySample(1028, 0.015)) resetwaitHist = metrics.NewRegisteredHistogram("blobpool/resetwait", nil, metrics.NewExpDecaySample(1028, 0.015)) resettimeHist = metrics.NewRegisteredHistogram("blobpool/resettime", nil, metrics.NewExpDecaySample(1028, 0.015)) + + // The below metrics track various cases where transactions are dropped out + // of the pool. Most are exceptional, some are chain progression and some + // threshold cappings. + dropInvalidMeter = metrics.NewRegisteredMeter("blobpool/drop/invalid", nil) // Invalid transaction, consensus change or bugfix, neutral-ish + dropDanglingMeter = metrics.NewRegisteredMeter("blobpool/drop/dangling", nil) // First nonce gapped, bad + dropFilledMeter = metrics.NewRegisteredMeter("blobpool/drop/filled", nil) // State full-overlap, chain progress, ok + dropOverlappedMeter = metrics.NewRegisteredMeter("blobpool/drop/overlapped", nil) // State partial-overlap, chain progress, ok + dropRepeatedMeter = metrics.NewRegisteredMeter("blobpool/drop/repeated", nil) // Repeated nonce, bad + dropGappedMeter = metrics.NewRegisteredMeter("blobpool/drop/gapped", nil) // Non-first nonce gapped, bad + dropOverdraftedMeter = metrics.NewRegisteredMeter("blobpool/drop/overdrafted", nil) // Balance exceeded, bad + dropOvercappedMeter = metrics.NewRegisteredMeter("blobpool/drop/overcapped", nil) // Per-account cap exceeded, bad + dropOverflownMeter = metrics.NewRegisteredMeter("blobpool/drop/overflown", nil) // Global disk cap exceeded, neutral-ish + dropUnderpricedMeter = metrics.NewRegisteredMeter("blobpool/drop/underpriced", nil) // Gas tip changed, neutral + dropReplacedMeter = metrics.NewRegisteredMeter("blobpool/drop/replaced", nil) // Transaction replaced, neutral + + // The below metrics track various outcomes of transactions being added to + // the pool. + addInvalidMeter = metrics.NewRegisteredMeter("blobpool/add/invalid", nil) // Invalid transaction, reject, neutral + addUnderpricedMeter = metrics.NewRegisteredMeter("blobpool/add/underpriced", nil) // Gas tip too low, neutral + addStaleMeter = metrics.NewRegisteredMeter("blobpool/add/stale", nil) // Nonce already filled, reject, bad-ish + addGappedMeter = metrics.NewRegisteredMeter("blobpool/add/gapped", nil) // Nonce gapped, reject, bad-ish + addOverdraftedMeter = metrics.NewRegisteredMeter("blobpool/add/overdrafted", nil) // Balance exceeded, reject, neutral + addOvercappedMeter = metrics.NewRegisteredMeter("blobpool/add/overcapped", nil) // Per-account cap exceeded, reject, neutral + addNoreplaceMeter = metrics.NewRegisteredMeter("blobpool/add/noreplace", nil) // Replacement fees or tips too low, neutral + addNonExclusiveMeter = metrics.NewRegisteredMeter("blobpool/add/nonexclusive", nil) // Plain transaction from same account exists, reject, neutral + addValidMeter = metrics.NewRegisteredMeter("blobpool/add/valid", nil) // Valid transaction, add, neutral ) diff --git a/core/txpool/blobpool/priority_test.go b/core/txpool/blobpool/priority_test.go index 4aad919925..cf0e0454a0 100644 --- a/core/txpool/blobpool/priority_test.go +++ b/core/txpool/blobpool/priority_test.go @@ -64,7 +64,7 @@ func BenchmarkDynamicFeeJumpCalculation(b *testing.B) { // Benchmarks how many priority recalculations can be done. func BenchmarkPriorityCalculation(b *testing.B) { // The basefee and blob fee is constant for all transactions across a block, - // so we can assume theit absolute jump counts can be pre-computed. + // so we can assume their absolute jump counts can be pre-computed. basefee := uint256.NewInt(17_200_000_000) // 17.2 Gwei is the 22.03.2023 zero-emission basefee, random number blobfee := uint256.NewInt(123_456_789_000) // Completely random, no idea what this will be diff --git a/core/txpool/errors.go b/core/txpool/errors.go index 61daa999ff..3a6a913976 100644 --- a/core/txpool/errors.go +++ b/core/txpool/errors.go @@ -54,4 +54,10 @@ var ( // ErrFutureReplacePending is returned if a future transaction replaces a pending // one. Future transactions should only be able to replace other future transactions. ErrFutureReplacePending = errors.New("future transaction tries to replace pending") + + // ErrAlreadyReserved is returned if the sender address has a pending transaction + // in a different subpool. For example, this error is returned in response to any + // input transaction of non-blob type when a blob transaction from this sender + // remains pending (and vice-versa). + ErrAlreadyReserved = errors.New("address already reserved") ) diff --git a/core/txpool/legacypool/journal.go b/core/txpool/legacypool/journal.go index f04ab8fc14..899ed00bcc 100644 --- a/core/txpool/legacypool/journal.go +++ b/core/txpool/legacypool/journal.go @@ -164,7 +164,12 @@ func (journal *journal) rotate(all map[common.Address]types.Transactions) error return err } journal.writer = sink - log.Info("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all)) + + logger := log.Info + if len(all) == 0 { + logger = log.Debug + } + logger("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all)) return nil } diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 624dafc60d..4e1d26acf4 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) const ( @@ -202,7 +203,7 @@ type LegacyPool struct { config Config chainconfig *params.ChainConfig chain BlockChain - gasTip atomic.Pointer[big.Int] + gasTip atomic.Pointer[uint256.Int] txFeed event.Feed signer types.Signer mu sync.RWMutex @@ -287,15 +288,15 @@ func (pool *LegacyPool) Filter(tx *types.Transaction) bool { // head to allow balance / nonce checks. The transaction journal will be loaded // from disk and filtered based on the provided starting settings. The internal // goroutines will be spun up and the pool deemed operational afterwards. -func (pool *LegacyPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.AddressReserver) error { +func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserve txpool.AddressReserver) error { // Set the address reserver to request exclusive access to pooled accounts pool.reserve = reserve // Set the basic pool parameters - pool.gasTip.Store(gasTip) + pool.gasTip.Store(uint256.NewInt(gasTip)) // Initialize the state with head block, or fallback to empty one in - // case the head state is not available(might occur when node is not + // case the head state is not available (might occur when node is not // fully synced). statedb, err := pool.chain.StateAt(head.Root) if err != nil { @@ -433,11 +434,13 @@ func (pool *LegacyPool) SetGasTip(tip *big.Int) { pool.mu.Lock() defer pool.mu.Unlock() - old := pool.gasTip.Load() - pool.gasTip.Store(new(big.Int).Set(tip)) - + var ( + newTip = uint256.MustFromBig(tip) + old = pool.gasTip.Load() + ) + pool.gasTip.Store(newTip) // If the min miner fee increased, remove transactions below the new threshold - if tip.Cmp(old) > 0 { + if newTip.Cmp(old) > 0 { // pool.priced is sorted by GasFeeCap, so we have to iterate through pool.all instead drop := pool.all.RemotesBelowTip(tip) for _, tx := range drop { @@ -445,7 +448,7 @@ func (pool *LegacyPool) SetGasTip(tip *big.Int) { } pool.priced.Removed(len(drop)) } - log.Info("Legacy pool tip threshold updated", "tip", tip) + log.Info("Legacy pool tip threshold updated", "tip", newTip) } // Nonce returns the next nonce of an account, with all transactions executable @@ -515,24 +518,38 @@ func (pool *LegacyPool) ContentFrom(addr common.Address) ([]*types.Transaction, } // Pending retrieves all currently processable transactions, grouped by origin -// account and sorted by nonce. The returned transaction set is a copy and can be -// freely modified by calling code. +// account and sorted by nonce. // -// The enforceTips parameter can be used to do an extra filtering on the pending -// transactions and only return those whose **effective** tip is large enough in -// the next pending execution environment. -func (pool *LegacyPool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTransaction { +// The transactions can also be pre-filtered by the dynamic fee components to +// reduce allocations and load on downstream subsystems. +func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { + // If only blob transactions are requested, this pool is unsuitable as it + // contains none, don't even bother. + if filter.OnlyBlobTxs { + return nil + } pool.mu.Lock() defer pool.mu.Unlock() + // Convert the new uint256.Int types to the old big.Int ones used by the legacy pool + var ( + minTipBig *big.Int + baseFeeBig *big.Int + ) + if filter.MinTip != nil { + minTipBig = filter.MinTip.ToBig() + } + if filter.BaseFee != nil { + baseFeeBig = filter.BaseFee.ToBig() + } pending := make(map[common.Address][]*txpool.LazyTransaction, len(pool.pending)) for addr, list := range pool.pending { txs := list.Flatten() // If the miner requests tip enforcement, cap the lists now - if enforceTips && !pool.locals.contains(addr) { + if minTipBig != nil && !pool.locals.contains(addr) { for i, tx := range txs { - if tx.EffectiveGasTipIntCmp(pool.gasTip.Load(), pool.priced.urgent.baseFee) < 0 { + if tx.EffectiveGasTipIntCmp(minTipBig, baseFeeBig) < 0 { txs = txs[:i] break } @@ -546,8 +563,8 @@ func (pool *LegacyPool) Pending(enforceTips bool) map[common.Address][]*txpool.L Hash: txs[i].Hash(), Tx: txs[i], Time: txs[i].Time(), - GasFeeCap: txs[i].GasFeeCap(), - GasTipCap: txs[i].GasTipCap(), + GasFeeCap: uint256.MustFromBig(txs[i].GasFeeCap()), + GasTipCap: uint256.MustFromBig(txs[i].GasTipCap()), Gas: txs[i].Gas(), BlobGas: txs[i].BlobGas(), } @@ -594,7 +611,7 @@ func (pool *LegacyPool) validateTxBasics(tx *types.Transaction, local bool) erro 1< threshold; size-- { drops = append(drops, m.items[(*m.index)[size-1]]) delete(m.items, (*m.index)[size-1]) } *m.index = (*m.index)[:threshold] - heap.Init(m.index) + // The sorted m.index slice is still a valid heap, so there is no need to + // reheap after deleting tail items. // If we had a cache, shift the back m.cacheMu.Lock() @@ -271,19 +273,19 @@ type list struct { strict bool // Whether nonces are strictly continuous or not txs *sortedMap // Heap indexed sorted hash map of the transactions - costcap *big.Int // Price of the highest costing transaction (reset only if exceeds balance) - gascap uint64 // Gas limit of the highest spending transaction (reset only if exceeds block limit) - totalcost *big.Int // Total cost of all transactions in the list + costcap *uint256.Int // Price of the highest costing transaction (reset only if exceeds balance) + gascap uint64 // Gas limit of the highest spending transaction (reset only if exceeds block limit) + totalcost *uint256.Int // Total cost of all transactions in the list } -// newList create a new transaction list for maintaining nonce-indexable fast, +// newList creates a new transaction list for maintaining nonce-indexable fast, // gapped, sortable transaction lists. func newList(strict bool) *list { return &list{ strict: strict, txs: newSortedMap(), - costcap: new(big.Int), - totalcost: new(big.Int), + costcap: new(uint256.Int), + totalcost: new(uint256.Int), } } @@ -325,10 +327,15 @@ func (l *list) Add(tx *types.Transaction, priceBump uint64) (bool, *types.Transa l.subTotalCost([]*types.Transaction{old}) } // Add new tx cost to totalcost - l.totalcost.Add(l.totalcost, tx.Cost()) + cost, overflow := uint256.FromBig(tx.Cost()) + if overflow { + return false, nil + } + l.totalcost.Add(l.totalcost, cost) + // Otherwise overwrite the old transaction with the current one l.txs.Put(tx) - if cost := tx.Cost(); l.costcap.Cmp(cost) < 0 { + if l.costcap.Cmp(cost) < 0 { l.costcap = cost } if gas := tx.Gas(); l.gascap < gas { @@ -355,17 +362,17 @@ func (l *list) Forward(threshold uint64) types.Transactions { // a point in calculating all the costs or if the balance covers all. If the threshold // is lower than the costgas cap, the caps will be reset to a new high after removing // the newly invalidated transactions. -func (l *list) Filter(costLimit *big.Int, gasLimit uint64) (types.Transactions, types.Transactions) { +func (l *list) Filter(costLimit *uint256.Int, gasLimit uint64) (types.Transactions, types.Transactions) { // If all transactions are below the threshold, short circuit if l.costcap.Cmp(costLimit) <= 0 && l.gascap <= gasLimit { return nil, nil } - l.costcap = new(big.Int).Set(costLimit) // Lower the caps to the thresholds + l.costcap = new(uint256.Int).Set(costLimit) // Lower the caps to the thresholds l.gascap = gasLimit // Filter out all the transactions above the account's funds removed := l.txs.Filter(func(tx *types.Transaction) bool { - return tx.Gas() > gasLimit || tx.Cost().Cmp(costLimit) > 0 + return tx.Gas() > gasLimit || tx.Cost().Cmp(costLimit.ToBig()) > 0 }) if len(removed) == 0 { @@ -456,7 +463,10 @@ func (l *list) LastElement() *types.Transaction { // total cost of all transactions. func (l *list) subTotalCost(txs []*types.Transaction) { for _, tx := range txs { - l.totalcost.Sub(l.totalcost, tx.Cost()) + _, underflow := l.totalcost.SubOverflow(l.totalcost, uint256.MustFromBig(tx.Cost())) + if underflow { + panic("totalcost underflow") + } } } diff --git a/core/txpool/legacypool/list_test.go b/core/txpool/legacypool/list_test.go index b5cd34b23b..8587c66f7d 100644 --- a/core/txpool/legacypool/list_test.go +++ b/core/txpool/legacypool/list_test.go @@ -21,8 +21,10 @@ import ( "math/rand" "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/holiman/uint256" ) // Tests that transactions can be added to strict lists and list contents and @@ -51,6 +53,21 @@ func TestStrictListAdd(t *testing.T) { } } +// TestListAddVeryExpensive tests adding txs which exceed 256 bits in cost. It is +// expected that the list does not panic. +func TestListAddVeryExpensive(t *testing.T) { + key, _ := crypto.GenerateKey() + list := newList(true) + for i := 0; i < 3; i++ { + value := big.NewInt(100) + gasprice, _ := new(big.Int).SetString("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 0) + gaslimit := uint64(i) + tx, _ := types.SignTx(types.NewTransaction(uint64(i), common.Address{}, value, gaslimit, gasprice, nil), types.HomesteadSigner{}, key) + t.Logf("cost: %x bitlen: %d\n", tx.Cost(), tx.Cost().BitLen()) + list.Add(tx, DefaultConfig.PriceBump) + } +} + func BenchmarkListAdd(b *testing.B) { // Generate a list of transactions to insert key, _ := crypto.GenerateKey() @@ -60,7 +77,7 @@ func BenchmarkListAdd(b *testing.B) { txs[i] = transaction(uint64(i), 0, key) } // Insert the transactions in a random order - priceLimit := big.NewInt(int64(DefaultConfig.PriceLimit)) + priceLimit := uint256.NewInt(DefaultConfig.PriceLimit) b.ResetTimer() for i := 0; i < b.N; i++ { list := newList(true) @@ -70,3 +87,25 @@ func BenchmarkListAdd(b *testing.B) { } } } + +func BenchmarkListCapOneTx(b *testing.B) { + // Generate a list of transactions to insert + key, _ := crypto.GenerateKey() + + txs := make(types.Transactions, 32) + for i := 0; i < len(txs); i++ { + txs[i] = transaction(uint64(i), 0, key) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + list := newList(true) + // Insert the transactions in a random order + for _, v := range rand.Perm(len(txs)) { + list.Add(txs[v], DefaultConfig.PriceBump) + } + b.StartTimer() + list.Cap(list.Len() - 1) + b.StopTimer() + } +} diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index de05b38d43..9881ed1b8f 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" + "github.com/holiman/uint256" ) // LazyTransaction contains a small subset of the transaction properties that is @@ -34,9 +35,9 @@ type LazyTransaction struct { Hash common.Hash // Transaction hash to pull up if needed Tx *types.Transaction // Transaction if already resolved - Time time.Time // Time when the transaction was first seen - GasFeeCap *big.Int // Maximum fee per gas the transaction may consume - GasTipCap *big.Int // Maximum miner tip per gas the transaction can pay + Time time.Time // Time when the transaction was first seen + GasFeeCap *uint256.Int // Maximum fee per gas the transaction may consume + GasTipCap *uint256.Int // Maximum miner tip per gas the transaction can pay Gas uint64 // Amount of gas required by the transaction BlobGas uint64 // Amount of blob gas required by the transaction @@ -44,11 +45,17 @@ type LazyTransaction struct { // Resolve retrieves the full transaction belonging to a lazy handle if it is still // maintained by the transaction pool. +// +// Note, the method will *not* cache the retrieved transaction if the original +// pool has not cached it. The idea being, that if the tx was too big to insert +// originally, silently saving it will cause more trouble down the line (and +// indeed seems to have caused a memory bloat in the original implementation +// which did just that). func (ltx *LazyTransaction) Resolve() *types.Transaction { - if ltx.Tx == nil { - ltx.Tx = ltx.Pool.Get(ltx.Hash) + if ltx.Tx != nil { + return ltx.Tx } - return ltx.Tx + return ltx.Pool.Get(ltx.Hash) } // LazyResolver is a minimal interface needed for a transaction pool to satisfy @@ -63,13 +70,28 @@ type LazyResolver interface { // may request (and relinquish) exclusive access to certain addresses. type AddressReserver func(addr common.Address, reserve bool) error +// PendingFilter is a collection of filter rules to allow retrieving a subset +// of transactions for announcement or mining. +// +// Note, the entries here are not arbitrary useful filters, rather each one has +// a very specific call site in mind and each one can be evaluated very cheaply +// by the pool implementations. Only add new ones that satisfy those constraints. +type PendingFilter struct { + MinTip *uint256.Int // Minimum miner tip required to include a transaction + BaseFee *uint256.Int // Minimum 1559 basefee needed to include a transaction + BlobFee *uint256.Int // Minimum 4844 blobfee needed to include a blob transaction + + OnlyPlainTxs bool // Return only plain EVM transactions (peer-join announces, block space filling) + OnlyBlobTxs bool // Return only blob transactions (block blob-space filling) +} + // SubPool represents a specialized transaction pool that lives on its own (e.g. // blob pool). Since independent of how many specialized pools we have, they do // need to be updated in lockstep and assemble into one coherent view for block // production, this interface defines the common methods that allow the primary // transaction pool to manage the subpools. type SubPool interface { - // Filter is a selector used to decide whether a transaction whould be added + // Filter is a selector used to decide whether a transaction would be added // to this particular subpool. Filter(tx *types.Transaction) bool @@ -80,7 +102,7 @@ type SubPool interface { // These should not be passed as a constructor argument - nor should the pools // start by themselves - in order to keep multiple subpools in lockstep with // one another. - Init(gasTip *big.Int, head *types.Header, reserve AddressReserver) error + Init(gasTip uint64, head *types.Header, reserve AddressReserver) error // Close terminates any background processing threads and releases any held // resources. @@ -108,7 +130,10 @@ type SubPool interface { // Pending retrieves all currently processable transactions, grouped by origin // account and sorted by nonce. - Pending(enforceTips bool) map[common.Address][]*LazyTransaction + // + // The transactions can also be pre-filtered by the dynamic fee components to + // reduce allocations and load on downstream subsystems. + Pending(filter PendingFilter) map[common.Address][]*LazyTransaction // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index d03e025a9e..be7435247d 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -79,7 +79,7 @@ type TxPool struct { // New creates a new transaction pool to gather, sort and filter inbound // transactions from the network. -func New(gasTip *big.Int, chain BlockChain, subpools []SubPool) (*TxPool, error) { +func New(gasTip uint64, chain BlockChain, subpools []SubPool) (*TxPool, error) { // Retrieve the current head so that all subpools and this main coordinator // pool will have the same starting state, even if the chain moves forward // during initialization. @@ -122,7 +122,7 @@ func (p *TxPool) reserver(id int, subpool SubPool) AddressReserver { log.Error("pool attempted to reserve already-owned address", "address", addr) return nil // Ignore fault to give the pool a chance to recover while the bug gets fixed } - return errors.New("address already reserved") + return ErrAlreadyReserved } p.reservations[addr] = subpool if metrics.Enabled { @@ -353,10 +353,13 @@ func (p *TxPool) Add(txs []*types.Transaction, local bool, sync bool) []error { // Pending retrieves all currently processable transactions, grouped by origin // account and sorted by nonce. -func (p *TxPool) Pending(enforceTips bool) map[common.Address][]*LazyTransaction { +// +// The transactions can also be pre-filtered by the dynamic fee components to +// reduce allocations and load on downstream subsystems. +func (p *TxPool) Pending(filter PendingFilter) map[common.Address][]*LazyTransaction { txs := make(map[common.Address][]*LazyTransaction) for _, subpool := range p.subpools { - for addr, set := range subpool.Pending(enforceTips) { + for addr, set := range subpool.Pending(filter) { txs[addr] = set } } diff --git a/core/txpool/validation.go b/core/txpool/validation.go index a9bd14020b..d9a85a435d 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -18,6 +18,7 @@ package txpool import ( "crypto/sha256" + "errors" "fmt" "math/big" @@ -30,6 +31,12 @@ import ( "github.com/ethereum/go-ethereum/params" ) +var ( + // blobTxMinBlobGasPrice is the big.Int version of the configured protocol + // parameter to avoid constucting a new big integer for every transaction. + blobTxMinBlobGasPrice = big.NewInt(params.BlobTxMinBlobGasprice) +) + // ValidationOptions define certain differences between transaction validation // across the different pools without having to duplicate those checks. type ValidationOptions struct { @@ -101,28 +108,31 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types return err } if tx.Gas() < intrGas { - return fmt.Errorf("%w: needed %v, allowed %v", core.ErrIntrinsicGas, intrGas, tx.Gas()) + return fmt.Errorf("%w: gas %v, minimum needed %v", core.ErrIntrinsicGas, tx.Gas(), intrGas) } - // Ensure the gasprice is high enough to cover the requirement of the calling - // pool and/or block producer + // Ensure the gasprice is high enough to cover the requirement of the calling pool if tx.GasTipCapIntCmp(opts.MinTip) < 0 { - return fmt.Errorf("%w: tip needed %v, tip permitted %v", ErrUnderpriced, opts.MinTip, tx.GasTipCap()) + return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrUnderpriced, tx.GasTipCap(), opts.MinTip) } - // Ensure blob transactions have valid commitments if tx.Type() == types.BlobTxType { + // Ensure the blob fee cap satisfies the minimum blob gas price + if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 { + return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrUnderpriced, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice) + } sidecar := tx.BlobTxSidecar() if sidecar == nil { - return fmt.Errorf("missing sidecar in blob transaction") + return errors.New("missing sidecar in blob transaction") } // Ensure the number of items in the blob transaction and various side // data match up before doing any expensive validations hashes := tx.BlobHashes() if len(hashes) == 0 { - return fmt.Errorf("blobless blob transaction") + return errors.New("blobless blob transaction") } if len(hashes) > params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob { return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) } + // Ensure commitments, proofs and hashes are valid if err := validateBlobSidecar(hashes, sidecar); err != nil { return err } @@ -152,7 +162,7 @@ func validateBlobSidecar(hashes []common.Hash, sidecar *types.BlobTxSidecar) err // Blob commitments match with the hashes in the transaction, verify the // blobs themselves via KZG for i := range sidecar.Blobs { - if err := kzg4844.VerifyBlobProof(sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil { + if err := kzg4844.VerifyBlobProof(&sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil { return fmt.Errorf("invalid blob %d: %v", i, err) } } diff --git a/core/types/account.go b/core/types/account.go new file mode 100644 index 0000000000..52ce184cda --- /dev/null +++ b/core/types/account.go @@ -0,0 +1,87 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common/math" +) + +//go:generate go run github.com/fjl/gencodec -type Account -field-override accountMarshaling -out gen_account.go + +// Account represents an Ethereum account and its attached data. +// This type is used to specify accounts in the genesis block state, and +// is also useful for JSON encoding/decoding of accounts. +type Account struct { + Code []byte `json:"code,omitempty"` + Storage map[common.Hash]common.Hash `json:"storage,omitempty"` + Balance *big.Int `json:"balance" gencodec:"required"` + Nonce uint64 `json:"nonce,omitempty"` + + // used in tests + PrivateKey []byte `json:"secretKey,omitempty"` +} + +type accountMarshaling struct { + Code hexutil.Bytes + Balance *math.HexOrDecimal256 + Nonce math.HexOrDecimal64 + Storage map[storageJSON]storageJSON + PrivateKey hexutil.Bytes +} + +// storageJSON represents a 256 bit byte array, but allows less than 256 bits when +// unmarshalling from hex. +type storageJSON common.Hash + +func (h *storageJSON) UnmarshalText(text []byte) error { + text = bytes.TrimPrefix(text, []byte("0x")) + if len(text) > 64 { + return fmt.Errorf("too many hex characters in storage key/value %q", text) + } + offset := len(h) - len(text)/2 // pad on the left + if _, err := hex.Decode(h[offset:], text); err != nil { + return fmt.Errorf("invalid hex storage key/value %q", text) + } + return nil +} + +func (h storageJSON) MarshalText() ([]byte, error) { + return hexutil.Bytes(h[:]).MarshalText() +} + +// GenesisAlloc specifies the initial state of a genesis block. +type GenesisAlloc map[common.Address]Account + +func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { + m := make(map[common.UnprefixedAddress]Account) + if err := json.Unmarshal(data, &m); err != nil { + return err + } + *ga = make(GenesisAlloc) + for addr, a := range m { + (*ga)[common.Address(addr)] = a + } + return nil +} diff --git a/core/types/block_test.go b/core/types/block_test.go index cf0b1dd85c..982d002242 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -196,7 +196,7 @@ func TestEIP2718BlockEncoding(t *testing.T) { func TestUncleHash(t *testing.T) { uncles := make([]*Header, 0) h := CalcUncleHash(uncles) - exp := common.HexToHash("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347") + exp := EmptyUncleHash if h != exp { t.Fatalf("empty uncle hash is wrong, got %x != %x", h, exp) } diff --git a/core/gen_genesis_account.go b/core/types/gen_account.go similarity index 61% rename from core/gen_genesis_account.go rename to core/types/gen_account.go index a9d47e6ba3..4e475896a7 100644 --- a/core/gen_genesis_account.go +++ b/core/types/gen_account.go @@ -1,6 +1,6 @@ // Code generated by github.com/fjl/gencodec. DO NOT EDIT. -package core +package types import ( "encoding/json" @@ -12,62 +12,62 @@ import ( "github.com/ethereum/go-ethereum/common/math" ) -var _ = (*genesisAccountMarshaling)(nil) +var _ = (*accountMarshaling)(nil) // MarshalJSON marshals as JSON. -func (g GenesisAccount) MarshalJSON() ([]byte, error) { - type GenesisAccount struct { +func (a Account) MarshalJSON() ([]byte, error) { + type Account struct { Code hexutil.Bytes `json:"code,omitempty"` Storage map[storageJSON]storageJSON `json:"storage,omitempty"` Balance *math.HexOrDecimal256 `json:"balance" gencodec:"required"` Nonce math.HexOrDecimal64 `json:"nonce,omitempty"` PrivateKey hexutil.Bytes `json:"secretKey,omitempty"` } - var enc GenesisAccount - enc.Code = g.Code - if g.Storage != nil { - enc.Storage = make(map[storageJSON]storageJSON, len(g.Storage)) - for k, v := range g.Storage { + var enc Account + enc.Code = a.Code + if a.Storage != nil { + enc.Storage = make(map[storageJSON]storageJSON, len(a.Storage)) + for k, v := range a.Storage { enc.Storage[storageJSON(k)] = storageJSON(v) } } - enc.Balance = (*math.HexOrDecimal256)(g.Balance) - enc.Nonce = math.HexOrDecimal64(g.Nonce) - enc.PrivateKey = g.PrivateKey + enc.Balance = (*math.HexOrDecimal256)(a.Balance) + enc.Nonce = math.HexOrDecimal64(a.Nonce) + enc.PrivateKey = a.PrivateKey return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. -func (g *GenesisAccount) UnmarshalJSON(input []byte) error { - type GenesisAccount struct { +func (a *Account) UnmarshalJSON(input []byte) error { + type Account struct { Code *hexutil.Bytes `json:"code,omitempty"` Storage map[storageJSON]storageJSON `json:"storage,omitempty"` Balance *math.HexOrDecimal256 `json:"balance" gencodec:"required"` Nonce *math.HexOrDecimal64 `json:"nonce,omitempty"` PrivateKey *hexutil.Bytes `json:"secretKey,omitempty"` } - var dec GenesisAccount + var dec Account if err := json.Unmarshal(input, &dec); err != nil { return err } if dec.Code != nil { - g.Code = *dec.Code + a.Code = *dec.Code } if dec.Storage != nil { - g.Storage = make(map[common.Hash]common.Hash, len(dec.Storage)) + a.Storage = make(map[common.Hash]common.Hash, len(dec.Storage)) for k, v := range dec.Storage { - g.Storage[common.Hash(k)] = common.Hash(v) + a.Storage[common.Hash(k)] = common.Hash(v) } } if dec.Balance == nil { - return errors.New("missing required field 'balance' for GenesisAccount") + return errors.New("missing required field 'balance' for Account") } - g.Balance = (*big.Int)(dec.Balance) + a.Balance = (*big.Int)(dec.Balance) if dec.Nonce != nil { - g.Nonce = uint64(*dec.Nonce) + a.Nonce = uint64(*dec.Nonce) } if dec.PrivateKey != nil { - g.PrivateKey = *dec.PrivateKey + a.PrivateKey = *dec.PrivateKey } return nil } diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go index d2a98ed7bf..a6949414f3 100644 --- a/core/types/hashing_test.go +++ b/core/types/hashing_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) func TestDeriveSha(t *testing.T) { @@ -39,7 +40,7 @@ func TestDeriveSha(t *testing.T) { t.Fatal(err) } for len(txs) < 1000 { - exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) + exp := types.DeriveSha(txs, trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(txs, trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) @@ -86,7 +87,7 @@ func BenchmarkDeriveSha200(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) + exp = types.DeriveSha(txs, trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil))) } }) @@ -107,7 +108,7 @@ func TestFuzzDeriveSha(t *testing.T) { rndSeed := mrand.Int() for i := 0; i < 10; i++ { seed := rndSeed + i - exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) + exp := types.DeriveSha(newDummy(i), trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { printList(newDummy(seed)) @@ -135,7 +136,7 @@ func TestDerivableList(t *testing.T) { }, } for i, tc := range tcs[1:] { - exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) + exp := types.DeriveSha(flatList(tc), trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("case %d: got %x exp %x", i, got, exp) diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index a7b2644471..fc51eb11a5 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -343,7 +343,7 @@ func TestReceiptJSON(t *testing.T) { r := Receipt{} err = r.UnmarshalJSON(b) if err != nil { - t.Fatal("error unmarshaling receipt from json:", err) + t.Fatal("error unmarshalling receipt from json:", err) } } } @@ -360,7 +360,7 @@ func TestEffectiveGasPriceNotRequired(t *testing.T) { r2 := Receipt{} err = r2.UnmarshalJSON(b) if err != nil { - t.Fatal("error unmarshaling receipt from json:", err) + t.Fatal("error unmarshalling receipt from json:", err) } } diff --git a/core/types/transaction_marshalling.go b/core/types/transaction_marshalling.go index 08ce80b07c..4d5b2bcdd4 100644 --- a/core/types/transaction_marshalling.go +++ b/core/types/transaction_marshalling.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/holiman/uint256" ) @@ -47,6 +48,11 @@ type txJSON struct { S *hexutil.Big `json:"s"` YParity *hexutil.Uint64 `json:"yParity,omitempty"` + // Blob transaction sidecar encoding: + Blobs []kzg4844.Blob `json:"blobs,omitempty"` + Commitments []kzg4844.Commitment `json:"commitments,omitempty"` + Proofs []kzg4844.Proof `json:"proofs,omitempty"` + // Only used for encoding: Hash common.Hash `json:"hash"` } @@ -142,6 +148,11 @@ func (tx *Transaction) MarshalJSON() ([]byte, error) { enc.S = (*hexutil.Big)(itx.S.ToBig()) yparity := itx.V.Uint64() enc.YParity = (*hexutil.Uint64)(&yparity) + if sidecar := itx.Sidecar; sidecar != nil { + enc.Blobs = itx.Sidecar.Blobs + enc.Commitments = itx.Sidecar.Commitments + enc.Proofs = itx.Sidecar.Proofs + } } return json.Marshal(&enc) } diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 9e26642f75..70dee0776e 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -107,13 +107,7 @@ func SignTx(tx *Transaction, s Signer, prv *ecdsa.PrivateKey) (*Transaction, err // SignNewTx creates a transaction and signs it. func SignNewTx(prv *ecdsa.PrivateKey, s Signer, txdata TxData) (*Transaction, error) { - tx := NewTx(txdata) - h := s.Hash(tx) - sig, err := crypto.Sign(h[:], prv) - if err != nil { - return nil, err - } - return tx.WithSignature(s, sig) + return SignTx(NewTx(txdata), s, prv) } // MustSignNewTx creates a transaction and signs it. diff --git a/core/types/transaction_signing_test.go b/core/types/transaction_signing_test.go index 61b78fe029..b66577f7ed 100644 --- a/core/types/transaction_signing_test.go +++ b/core/types/transaction_signing_test.go @@ -43,7 +43,7 @@ func TestEIP155Signing(t *testing.T) { t.Fatal(err) } if from != addr { - t.Errorf("exected from and address to be equal. Got %x want %x", from, addr) + t.Errorf("expected from and address to be equal. Got %x want %x", from, addr) } } diff --git a/core/types/tx_blob.go b/core/types/tx_blob.go index caede7cc53..25a85695ef 100644 --- a/core/types/tx_blob.go +++ b/core/types/tx_blob.go @@ -43,7 +43,7 @@ type BlobTx struct { BlobHashes []common.Hash // A blob transaction can optionally contain blobs. This field must be set when BlobTx - // is used to create a transaction for sigining. + // is used to create a transaction for signing. Sidecar *BlobTxSidecar `rlp:"-"` // Signature values diff --git a/core/types/tx_blob_test.go b/core/types/tx_blob_test.go index 25d09e31ce..6bd0f183b7 100644 --- a/core/types/tx_blob_test.go +++ b/core/types/tx_blob_test.go @@ -59,7 +59,7 @@ func TestBlobTxSize(t *testing.T) { } var ( - emptyBlob = kzg4844.Blob{} + emptyBlob = new(kzg4844.Blob) emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) ) @@ -72,7 +72,7 @@ func createEmptyBlobTx(key *ecdsa.PrivateKey, withSidecar bool) *Transaction { func createEmptyBlobTxInner(withSidecar bool) *BlobTx { sidecar := &BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: []kzg4844.Blob{*emptyBlob}, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, } diff --git a/core/vm/contract.go b/core/vm/contract.go index 6980337deb..4e28260a67 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -18,6 +18,7 @@ package vm import ( "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/holiman/uint256" ) @@ -157,17 +158,28 @@ func (c *Contract) Caller() common.Address { } // UseGas attempts the use gas and subtracts it and returns true on success -func (c *Contract) UseGas(gas uint64, logger EVMLogger, reason GasChangeReason) (ok bool) { +func (c *Contract) UseGas(gas uint64, logger *tracing.Hooks, reason tracing.GasChangeReason) (ok bool) { if c.Gas < gas { return false } - if logger != nil && reason != GasChangeIgnored { + if logger != nil && logger.OnGasChange != nil && reason != tracing.GasChangeIgnored { logger.OnGasChange(c.Gas, c.Gas-gas, reason) } c.Gas -= gas return true } +// RefundGas refunds gas to the contract +func (c *Contract) RefundGas(gas uint64, logger *tracing.Hooks, reason tracing.GasChangeReason) { + if gas == 0 { + return + } + if logger != nil && logger.OnGasChange != nil && reason != tracing.GasChangeIgnored { + logger.OnGasChange(c.Gas, c.Gas+gas, reason) + } + c.Gas += gas +} + // Address returns the contracts address func (c *Contract) Address() common.Address { return c.self.Address() diff --git a/core/vm/contracts.go b/core/vm/contracts.go index b1d094f7c8..a6af31f584 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/blake2b" "github.com/ethereum/go-ethereum/crypto/bls12381" @@ -168,13 +169,13 @@ func ActivePrecompiles(rules params.Rules) []common.Address { // - the returned bytes, // - the _remaining_ gas, // - any error that occurred -func RunPrecompiledContract(p PrecompiledContract, input []byte, suppliedGas uint64, logger EVMLogger) (ret []byte, remainingGas uint64, err error) { +func RunPrecompiledContract(p PrecompiledContract, input []byte, suppliedGas uint64, logger *tracing.Hooks) (ret []byte, remainingGas uint64, err error) { gasCost := p.RequiredGas(input) if suppliedGas < gasCost { return nil, 0, ErrOutOfGas } - if logger != nil { - logger.OnGasChange(suppliedGas, suppliedGas-gasCost, GasChangeCallPrecompiledContract) + if logger != nil && logger.OnGasChange != nil { + logger.OnGasChange(suppliedGas, suppliedGas-gasCost, tracing.GasChangeCallPrecompiledContract) } suppliedGas -= gasCost output, err := p.Run(input) diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index 04aa8a116d..6608ff09fc 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -223,7 +223,7 @@ func BenchmarkPrecompiledRipeMD(bench *testing.B) { benchmarkPrecompiled("03", t, bench) } -// Benchmarks the sample inputs from the identiy precompile. +// Benchmarks the sample inputs from the identity precompile. func BenchmarkPrecompiledIdentity(bench *testing.B) { t := precompiledTest{ Input: "38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e000000000000000000000000000000000000000000000000000000000000001b38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e789d1dd423d25f0772d2748d60f7e4b81bb14d086eba8e8e8efb6dcff8a4ae02", diff --git a/core/vm/errors.go b/core/vm/errors.go index 9f6605f030..ba3261c797 100644 --- a/core/vm/errors.go +++ b/core/vm/errors.go @@ -30,7 +30,6 @@ var ( ErrInsufficientBalance = errors.New("insufficient balance for transfer") ErrContractAddressCollision = errors.New("contract address collision") ErrExecutionReverted = errors.New("execution reverted") - ErrMaxInitCodeSizeExceeded = errors.New("max initcode size exceeded") ErrMaxCodeSizeExceeded = errors.New("max code size exceeded") ErrInvalidJump = errors.New("invalid jump destination") ErrWriteProtection = errors.New("write protection") @@ -129,7 +128,6 @@ const ( VMErrorCodeInsufficientBalance VMErrorCodeContractAddressCollision VMErrorCodeExecutionReverted - VMErrorCodeMaxInitCodeSizeExceeded VMErrorCodeMaxCodeSizeExceeded VMErrorCodeInvalidJump VMErrorCodeWriteProtection @@ -160,8 +158,6 @@ func vmErrorCodeFromErr(err error) int { return VMErrorCodeContractAddressCollision case errors.Is(err, ErrExecutionReverted): return VMErrorCodeExecutionReverted - case errors.Is(err, ErrMaxInitCodeSizeExceeded): - return VMErrorCodeMaxInitCodeSizeExceeded case errors.Is(err, ErrMaxCodeSizeExceeded): return VMErrorCodeMaxCodeSizeExceeded case errors.Is(err, ErrInvalidJump): diff --git a/core/vm/evm.go b/core/vm/evm.go index 2c0b4e1e9f..f9e30fbf09 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -22,7 +22,7 @@ import ( "sync/atomic" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" @@ -186,9 +186,9 @@ func (evm *EVM) Interpreter() *EVMInterpreter { func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas uint64, value *uint256.Int) (ret []byte, leftOverGas uint64, err error) { // Capture the tracer start/end events in debug mode if evm.Config.Tracer != nil { - evm.captureBegin(evm.depth == 0, CALL, caller.Address(), addr, input, gas, value.ToBig()) + evm.captureBegin(evm.depth, CALL, caller.Address(), addr, input, gas, value.ToBig()) defer func(startGas uint64) { - evm.captureEnd(evm.depth == 0, CALL, startGas, leftOverGas, ret, err) + evm.captureEnd(evm.depth, CALL, startGas, leftOverGas, ret, err) }(gas) } // Fail if we're trying to execute above the call depth limit @@ -196,14 +196,14 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas return nil, gas, ErrDepth } // Fail if we're trying to transfer more than the available balance - if value.Sign() != 0 && !evm.Context.CanTransfer(evm.StateDB, caller.Address(), value) { + if !value.IsZero() && !evm.Context.CanTransfer(evm.StateDB, caller.Address(), value) { return nil, gas, ErrInsufficientBalance } snapshot := evm.StateDB.Snapshot() p, isPrecompile := evm.precompile(addr) if !evm.StateDB.Exist(addr) { - if !isPrecompile && evm.chainRules.IsEIP158 && value.Sign() == 0 { + if !isPrecompile && evm.chainRules.IsEIP158 && value.IsZero() { // Calling a non-existing account, don't do anything. return nil, gas, nil } @@ -235,8 +235,8 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas if err != nil { evm.StateDB.RevertToSnapshot(snapshot) if err != ErrExecutionReverted { - if evm.Config.Tracer != nil { - evm.Config.Tracer.OnGasChange(gas, 0, GasChangeCallFailedExecution) + if evm.Config.Tracer != nil && evm.Config.Tracer.OnGasChange != nil { + evm.Config.Tracer.OnGasChange(gas, 0, tracing.GasChangeCallFailedExecution) } gas = 0 @@ -258,9 +258,9 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, gas uint64, value *uint256.Int) (ret []byte, leftOverGas uint64, err error) { // Invoke tracer hooks that signal entering/exiting a call frame if evm.Config.Tracer != nil { - evm.captureBegin(false, CALLCODE, caller.Address(), addr, input, gas, value.ToBig()) + evm.captureBegin(evm.depth, CALLCODE, caller.Address(), addr, input, gas, value.ToBig()) defer func(startGas uint64) { - evm.captureEnd(false, CALLCODE, startGas, leftOverGas, ret, err) + evm.captureEnd(evm.depth, CALLCODE, startGas, leftOverGas, ret, err) }(gas) } // Fail if we're trying to execute above the call depth limit @@ -291,8 +291,8 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, if err != nil { evm.StateDB.RevertToSnapshot(snapshot) if err != ErrExecutionReverted { - if evm.Config.Tracer != nil { - evm.Config.Tracer.OnGasChange(gas, 0, GasChangeCallFailedExecution) + if evm.Config.Tracer != nil && evm.Config.Tracer.OnGasChange != nil { + evm.Config.Tracer.OnGasChange(gas, 0, tracing.GasChangeCallFailedExecution) } gas = 0 @@ -313,9 +313,9 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by // that caller is something other than a Contract. parent := caller.(*Contract) // DELEGATECALL inherits value from parent call - evm.captureBegin(false, DELEGATECALL, caller.Address(), addr, input, gas, parent.value.ToBig()) + evm.captureBegin(evm.depth, DELEGATECALL, caller.Address(), addr, input, gas, parent.value.ToBig()) defer func(startGas uint64) { - evm.captureEnd(false, DELEGATECALL, startGas, leftOverGas, ret, err) + evm.captureEnd(evm.depth, DELEGATECALL, startGas, leftOverGas, ret, err) }(gas) } // Fail if we're trying to execute above the call depth limit @@ -338,8 +338,8 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by if err != nil { evm.StateDB.RevertToSnapshot(snapshot) if err != ErrExecutionReverted { - if evm.Config.Tracer != nil { - evm.Config.Tracer.OnGasChange(gas, 0, GasChangeCallFailedExecution) + if evm.Config.Tracer != nil && evm.Config.Tracer.OnGasChange != nil { + evm.Config.Tracer.OnGasChange(gas, 0, tracing.GasChangeCallFailedExecution) } gas = 0 } @@ -354,9 +354,9 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte, gas uint64) (ret []byte, leftOverGas uint64, err error) { // Invoke tracer hooks that signal entering/exiting a call frame if evm.Config.Tracer != nil { - evm.captureBegin(false, STATICCALL, caller.Address(), addr, input, gas, nil) + evm.captureBegin(evm.depth, STATICCALL, caller.Address(), addr, input, gas, nil) defer func(startGas uint64) { - evm.captureEnd(false, STATICCALL, startGas, leftOverGas, ret, err) + evm.captureEnd(evm.depth, STATICCALL, startGas, leftOverGas, ret, err) }(gas) } // Fail if we're trying to execute above the call depth limit @@ -374,7 +374,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // This doesn't matter on Mainnet, where all empties are gone at the time of Byzantium, // but is the correct thing to do and matters on other networks, in tests, and potential // future scenarios - evm.StateDB.AddBalance(addr, new(uint256.Int), state.BalanceChangeTouchAccount) + evm.StateDB.AddBalance(addr, new(uint256.Int), tracing.BalanceChangeTouchAccount) if p, isPrecompile := evm.precompile(addr); isPrecompile { ret, gas, err = RunPrecompiledContract(p, input, gas, evm.Config.Tracer) @@ -396,8 +396,8 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte if err != nil { evm.StateDB.RevertToSnapshot(snapshot) if err != ErrExecutionReverted { - if evm.Config.Tracer != nil { - evm.Config.Tracer.OnGasChange(gas, 0, GasChangeCallFailedExecution) + if evm.Config.Tracer != nil && evm.Config.Tracer.OnGasChange != nil { + evm.Config.Tracer.OnGasChange(gas, 0, tracing.GasChangeCallFailedExecution) } gas = 0 @@ -421,9 +421,9 @@ func (c *codeAndHash) Hash() common.Hash { // create creates a new contract using code as deployment code. func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *uint256.Int, address common.Address, typ OpCode) (ret []byte, createAddress common.Address, leftOverGas uint64, err error) { if evm.Config.Tracer != nil { - evm.captureBegin(evm.depth == 0, typ, caller.Address(), address, codeAndHash.code, gas, value.ToBig()) + evm.captureBegin(evm.depth, typ, caller.Address(), address, codeAndHash.code, gas, value.ToBig()) defer func(startGas uint64) { - evm.captureEnd(evm.depth == 0, typ, startGas, leftOverGas, ret, err) + evm.captureEnd(evm.depth, typ, startGas, leftOverGas, ret, err) }(gas) } // Depth check execution. Fail if we're trying to execute above the @@ -447,8 +447,8 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, // Ensure there's no existing contract already at the designated address contractHash := evm.StateDB.GetCodeHash(address) if evm.StateDB.GetNonce(address) != 0 || (contractHash != (common.Hash{}) && contractHash != types.EmptyCodeHash) { - if evm.Config.Tracer != nil { - evm.Config.Tracer.OnGasChange(gas, 0, GasChangeCallFailedExecution) + if evm.Config.Tracer != nil && evm.Config.Tracer.OnGasChange != nil { + evm.Config.Tracer.OnGasChange(gas, 0, tracing.GasChangeCallFailedExecution) } return nil, common.Address{}, 0, ErrContractAddressCollision @@ -484,7 +484,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, // by the error checking condition below. if err == nil { createDataGas := uint64(len(ret)) * params.CreateDataGas - if contract.UseGas(createDataGas, evm.Config.Tracer, GasChangeCallCodeStorage) { + if contract.UseGas(createDataGas, evm.Config.Tracer, tracing.GasChangeCallCodeStorage) { evm.StateDB.SetCode(address, ret) } else { err = ErrCodeStoreOutOfGas @@ -497,7 +497,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, if err != nil && (evm.chainRules.IsHomestead || err != ErrCodeStoreOutOfGas) { evm.StateDB.RevertToSnapshot(snapshot) if err != ErrExecutionReverted { - contract.UseGas(contract.Gas, evm.Config.Tracer, GasChangeCallFailedExecution) + contract.UseGas(contract.Gas, evm.Config.Tracer, tracing.GasChangeCallFailedExecution) } } @@ -523,23 +523,20 @@ func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment * // ChainConfig returns the environment's chain configuration func (evm *EVM) ChainConfig() *params.ChainConfig { return evm.chainConfig } -func (evm *EVM) captureBegin(isRoot bool, typ OpCode, from common.Address, to common.Address, input []byte, startGas uint64, value *big.Int) { +func (evm *EVM) captureBegin(depth int, typ OpCode, from common.Address, to common.Address, input []byte, startGas uint64, value *big.Int) { tracer := evm.Config.Tracer - - if isRoot { - tracer.CaptureStart(from, to, typ == CREATE || typ == CREATE2, input, startGas, value) - } else { - tracer.CaptureEnter(typ, from, to, input, startGas, value) + if tracer.OnEnter != nil { + tracer.OnEnter(depth, byte(typ), from, to, input, startGas, value) + } + if tracer.OnGasChange != nil { + tracer.OnGasChange(0, startGas, tracing.GasChangeCallInitialBalance) } - - tracer.OnGasChange(0, startGas, GasChangeCallInitialBalance) } -func (evm *EVM) captureEnd(isRoot bool, typ OpCode, startGas uint64, leftOverGas uint64, ret []byte, err error) { +func (evm *EVM) captureEnd(depth int, typ OpCode, startGas uint64, leftOverGas uint64, ret []byte, err error) { tracer := evm.Config.Tracer - - if leftOverGas != 0 { - tracer.OnGasChange(leftOverGas, 0, GasChangeCallLeftOverReturned) + if leftOverGas != 0 && tracer.OnGasChange != nil { + tracer.OnGasChange(leftOverGas, 0, tracing.GasChangeCallLeftOverReturned) } var reverted bool if err != nil { @@ -548,9 +545,21 @@ func (evm *EVM) captureEnd(isRoot bool, typ OpCode, startGas uint64, leftOverGas if !evm.chainRules.IsHomestead && errors.Is(err, ErrCodeStoreOutOfGas) { reverted = false } - if isRoot { - tracer.CaptureEnd(ret, startGas-leftOverGas, VMErrorFromErr(err), reverted) - } else { - tracer.CaptureExit(ret, startGas-leftOverGas, VMErrorFromErr(err), reverted) + if tracer.OnExit != nil { + tracer.OnExit(depth, ret, startGas-leftOverGas, VMErrorFromErr(err), reverted) + } +} + +// GetVMContext provides context about the block being executed as well as state +// to the tracers. +func (evm *EVM) GetVMContext() *tracing.VMContext { + return &tracing.VMContext{ + Coinbase: evm.Context.Coinbase, + BlockNumber: evm.Context.BlockNumber, + Time: evm.Context.Time, + Random: evm.Context.Random, + GasPrice: evm.TxContext.GasPrice, + ChainConfig: evm.ChainConfig(), + StateDB: evm.StateDB, } } diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 31c9b18328..990bdbf925 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -17,8 +17,10 @@ package vm import ( + "math" + "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" @@ -248,9 +250,6 @@ func opKeccak256(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ( if evm.Config.EnablePreimageRecording { evm.StateDB.AddPreimage(interpreter.hasherBuf, data) } - if interpreter.evm.Config.Tracer != nil { - interpreter.evm.Config.Tracer.CaptureKeccakPreimage(common.BytesToHash(interpreter.hasherBuf[:]), data) - } size.SetBytes(interpreter.hasherBuf[:]) return nil, nil } @@ -306,7 +305,7 @@ func opCallDataCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext ) dataOffset64, overflow := dataOffset.Uint64WithOverflow() if overflow { - dataOffset64 = 0xffffffffffffffff + dataOffset64 = math.MaxUint64 } // These values are checked for overflow during gas cost calculation memOffset64 := memOffset.Uint64() @@ -350,9 +349,7 @@ func opExtCodeSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) } func opCodeSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { - l := new(uint256.Int) - l.SetUint64(uint64(len(scope.Contract.Code))) - scope.Stack.push(l) + scope.Stack.push(new(uint256.Int).SetUint64(uint64(len(scope.Contract.Code)))) return nil, nil } @@ -364,7 +361,7 @@ func opCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ ) uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow() if overflow { - uint64CodeOffset = 0xffffffffffffffff + uint64CodeOffset = math.MaxUint64 } codeCopy := getData(scope.Contract.Code, uint64CodeOffset, length.Uint64()) scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) @@ -382,7 +379,7 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ) uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow() if overflow { - uint64CodeOffset = 0xffffffffffffffff + uint64CodeOffset = math.MaxUint64 } addr := common.Address(a.Bytes20()) codeCopy := getData(interpreter.evm.StateDB.GetCode(addr), uint64CodeOffset, length.Uint64()) @@ -593,7 +590,7 @@ func opCreate(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]b // reuse size int for stackvalue stackvalue := size - scope.Contract.UseGas(gas, interpreter.evm.Config.Tracer, GasChangeCallContractCreation) + scope.Contract.UseGas(gas, interpreter.evm.Config.Tracer, tracing.GasChangeCallContractCreation) res, addr, returnGas, suberr := interpreter.evm.Create(scope.Contract, input, gas, &value) // Push item on the stack based on the returned error. If the ruleset is @@ -609,11 +606,7 @@ func opCreate(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]b } scope.Stack.push(&stackvalue) - if interpreter.evm.Config.Tracer != nil && returnGas > 0 { - interpreter.evm.Config.Tracer.OnGasChange(scope.Contract.Gas, scope.Contract.Gas+returnGas, GasChangeCallLeftOverRefunded) - } - - scope.Contract.Gas += returnGas + scope.Contract.RefundGas(returnGas, interpreter.evm.Config.Tracer, tracing.GasChangeCallLeftOverRefunded) if suberr == ErrExecutionReverted { interpreter.returnData = res // set REVERT data to return data buffer @@ -636,7 +629,7 @@ func opCreate2(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([] ) // Apply EIP150 gas -= gas / 64 - scope.Contract.UseGas(gas, interpreter.evm.Config.Tracer, GasChangeCallContractCreation2) + scope.Contract.UseGas(gas, interpreter.evm.Config.Tracer, tracing.GasChangeCallContractCreation2) // reuse size int for stackvalue stackvalue := size res, addr, returnGas, suberr := interpreter.evm.Create2(scope.Contract, input, gas, @@ -649,11 +642,7 @@ func opCreate2(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([] } scope.Stack.push(&stackvalue) - if interpreter.evm.Config.Tracer != nil && returnGas > 0 { - interpreter.evm.Config.Tracer.OnGasChange(scope.Contract.Gas, scope.Contract.Gas+returnGas, GasChangeCallLeftOverRefunded) - } - - scope.Contract.Gas += returnGas + scope.Contract.RefundGas(returnGas, interpreter.evm.Config.Tracer, tracing.GasChangeCallLeftOverRefunded) if suberr == ErrExecutionReverted { interpreter.returnData = res // set REVERT data to return data buffer @@ -693,11 +682,7 @@ func opCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byt scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } - if interpreter.evm.Config.Tracer != nil && returnGas > 0 { - interpreter.evm.Config.Tracer.OnGasChange(scope.Contract.Gas, scope.Contract.Gas+returnGas, GasChangeCallLeftOverRefunded) - } - - scope.Contract.Gas += returnGas + scope.Contract.RefundGas(returnGas, interpreter.evm.Config.Tracer, tracing.GasChangeCallLeftOverRefunded) interpreter.returnData = ret return ret, nil @@ -730,11 +715,7 @@ func opCallCode(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } - if interpreter.evm.Config.Tracer != nil && returnGas > 0 { - interpreter.evm.Config.Tracer.OnGasChange(scope.Contract.Gas, scope.Contract.Gas+returnGas, GasChangeCallLeftOverRefunded) - } - - scope.Contract.Gas += returnGas + scope.Contract.RefundGas(returnGas, interpreter.evm.Config.Tracer, tracing.GasChangeCallLeftOverRefunded) interpreter.returnData = ret return ret, nil @@ -763,11 +744,7 @@ func opDelegateCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } - if interpreter.evm.Config.Tracer != nil && returnGas > 0 { - interpreter.evm.Config.Tracer.OnGasChange(scope.Contract.Gas, scope.Contract.Gas+returnGas, GasChangeCallLeftOverRefunded) - } - - scope.Contract.Gas += returnGas + scope.Contract.RefundGas(returnGas, interpreter.evm.Config.Tracer, tracing.GasChangeCallLeftOverRefunded) interpreter.returnData = ret return ret, nil @@ -796,11 +773,7 @@ func opStaticCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } - if interpreter.evm.Config.Tracer != nil && returnGas > 0 { - interpreter.evm.Config.Tracer.OnGasChange(scope.Contract.Gas, scope.Contract.Gas+returnGas, GasChangeCallLeftOverRefunded) - } - - scope.Contract.Gas += returnGas + scope.Contract.RefundGas(returnGas, interpreter.evm.Config.Tracer, tracing.GasChangeCallLeftOverRefunded) interpreter.returnData = ret return ret, nil @@ -835,11 +808,15 @@ func opSelfdestruct(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext } beneficiary := scope.Stack.pop() balance := interpreter.evm.StateDB.GetBalance(scope.Contract.Address()) - interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, state.BalanceIncreaseSelfdestruct) + interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, tracing.BalanceIncreaseSelfdestruct) interpreter.evm.StateDB.SelfDestruct(scope.Contract.Address()) if tracer := interpreter.evm.Config.Tracer; tracer != nil { - tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) - tracer.CaptureExit([]byte{}, 0, nil, false) + if tracer.OnEnter != nil { + tracer.OnEnter(interpreter.evm.depth, byte(SELFDESTRUCT), scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) + } + if tracer.OnExit != nil { + tracer.OnExit(interpreter.evm.depth, []byte{}, 0, nil, false) + } } return nil, errStopToken } @@ -850,12 +827,16 @@ func opSelfdestruct6780(pc *uint64, interpreter *EVMInterpreter, scope *ScopeCon } beneficiary := scope.Stack.pop() balance := interpreter.evm.StateDB.GetBalance(scope.Contract.Address()) - interpreter.evm.StateDB.SubBalance(scope.Contract.Address(), balance, state.BalanceDecreaseSelfdestruct) - interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, state.BalanceIncreaseSelfdestruct) + interpreter.evm.StateDB.SubBalance(scope.Contract.Address(), balance, tracing.BalanceDecreaseSelfdestruct) + interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, tracing.BalanceIncreaseSelfdestruct) interpreter.evm.StateDB.Selfdestruct6780(scope.Contract.Address()) if tracer := interpreter.evm.Config.Tracer; tracer != nil { - tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) - tracer.CaptureExit([]byte{}, 0, nil, false) + if tracer.OnEnter != nil { + tracer.OnEnter(interpreter.evm.depth, byte(SELFDESTRUCT), scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) + } + if tracer.OnExit != nil { + tracer.OnExit(interpreter.evm.depth, []byte{}, 0, nil, false) + } } return nil, errStopToken } diff --git a/core/vm/interface.go b/core/vm/interface.go index 9d11bf3b72..d7028cc7c7 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -20,7 +20,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" @@ -30,8 +30,8 @@ import ( type StateDB interface { CreateAccount(common.Address) - SubBalance(common.Address, *uint256.Int, state.BalanceChangeReason) - AddBalance(common.Address, *uint256.Int, state.BalanceChangeReason) + SubBalance(common.Address, *uint256.Int, tracing.BalanceChangeReason) + AddBalance(common.Address, *uint256.Int, tracing.BalanceChangeReason) GetBalance(common.Address) *uint256.Int GetNonce(common.Address) uint64 diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 3eff0d25db..47d75a89c9 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -19,16 +19,18 @@ package vm import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/holiman/uint256" ) // Config are the configuration options for the Interpreter type Config struct { - Tracer EVMLogger // Opcode logger - NoBaseFee bool // Forces the EIP-1559 baseFee to 0 (needed for 0 price calls) - EnablePreimageRecording bool // Enables recording of SHA3/keccak preimages - ExtraEips []int // Additional EIPS that are to be enabled + Tracer *tracing.Hooks + NoBaseFee bool // Forces the EIP-1559 baseFee to 0 (needed for 0 price calls) + EnablePreimageRecording bool // Enables recording of SHA3/keccak preimages + ExtraEips []int // Additional EIPS that are to be enabled } // ScopeContext contains the things that are per-call, such as stack and memory, @@ -39,6 +41,36 @@ type ScopeContext struct { Contract *Contract } +func (ctx *ScopeContext) MemoryData() []byte { + if ctx.Memory == nil { + return nil + } + return ctx.Memory.Data() +} + +func (ctx *ScopeContext) StackData() []uint256.Int { + if ctx.Stack == nil { + return nil + } + return ctx.Stack.Data() +} + +func (ctx *ScopeContext) Caller() common.Address { + return ctx.Contract.Caller() +} + +func (ctx *ScopeContext) Address() common.Address { + return ctx.Contract.Address() +} + +func (ctx *ScopeContext) CallValue() *uint256.Int { + return ctx.Contract.Value() +} + +func (ctx *ScopeContext) CallInput() []byte { + return ctx.Contract.Input +} + // EVMInterpreter represents an EVM interpreter type EVMInterpreter struct { evm *EVM @@ -146,8 +178,8 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( res []byte // result of the opcode execution function debug = in.evm.Config.Tracer != nil ) - // Don't move this deferred function, it's placed before the capturestate-deferred method, - // so that it get's executed _after_: the capturestate needs the stacks before + // Don't move this deferred function, it's placed before the OnOpcode-deferred method, + // so that it gets executed _after_: the OnOpcode needs the stacks before // they are returned to the pools defer func() { returnStack(stack) @@ -158,9 +190,11 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( defer func() { if err != nil { if !logged { - in.evm.Config.Tracer.CaptureState(pcCopy, op, gasCopy, cost, callContext, in.returnData, in.evm.depth, VMErrorFromErr(err)) - } else { - in.evm.Config.Tracer.CaptureFault(pcCopy, op, gasCopy, cost, callContext, in.evm.depth, VMErrorFromErr(err)) + if in.evm.Config.Tracer.OnOpcode != nil { + in.evm.Config.Tracer.OnOpcode(pcCopy, byte(op), gasCopy, cost, callContext, in.returnData, in.evm.depth, VMErrorFromErr(err)) + } + } else if in.evm.Config.Tracer.OnFault != nil { + in.evm.Config.Tracer.OnFault(pcCopy, byte(op), gasCopy, cost, callContext, in.evm.depth, VMErrorFromErr(err)) } } }() @@ -185,7 +219,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( } else if sLen > operation.maxStack { return nil, &ErrStackOverflow{stackLen: sLen, limit: operation.maxStack} } - if !contract.UseGas(cost, in.evm.Config.Tracer, GasChangeIgnored) { + if !contract.UseGas(cost, in.evm.Config.Tracer, tracing.GasChangeIgnored) { return nil, ErrOutOfGas } @@ -212,23 +246,31 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( var dynamicCost uint64 dynamicCost, err = operation.dynamicGas(in.evm, contract, stack, mem, memorySize) cost += dynamicCost // for tracing - if err != nil || !contract.UseGas(dynamicCost, in.evm.Config.Tracer, GasChangeIgnored) { + if err != nil || !contract.UseGas(dynamicCost, in.evm.Config.Tracer, tracing.GasChangeIgnored) { return nil, ErrOutOfGas } // Do tracing before memory expansion if debug { - in.evm.Config.Tracer.OnGasChange(gasCopy, gasCopy-cost, GasChangeCallOpCode) - in.evm.Config.Tracer.CaptureState(pc, op, gasCopy, cost, callContext, in.returnData, in.evm.depth, VMErrorFromErr(err)) - logged = true + if in.evm.Config.Tracer.OnGasChange != nil { + in.evm.Config.Tracer.OnGasChange(gasCopy, gasCopy-cost, tracing.GasChangeCallOpCode) + } + if in.evm.Config.Tracer.OnOpcode != nil { + in.evm.Config.Tracer.OnOpcode(pc, byte(op), gasCopy, cost, callContext, in.returnData, in.evm.depth, VMErrorFromErr(err)) + logged = true + } } if memorySize > 0 { mem.Resize(memorySize) } } else if debug { - in.evm.Config.Tracer.OnGasChange(gasCopy, gasCopy-cost, GasChangeCallOpCode) - in.evm.Config.Tracer.CaptureState(pc, op, gasCopy, cost, callContext, in.returnData, in.evm.depth, VMErrorFromErr(err)) - logged = true + if in.evm.Config.Tracer.OnGasChange != nil { + in.evm.Config.Tracer.OnGasChange(gasCopy, gasCopy-cost, tracing.GasChangeCallOpCode) + } + if in.evm.Config.Tracer.OnOpcode != nil { + in.evm.Config.Tracer.OnOpcode(pc, byte(op), gasCopy, cost, callContext, in.returnData, in.evm.depth, VMErrorFromErr(err)) + logged = true + } } // execute the operation diff --git a/core/vm/jump_table_test.go b/core/vm/jump_table_test.go index f67915fff3..02558035c0 100644 --- a/core/vm/jump_table_test.go +++ b/core/vm/jump_table_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" ) -// TestJumpTableCopy tests that deep copy is necessery to prevent modify shared jump table +// TestJumpTableCopy tests that deep copy is necessary to prevent modify shared jump table func TestJumpTableCopy(t *testing.T) { tbl := newMergeInstructionSet() require.Equal(t, uint64(0), tbl[SLOAD].constantGas) diff --git a/core/vm/logger.go b/core/vm/logger.go deleted file mode 100644 index e86bb5d98e..0000000000 --- a/core/vm/logger.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package vm - -import ( - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" -) - -// EVMLogger is used to collect execution traces from an EVM transaction -// execution. CaptureState is called for each step of the VM with the -// current VM state. -// Note that reference types are actual VM data structures; make copies -// if you need to retain them beyond the current call. -type EVMLogger interface { - // Transaction level - // Call simulations don't come with a valid signature. `from` field - // to be used for address of the caller. - CaptureTxStart(evm *EVM, tx *types.Transaction, from common.Address) - CaptureTxEnd(receipt *types.Receipt, err error) - // Top call frame - CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) - // CaptureEnd is invoked when the processing of the top call ends. - // See docs for `CaptureExit` for info on the `reverted` parameter. - CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) - // Rest of call frames - CaptureEnter(typ OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) - // CaptureExit is invoked when the processing of a message ends. - // `revert` is true when there was an error during the execution. - // Exceptionally, before the homestead hardfork a contract creation that - // ran out of gas when attempting to persist the code to database did not - // count as a call failure and did not cause a revert of the call. This will - // be indicated by `reverted == false` and `err == ErrCodeStoreOutOfGas`. - CaptureExit(output []byte, gasUsed uint64, err error, reverted bool) - // Opcode level - CaptureState(pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, err error) - CaptureFault(pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, depth int, err error) - CaptureKeccakPreimage(hash common.Hash, data []byte) - // Misc - OnGasChange(old, new uint64, reason GasChangeReason) -} - -// GasChangeReason is used to indicate the reason for a gas change, useful -// for tracing and reporting. -// -// There is essentially two types of gas changes, those that can be emitted once per transaction -// and those that can be emitted on a call basis, so possibly multiple times per transaction. -// -// They can be recognized easily by their name, those that start with `GasChangeTx` are emitted -// once per transaction, while those that start with `GasChangeCall` are emitted on a call basis. -type GasChangeReason byte - -const ( - GasChangeUnspecified GasChangeReason = iota - - // GasChangeTxInitialBalance is the initial balance for the call which will be equal to the gasLimit of the call. There is only - // one such gas change per transaction. - GasChangeTxInitialBalance - // GasChangeTxIntrinsicGas is the amount of gas that will be charged for the intrinsic cost of the transaction, there is - // always exactly one of those per transaction. - GasChangeTxIntrinsicGas - // GasChangeTxRefunds is the sum of all refunds which happened during the tx execution (e.g. storage slot being cleared) - // this generates an increase in gas. There is at most one of such gas change per transaction. - GasChangeTxRefunds - // GasChangeTxLeftOverReturned is the amount of gas left over at the end of transaction's execution that will be returned - // to the chain. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas - // left at the end of execution, no such even will be emitted. The returned gas's value in Wei is returned to caller. - // There is at most one of such gas change per transaction. - GasChangeTxLeftOverReturned - - // GasChangeCallInitialBalance is the initial balance for the call which will be equal to the gasLimit of the call. There is only - // one such gas change per call. - GasChangeCallInitialBalance - // GasChangeCallLeftOverReturned is the amount of gas left over that will be returned to the caller, this change will always - // be a negative change as we "drain" left over gas towards 0. If there was no gas left at the end of execution, no such even - // will be emitted. - GasChangeCallLeftOverReturned - // GasChangeCallLeftOverRefunded is the amount of gas that will be refunded to the call after the child call execution it - // executed completed. This value is always positive as we are giving gas back to the you, the left over gas of the child. - // If there was no gas left to be refunded, no such even will be emitted. - GasChangeCallLeftOverRefunded - // GasChangeCallContractCreation is the amount of gas that will be burned for a CREATE. - GasChangeCallContractCreation - // GasChangeContractCreation is the amount of gas that will be burned for a CREATE2. - GasChangeCallContractCreation2 - // GasChangeCallCodeStorage is the amount of gas that will be charged for code storage. - GasChangeCallCodeStorage - // GasChangeCallOpCode is the amount of gas that will be charged for an opcode executed by the EVM, exact opcode that was - // performed can be check by `CaptureState` handling. - GasChangeCallOpCode - // GasChangeCallPrecompiledContract is the amount of gas that will be charged for a precompiled contract execution. - GasChangeCallPrecompiledContract - // GasChangeCallStorageColdAccess is the amount of gas that will be charged for a cold storage access as controlled by EIP2929 rules. - GasChangeCallStorageColdAccess - // GasChangeCallFailedExecution is the burning of the remaining gas when the execution failed without a revert. - GasChangeCallFailedExecution - - // GasChangeIgnored is a special value that can be used to indicate that the gas change should be ignored as - // it will be "manually" tracked by a direct emit of the gas change event. - GasChangeIgnored GasChangeReason = 0xFF -) diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index d9a4952d41..289da44be3 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/params" ) @@ -169,7 +170,7 @@ func makeCallVariantGasCallEIP2929(oldCalculator gasFunc) gasFunc { evm.StateDB.AddAddressToAccessList(addr) // Charge the remaining difference here already, to correctly calculate available // gas for call - if !contract.UseGas(coldCost, evm.Config.Tracer, GasChangeCallStorageColdAccess) { + if !contract.UseGas(coldCost, evm.Config.Tracer, tracing.GasChangeCallStorageColdAccess) { return 0, ErrOutOfGas } } @@ -187,7 +188,12 @@ func makeCallVariantGasCallEIP2929(oldCalculator gasFunc) gasFunc { // outside of this function, as part of the dynamic gas, and that will make it // also become correctly reported to tracers. contract.Gas += coldCost - return gas + coldCost, nil + + var overflow bool + if gas, overflow = math.SafeAdd(gas, coldCost); overflow { + return 0, ErrGasUintOverflow + } + return gas, nil } } diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index ff03402f94..b50cc15e48 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -124,7 +124,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { rules = cfg.ChainConfig.Rules(vmenv.Context.BlockNumber, vmenv.Context.Random != nil, vmenv.Context.Time) ) if cfg.EVMConfig.Tracer != nil { - cfg.EVMConfig.Tracer.CaptureTxStart(vmenv, types.NewTx(&types.LegacyTx{To: &address, Data: input, Value: cfg.Value, Gas: cfg.GasLimit}), cfg.Origin) + cfg.EVMConfig.Tracer.OnTxStart(vmenv.GetVMContext(), types.NewTx(&types.LegacyTx{To: &address, Data: input, Value: cfg.Value, Gas: cfg.GasLimit}), cfg.Origin) } // Execute the preparatory steps for state transition which includes: // - prepare accessList(post-berlin) @@ -160,7 +160,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { rules = cfg.ChainConfig.Rules(vmenv.Context.BlockNumber, vmenv.Context.Random != nil, vmenv.Context.Time) ) if cfg.EVMConfig.Tracer != nil { - cfg.EVMConfig.Tracer.CaptureTxStart(vmenv, types.NewTx(&types.LegacyTx{Data: input, Value: cfg.Value, Gas: cfg.GasLimit}), cfg.Origin) + cfg.EVMConfig.Tracer.OnTxStart(vmenv.GetVMContext(), types.NewTx(&types.LegacyTx{Data: input, Value: cfg.Value, Gas: cfg.GasLimit}), cfg.Origin) } // Execute the preparatory steps for state transition which includes: // - prepare accessList(post-berlin) @@ -191,7 +191,7 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, er rules = cfg.ChainConfig.Rules(vmenv.Context.BlockNumber, vmenv.Context.Random != nil, vmenv.Context.Time) ) if cfg.EVMConfig.Tracer != nil { - cfg.EVMConfig.Tracer.CaptureTxStart(vmenv, types.NewTx(&types.LegacyTx{To: &address, Data: input, Value: cfg.Value, Gas: cfg.GasLimit}), cfg.Origin) + cfg.EVMConfig.Tracer.OnTxStart(vmenv.GetVMContext(), types.NewTx(&types.LegacyTx{To: &address, Data: input, Value: cfg.Value, Gas: cfg.GasLimit}), cfg.Origin) } // Execute the preparatory steps for state transition which includes: // - prepare accessList(post-berlin) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 7caff62dc4..97398ebb58 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -336,7 +336,7 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode b.Fatal(err) } cfg.EVMConfig = vm.Config{ - Tracer: tracer, + Tracer: tracer.Hooks, } } var ( @@ -511,7 +511,7 @@ func TestEip2929Cases(t *testing.T) { code, ops) Execute(code, nil, &Config{ EVMConfig: vm.Config{ - Tracer: logger.NewMarkdownLogger(nil, os.Stdout), + Tracer: logger.NewMarkdownLogger(nil, os.Stdout).Hooks(), ExtraEips: []int{2929}, }, }) @@ -664,7 +664,7 @@ func TestColdAccountAccessCost(t *testing.T) { tracer := logger.NewStructLogger(nil) Execute(tc.code, nil, &Config{ EVMConfig: vm.Config{ - Tracer: tracer, + Tracer: tracer.Hooks(), }, }) have := tracer.StructLogs()[tc.step].GasCost @@ -835,7 +835,7 @@ func TestRuntimeJSTracer(t *testing.T) { GasLimit: 1000000, State: statedb, EVMConfig: vm.Config{ - Tracer: tracer, + Tracer: tracer.Hooks, }}) if err != nil { t.Fatal("didn't expect error", err) @@ -869,7 +869,7 @@ func TestJSTracerCreateTx(t *testing.T) { _, _, _, err = Create(code, &Config{ State: statedb, EVMConfig: vm.Config{ - Tracer: tracer, + Tracer: tracer.Hooks, }}) if err != nil { t.Fatal(err) diff --git a/crypto/bls12381/g2.go b/crypto/bls12381/g2.go index e5fe75af20..b942bf94fd 100644 --- a/crypto/bls12381/g2.go +++ b/crypto/bls12381/g2.go @@ -27,7 +27,7 @@ import ( // If z is equal to one the point is considered as in affine form. type PointG2 [3]fe2 -// Set copies valeus of one point to another. +// Set copies values of one point to another. func (p *PointG2) Set(p2 *PointG2) *PointG2 { p[0].set(&p2[0]) p[1].set(&p2[1]) diff --git a/crypto/bn256/cloudflare/gfp_decl.go b/crypto/bn256/cloudflare/gfp_decl.go index cf7f565423..1954d14a4a 100644 --- a/crypto/bn256/cloudflare/gfp_decl.go +++ b/crypto/bn256/cloudflare/gfp_decl.go @@ -13,7 +13,7 @@ import ( //nolint:varcheck,unused,deadcode var hasBMI2 = cpu.X86.HasBMI2 -// go:noescape +//go:noescape func gfpNeg(c, a *gfP) //go:noescape diff --git a/crypto/bn256/google/bn256.go b/crypto/bn256/google/bn256.go index 0a9d5cd35d..93953e23a9 100644 --- a/crypto/bn256/google/bn256.go +++ b/crypto/bn256/google/bn256.go @@ -166,7 +166,7 @@ type G2 struct { p *twistPoint } -// RandomG1 returns x and gâ‚‚Ë£ where x is a random, non-zero number read from r. +// RandomG2 returns x and gâ‚‚Ë£ where x is a random, non-zero number read from r. func RandomG2(r io.Reader) (*big.Int, *G2, error) { var k *big.Int var err error diff --git a/crypto/kzg4844/kzg4844.go b/crypto/kzg4844/kzg4844.go index 4561ef9de9..39fdfbe740 100644 --- a/crypto/kzg4844/kzg4844.go +++ b/crypto/kzg4844/kzg4844.go @@ -21,21 +21,60 @@ import ( "embed" "errors" "hash" + "reflect" "sync/atomic" + + "github.com/ethereum/go-ethereum/common/hexutil" ) //go:embed trusted_setup.json var content embed.FS +var ( + blobT = reflect.TypeOf(Blob{}) + commitmentT = reflect.TypeOf(Commitment{}) + proofT = reflect.TypeOf(Proof{}) +) + // Blob represents a 4844 data blob. type Blob [131072]byte +// UnmarshalJSON parses a blob in hex syntax. +func (b *Blob) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(blobT, input, b[:]) +} + +// MarshalText returns the hex representation of b. +func (b Blob) MarshalText() ([]byte, error) { + return hexutil.Bytes(b[:]).MarshalText() +} + // Commitment is a serialized commitment to a polynomial. type Commitment [48]byte +// UnmarshalJSON parses a commitment in hex syntax. +func (c *Commitment) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(commitmentT, input, c[:]) +} + +// MarshalText returns the hex representation of c. +func (c Commitment) MarshalText() ([]byte, error) { + return hexutil.Bytes(c[:]).MarshalText() +} + // Proof is a serialized commitment to the quotient polynomial. type Proof [48]byte +// UnmarshalJSON parses a proof in hex syntax. +func (p *Proof) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(proofT, input, p[:]) +} + +// MarshalText returns the hex representation of p. +func (p Proof) MarshalText() ([]byte, error) { + return hexutil.Bytes(p[:]).MarshalText() +} + // Point is a BLS field element. type Point [32]byte @@ -46,7 +85,7 @@ type Claim [32]byte var useCKZG atomic.Bool // UseCKZG can be called to switch the default Go implementation of KZG to the C -// library if fo some reason the user wishes to do so (e.g. consensus bug in one +// library if for some reason the user wishes to do so (e.g. consensus bug in one // or the other). func UseCKZG(use bool) error { if use && !ckzgAvailable { @@ -66,7 +105,7 @@ func UseCKZG(use bool) error { } // BlobToCommitment creates a small commitment out of a data blob. -func BlobToCommitment(blob Blob) (Commitment, error) { +func BlobToCommitment(blob *Blob) (Commitment, error) { if useCKZG.Load() { return ckzgBlobToCommitment(blob) } @@ -75,7 +114,7 @@ func BlobToCommitment(blob Blob) (Commitment, error) { // ComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func ComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func ComputeProof(blob *Blob, point Point) (Proof, Claim, error) { if useCKZG.Load() { return ckzgComputeProof(blob, point) } @@ -95,7 +134,7 @@ func VerifyProof(commitment Commitment, point Point, claim Claim, proof Proof) e // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func ComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func ComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { if useCKZG.Load() { return ckzgComputeBlobProof(blob, commitment) } @@ -103,7 +142,7 @@ func ComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { } // VerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func VerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func VerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { if useCKZG.Load() { return ckzgVerifyBlobProof(blob, commitment, proof) } diff --git a/crypto/kzg4844/kzg4844_ckzg_cgo.go b/crypto/kzg4844/kzg4844_ckzg_cgo.go index 5400285698..11bc451b58 100644 --- a/crypto/kzg4844/kzg4844_ckzg_cgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_cgo.go @@ -61,10 +61,10 @@ func ckzgInit() { } // ckzgBlobToCommitment creates a small commitment out of a data blob. -func ckzgBlobToCommitment(blob Blob) (Commitment, error) { +func ckzgBlobToCommitment(blob *Blob) (Commitment, error) { ckzgIniter.Do(ckzgInit) - commitment, err := ckzg4844.BlobToKZGCommitment((ckzg4844.Blob)(blob)) + commitment, err := ckzg4844.BlobToKZGCommitment((*ckzg4844.Blob)(blob)) if err != nil { return Commitment{}, err } @@ -73,10 +73,10 @@ func ckzgBlobToCommitment(blob Blob) (Commitment, error) { // ckzgComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func ckzgComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func ckzgComputeProof(blob *Blob, point Point) (Proof, Claim, error) { ckzgIniter.Do(ckzgInit) - proof, claim, err := ckzg4844.ComputeKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes32)(point)) + proof, claim, err := ckzg4844.ComputeKZGProof((*ckzg4844.Blob)(blob), (ckzg4844.Bytes32)(point)) if err != nil { return Proof{}, Claim{}, err } @@ -102,10 +102,10 @@ func ckzgVerifyProof(commitment Commitment, point Point, claim Claim, proof Proo // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func ckzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func ckzgComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { ckzgIniter.Do(ckzgInit) - proof, err := ckzg4844.ComputeBlobKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment)) + proof, err := ckzg4844.ComputeBlobKZGProof((*ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment)) if err != nil { return Proof{}, err } @@ -113,10 +113,10 @@ func ckzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { } // ckzgVerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func ckzgVerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func ckzgVerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { ckzgIniter.Do(ckzgInit) - valid, err := ckzg4844.VerifyBlobKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment), (ckzg4844.Bytes48)(proof)) + valid, err := ckzg4844.VerifyBlobKZGProof((*ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment), (ckzg4844.Bytes48)(proof)) if err != nil { return err } diff --git a/crypto/kzg4844/kzg4844_ckzg_nocgo.go b/crypto/kzg4844/kzg4844_ckzg_nocgo.go index ed840c75bb..70a78e80d1 100644 --- a/crypto/kzg4844/kzg4844_ckzg_nocgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_nocgo.go @@ -32,13 +32,13 @@ func ckzgInit() { } // ckzgBlobToCommitment creates a small commitment out of a data blob. -func ckzgBlobToCommitment(blob Blob) (Commitment, error) { +func ckzgBlobToCommitment(blob *Blob) (Commitment, error) { panic("unsupported platform") } // ckzgComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func ckzgComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func ckzgComputeProof(blob *Blob, point Point) (Proof, Claim, error) { panic("unsupported platform") } @@ -52,11 +52,11 @@ func ckzgVerifyProof(commitment Commitment, point Point, claim Claim, proof Proo // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func ckzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func ckzgComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { panic("unsupported platform") } // ckzgVerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func ckzgVerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func ckzgVerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { panic("unsupported platform") } diff --git a/crypto/kzg4844/kzg4844_gokzg.go b/crypto/kzg4844/kzg4844_gokzg.go index 3f03bb5273..b4af9b1671 100644 --- a/crypto/kzg4844/kzg4844_gokzg.go +++ b/crypto/kzg4844/kzg4844_gokzg.go @@ -46,10 +46,10 @@ func gokzgInit() { } // gokzgBlobToCommitment creates a small commitment out of a data blob. -func gokzgBlobToCommitment(blob Blob) (Commitment, error) { +func gokzgBlobToCommitment(blob *Blob) (Commitment, error) { gokzgIniter.Do(gokzgInit) - commitment, err := context.BlobToKZGCommitment((gokzg4844.Blob)(blob), 0) + commitment, err := context.BlobToKZGCommitment((*gokzg4844.Blob)(blob), 0) if err != nil { return Commitment{}, err } @@ -58,10 +58,10 @@ func gokzgBlobToCommitment(blob Blob) (Commitment, error) { // gokzgComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func gokzgComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func gokzgComputeProof(blob *Blob, point Point) (Proof, Claim, error) { gokzgIniter.Do(gokzgInit) - proof, claim, err := context.ComputeKZGProof((gokzg4844.Blob)(blob), (gokzg4844.Scalar)(point), 0) + proof, claim, err := context.ComputeKZGProof((*gokzg4844.Blob)(blob), (gokzg4844.Scalar)(point), 0) if err != nil { return Proof{}, Claim{}, err } @@ -80,10 +80,10 @@ func gokzgVerifyProof(commitment Commitment, point Point, claim Claim, proof Pro // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func gokzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func gokzgComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { gokzgIniter.Do(gokzgInit) - proof, err := context.ComputeBlobKZGProof((gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), 0) + proof, err := context.ComputeBlobKZGProof((*gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), 0) if err != nil { return Proof{}, err } @@ -91,8 +91,8 @@ func gokzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { } // gokzgVerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func gokzgVerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func gokzgVerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { gokzgIniter.Do(gokzgInit) - return context.VerifyBlobKZGProof((gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), (gokzg4844.KZGProof)(proof)) + return context.VerifyBlobKZGProof((*gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), (gokzg4844.KZGProof)(proof)) } diff --git a/crypto/kzg4844/kzg4844_test.go b/crypto/kzg4844/kzg4844_test.go index fae8a7a76e..a6782d4768 100644 --- a/crypto/kzg4844/kzg4844_test.go +++ b/crypto/kzg4844/kzg4844_test.go @@ -36,13 +36,13 @@ func randFieldElement() [32]byte { return gokzg4844.SerializeScalar(r) } -func randBlob() Blob { +func randBlob() *Blob { var blob Blob for i := 0; i < len(blob); i += gokzg4844.SerializedScalarSize { fieldElementBytes := randFieldElement() copy(blob[i:i+gokzg4844.SerializedScalarSize], fieldElementBytes[:]) } - return blob + return &blob } func TestCKZGWithPoint(t *testing.T) { testKZGWithPoint(t, true) } diff --git a/crypto/secp256k1/libsecp256k1/include/secp256k1.h b/crypto/secp256k1/libsecp256k1/include/secp256k1.h index f268e309d0..76af839691 100644 --- a/crypto/secp256k1/libsecp256k1/include/secp256k1.h +++ b/crypto/secp256k1/libsecp256k1/include/secp256k1.h @@ -357,7 +357,7 @@ SECP256K1_API int secp256k1_ecdsa_signature_serialize_compact( /** Verify an ECDSA signature. * * Returns: 1: correct signature - * 0: incorrect or unparseable signature + * 0: incorrect or unparsable signature * Args: ctx: a secp256k1 context object, initialized for verification. * In: sig: the signature being verified (cannot be NULL) * msg32: the 32-byte message hash being verified (cannot be NULL) diff --git a/crypto/secp256k1/libsecp256k1/sage/group_prover.sage b/crypto/secp256k1/libsecp256k1/sage/group_prover.sage index ab580c5b23..68882e9365 100644 --- a/crypto/secp256k1/libsecp256k1/sage/group_prover.sage +++ b/crypto/secp256k1/libsecp256k1/sage/group_prover.sage @@ -17,7 +17,7 @@ # - A constraint describing the requirements of the law, called "require" # * Implementations are transliterated into functions that operate as well on # algebraic input points, and are called once per combination of branches -# exectured. Each execution returns: +# executed. Each execution returns: # - A constraint describing the assumptions this implementation requires # (such as Z1=1), called "assumeFormula" # - A constraint describing the assumptions this specific branch requires, diff --git a/eth/api.go b/eth/api.go deleted file mode 100644 index 44e934fd04..0000000000 --- a/eth/api.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" -) - -// EthereumAPI provides an API to access Ethereum full node-related information. -type EthereumAPI struct { - e *Ethereum -} - -// NewEthereumAPI creates a new Ethereum protocol API for full nodes. -func NewEthereumAPI(e *Ethereum) *EthereumAPI { - return &EthereumAPI{e} -} - -// Etherbase is the address that mining rewards will be sent to. -func (api *EthereumAPI) Etherbase() (common.Address, error) { - return api.e.Etherbase() -} - -// Coinbase is the address that mining rewards will be sent to (alias for Etherbase). -func (api *EthereumAPI) Coinbase() (common.Address, error) { - return api.Etherbase() -} - -// Hashrate returns the POW hashrate. -func (api *EthereumAPI) Hashrate() hexutil.Uint64 { - return hexutil.Uint64(api.e.Miner().Hashrate()) -} - -// Mining returns an indication if this node is currently mining. -func (api *EthereumAPI) Mining() bool { - return api.e.IsMining() -} diff --git a/eth/api_backend.go b/eth/api_backend.go index 6f4cc28b98..a97942599c 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -37,7 +37,6 @@ import ( "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) @@ -67,7 +66,7 @@ func (b *EthAPIBackend) SetHead(number uint64) { func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) { // Pending block is only known by the miner if number == rpc.PendingBlockNumber { - block := b.eth.miner.PendingBlock() + block, _, _ := b.eth.miner.Pending() if block == nil { return nil, errors.New("pending block is not available") } @@ -118,7 +117,7 @@ func (b *EthAPIBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*ty func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) { // Pending block is only known by the miner if number == rpc.PendingBlockNumber { - block := b.eth.miner.PendingBlock() + block, _, _ := b.eth.miner.Pending() if block == nil { return nil, errors.New("pending block is not available") } @@ -182,14 +181,14 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r return nil, errors.New("invalid arguments; neither block nor hash specified") } -func (b *EthAPIBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { - return b.eth.miner.PendingBlockAndReceipts() +func (b *EthAPIBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { + return b.eth.miner.Pending() } func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) { // Pending state is only known by the miner if number == rpc.PendingBlockNumber { - block, state := b.eth.miner.Pending() + block, _, state := b.eth.miner.Pending() if block == nil || state == nil { return nil, nil, errors.New("pending state is not available") } @@ -267,10 +266,6 @@ func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEven return b.eth.BlockChain().SubscribeRemovedLogsEvent(ch) } -func (b *EthAPIBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - return b.eth.miner.SubscribePendingLogs(ch) -} - func (b *EthAPIBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainEvent(ch) } @@ -292,7 +287,7 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) } func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { - pending := b.eth.txPool.Pending(false) + pending := b.eth.txPool.Pending(txpool.PendingFilter{}) var txs types.Transactions for _, batch := range pending { for _, lazy := range batch { @@ -421,14 +416,6 @@ func (b *EthAPIBackend) CurrentHeader() *types.Header { return b.eth.blockchain.CurrentHeader() } -func (b *EthAPIBackend) Miner() *miner.Miner { - return b.eth.Miner() -} - -func (b *EthAPIBackend) StartMining() error { - return b.eth.StartMining() -} - func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { return b.eth.stateAtBlock(ctx, block, reexec, base, readOnly, preferDisk) } diff --git a/eth/api_debug.go b/eth/api_debug.go index 05010a3969..d5e4dda140 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -56,7 +56,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { // If we're dumping the pending state, we need to request // both the pending block as well as the pending state from // the miner and operate on those - _, stateDb := api.eth.miner.Pending() + _, _, stateDb := api.eth.miner.Pending() if stateDb == nil { return state.Dump{}, errors.New("pending state is not available") } @@ -142,7 +142,7 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex // If we're dumping the pending state, we need to request // both the pending block as well as the pending state from // the miner and operate on those - _, stateDb = api.eth.miner.Pending() + _, _, stateDb = api.eth.miner.Pending() if stateDb == nil { return state.Dump{}, errors.New("pending state is not available") } diff --git a/eth/api_debug_test.go b/eth/api_debug_test.go index 3e54e130f5..1d75c4c041 100644 --- a/eth/api_debug_test.go +++ b/eth/api_debug_test.go @@ -27,9 +27,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" "github.com/holiman/uint256" "golang.org/x/exp/slices" ) @@ -63,7 +64,7 @@ func TestAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) + statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &triedb.Config{Preimages: true}) sdb, _ = state.New(types.EmptyRootHash, statedb, nil) addrs = [AccountRangeMaxResults * 2]common.Address{} m = map[common.Address]bool{} @@ -73,7 +74,7 @@ func TestAccountRange(t *testing.T) { hash := common.HexToHash(fmt.Sprintf("%x", i)) addr := common.BytesToAddress(crypto.Keccak256Hash(hash.Bytes()).Bytes()) addrs[i] = addr - sdb.SetBalance(addrs[i], uint256.NewInt(1), state.BalanceChangeUnspecified) + sdb.SetBalance(addrs[i], uint256.NewInt(1), tracing.BalanceChangeUnspecified) if _, ok := m[addr]; ok { t.Fatalf("bad") } else { @@ -160,7 +161,7 @@ func TestStorageRangeAt(t *testing.T) { // Create a state where account 0x010000... has a few storage entries. var ( - db = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) + db = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &triedb.Config{Preimages: true}) sdb, _ = state.New(types.EmptyRootHash, db, nil) addr = common.Address{0x01} keys = []common.Hash{ // hashes of Keys of storage diff --git a/eth/api_miner.go b/eth/api_miner.go index 477531d494..8c96f4c54a 100644 --- a/eth/api_miner.go +++ b/eth/api_miner.go @@ -18,9 +18,7 @@ package eth import ( "math/big" - "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" ) @@ -29,26 +27,11 @@ type MinerAPI struct { e *Ethereum } -// NewMinerAPI create a new MinerAPI instance. +// NewMinerAPI creates a new MinerAPI instance. func NewMinerAPI(e *Ethereum) *MinerAPI { return &MinerAPI{e} } -// Start starts the miner with the given number of threads. If threads is nil, -// the number of workers started is equal to the number of logical CPUs that are -// usable by this process. If mining is already running, this method adjust the -// number of threads allowed to use and updates the minimum price required by the -// transaction pool. -func (api *MinerAPI) Start() error { - return api.e.StartMining() -} - -// Stop terminates the miner, both at the consensus engine level as well as at -// the block creation level. -func (api *MinerAPI) Stop() { - api.e.StopMining() -} - // SetExtra sets the extra data string that is included when this miner mines a block. func (api *MinerAPI) SetExtra(extra string) (bool, error) { if err := api.e.Miner().SetExtra([]byte(extra)); err != nil { @@ -64,6 +47,7 @@ func (api *MinerAPI) SetGasPrice(gasPrice hexutil.Big) bool { api.e.lock.Unlock() api.e.txPool.SetGasTip((*big.Int)(&gasPrice)) + api.e.Miner().SetGasTip((*big.Int)(&gasPrice)) return true } @@ -72,14 +56,3 @@ func (api *MinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool { api.e.Miner().SetGasCeil(uint64(gasLimit)) return true } - -// SetEtherbase sets the etherbase of the miner. -func (api *MinerAPI) SetEtherbase(etherbase common.Address) bool { - api.e.SetEtherbase(etherbase) - return true -} - -// SetRecommitInterval updates the interval for miner sealing work recommitting. -func (api *MinerAPI) SetRecommitInterval(interval int) { - api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond) -} diff --git a/eth/backend.go b/eth/backend.go index 11812245f3..e1f9650a1f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -18,6 +18,7 @@ package eth import ( + "encoding/json" "errors" "fmt" "math/big" @@ -28,8 +29,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/consensus/beacon" - "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" @@ -44,6 +43,7 @@ import ( "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" + "github.com/ethereum/go-ethereum/eth/tracers/directory/live" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -74,7 +74,6 @@ type Ethereum struct { handler *handler ethDialCandidates enode.Iterator snapDialCandidates enode.Iterator - merger *consensus.Merger // DB interfaces chainDb ethdb.Database // Block chain database @@ -89,9 +88,8 @@ type Ethereum struct { APIBackend *EthAPIBackend - miner *miner.Miner - gasPrice *big.Int - etherbase common.Address + miner *miner.Miner + gasPrice *big.Int networkID uint64 netRPCService *ethapi.NetAPI @@ -158,7 +156,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } eth := &Ethereum{ config: config, - merger: consensus.NewMerger(chainDb), chainDb: chainDb, eventMux: stack.EventMux(), accountManager: stack.AccountManager(), @@ -166,7 +163,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { closeBloomHandler: make(chan struct{}), networkID: networkID, gasPrice: config.Miner.GasPrice, - etherbase: config.Miner.Etherbase, bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms), p2pServer: stack.Server(), @@ -192,7 +188,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { var ( vmConfig = vm.Config{ EnablePreimageRecording: config.EnablePreimageRecording, - Tracer: config.VMTracer, } cacheConfig = &core.CacheConfig{ TrieCleanLimit: config.TrieCleanCache, @@ -206,7 +201,13 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { StateScheme: scheme, } ) - + if config.VMTrace != "" { + t, err := live.Directory.New(config.VMTrace, json.RawMessage(config.VMTraceConfig)) + if err != nil { + return nil, fmt.Errorf("Failed to create tracer %s: %v", config.VMTrace, err) + } + vmConfig.Tracer = t + } // Override the chain config with provided settings. var overrides core.ChainOverrides if config.OverrideCancun != nil { @@ -215,7 +216,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if config.OverrideVerkle != nil { overrides.OverrideVerkle = config.OverrideVerkle } - eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TransactionHistory) + // TODO (MariusVanDerWijden) get rid of shouldPreserve in a follow-up PR + shouldPreserve := func(header *types.Header) bool { + return false + } + eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, shouldPreserve, &config.TransactionHistory) if err != nil { return nil, err } @@ -231,17 +236,17 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } legacyPool := legacypool.New(config.TxPool, eth.blockchain) - eth.txPool, err = txpool.New(new(big.Int).SetUint64(config.TxPool.PriceLimit), eth.blockchain, []txpool.SubPool{legacyPool, blobPool}) + eth.txPool, err = txpool.New(config.TxPool.PriceLimit, eth.blockchain, []txpool.SubPool{legacyPool, blobPool}) if err != nil { return nil, err } // Permit the downloader to use the trie cache allowance during fast sync cacheLimit := cacheConfig.TrieCleanLimit + cacheConfig.TrieDirtyLimit + cacheConfig.SnapshotLimit if eth.handler, err = newHandler(&handlerConfig{ + NodeID: eth.p2pServer.Self().ID(), Database: chainDb, Chain: eth.blockchain, TxPool: eth.txPool, - Merger: eth.merger, Network: networkID, Sync: config.SyncMode, BloomCache: uint64(cacheLimit), @@ -251,7 +256,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return nil, err } - eth.miner = miner.New(eth, &config.Miner, eth.blockchain.Config(), eth.EventMux(), eth.engine, eth.isLocalBlock) + eth.miner = miner.New(eth, config.Miner, eth.engine) eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil} @@ -317,9 +322,6 @@ func (s *Ethereum) APIs() []rpc.API { // Append all the local APIs and return return append(apis, []rpc.API{ { - Namespace: "eth", - Service: NewEthereumAPI(s), - }, { Namespace: "miner", Service: NewMinerAPI(s), }, { @@ -342,138 +344,6 @@ func (s *Ethereum) ResetWithGenesisBlock(gb *types.Block) { s.blockchain.ResetWithGenesisBlock(gb) } -func (s *Ethereum) Etherbase() (eb common.Address, err error) { - s.lock.RLock() - etherbase := s.etherbase - s.lock.RUnlock() - - if etherbase != (common.Address{}) { - return etherbase, nil - } - return common.Address{}, errors.New("etherbase must be explicitly specified") -} - -// isLocalBlock checks whether the specified block is mined -// by local miner accounts. -// -// We regard two types of accounts as local miner account: etherbase -// and accounts specified via `txpool.locals` flag. -func (s *Ethereum) isLocalBlock(header *types.Header) bool { - author, err := s.engine.Author(header) - if err != nil { - log.Warn("Failed to retrieve block author", "number", header.Number.Uint64(), "hash", header.Hash(), "err", err) - return false - } - // Check whether the given address is etherbase. - s.lock.RLock() - etherbase := s.etherbase - s.lock.RUnlock() - if author == etherbase { - return true - } - // Check whether the given address is specified by `txpool.local` - // CLI flag. - for _, account := range s.config.TxPool.Locals { - if account == author { - return true - } - } - return false -} - -// shouldPreserve checks whether we should preserve the given block -// during the chain reorg depending on whether the author of block -// is a local account. -func (s *Ethereum) shouldPreserve(header *types.Header) bool { - // The reason we need to disable the self-reorg preserving for clique - // is it can be probable to introduce a deadlock. - // - // e.g. If there are 7 available signers - // - // r1 A - // r2 B - // r3 C - // r4 D - // r5 A [X] F G - // r6 [X] - // - // In the round5, the in-turn signer E is offline, so the worst case - // is A, F and G sign the block of round5 and reject the block of opponents - // and in the round6, the last available signer B is offline, the whole - // network is stuck. - if _, ok := s.engine.(*clique.Clique); ok { - return false - } - return s.isLocalBlock(header) -} - -// SetEtherbase sets the mining reward address. -func (s *Ethereum) SetEtherbase(etherbase common.Address) { - s.lock.Lock() - s.etherbase = etherbase - s.lock.Unlock() - - s.miner.SetEtherbase(etherbase) -} - -// StartMining starts the miner with the given number of CPU threads. If mining -// is already running, this method adjust the number of threads allowed to use -// and updates the minimum price required by the transaction pool. -func (s *Ethereum) StartMining() error { - // If the miner was not running, initialize it - if !s.IsMining() { - // Propagate the initial price point to the transaction pool - s.lock.RLock() - price := s.gasPrice - s.lock.RUnlock() - s.txPool.SetGasTip(price) - - // Configure the local mining address - eb, err := s.Etherbase() - if err != nil { - log.Error("Cannot start mining without etherbase", "err", err) - return fmt.Errorf("etherbase missing: %v", err) - } - var cli *clique.Clique - if c, ok := s.engine.(*clique.Clique); ok { - cli = c - } else if cl, ok := s.engine.(*beacon.Beacon); ok { - if c, ok := cl.InnerEngine().(*clique.Clique); ok { - cli = c - } - } - if cli != nil { - wallet, err := s.accountManager.Find(accounts.Account{Address: eb}) - if wallet == nil || err != nil { - log.Error("Etherbase account unavailable locally", "err", err) - return fmt.Errorf("signer missing: %v", err) - } - cli.Authorize(eb, wallet.SignData) - } - // If mining is started, we can disable the transaction rejection mechanism - // introduced to speed sync times. - s.handler.enableSyncedFeatures() - - go s.miner.Start() - } - return nil -} - -// StopMining terminates the miner, both at the consensus engine level as well as -// at the block creation level. -func (s *Ethereum) StopMining() { - // Update the thread count within the consensus engine - type threaded interface { - SetThreads(threads int) - } - if th, ok := s.engine.(threaded); ok { - th.SetThreads(-1) - } - // Stop the block creating itself - s.miner.Stop() -} - -func (s *Ethereum) IsMining() bool { return s.miner.Mining() } func (s *Ethereum) Miner() *miner.Miner { return s.miner } func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager } @@ -488,11 +358,6 @@ func (s *Ethereum) Synced() bool { return s.handler.synced func (s *Ethereum) SetSynced() { s.handler.enableSyncedFeatures() } func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning } func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } -func (s *Ethereum) Merger() *consensus.Merger { return s.merger } -func (s *Ethereum) SyncMode() downloader.SyncMode { - mode, _ := s.handler.chainSync.modeAndLocalHead() - return mode -} // Protocols returns all the currently configured // network protocols to start. @@ -540,7 +405,6 @@ func (s *Ethereum) Stop() error { s.bloomIndexer.Close() close(s.closeBloomHandler) s.txPool.Close() - s.miner.Close() s.blockchain.Stop() s.engine.Close() @@ -552,3 +416,29 @@ func (s *Ethereum) Stop() error { return nil } + +// SyncMode retrieves the current sync mode, either explicitly set, or derived +// from the chain status. +func (s *Ethereum) SyncMode() downloader.SyncMode { + // If we're in snap sync mode, return that directly + if s.handler.snapSync.Load() { + return downloader.SnapSync + } + // We are probably in full sync, but we might have rewound to before the + // snap sync pivot, check if we should re-enable snap sync. + head := s.blockchain.CurrentBlock() + if pivot := rawdb.ReadLastPivotNumber(s.chainDb); pivot != nil { + if head.Number.Uint64() < *pivot { + return downloader.SnapSync + } + } + // We are in a full sync, but the associated head state is missing. To complete + // the head state, forcefully rerun the snap sync. Note it doesn't mean the + // persistent state is corrupted, just mismatch with the head block. + if !s.blockchain.HasState(head.Root) { + log.Info("Reenabled snap sync as chain is stateless") + return downloader.SnapSync + } + // Nope, we're really full syncing + return downloader.FullSync +} diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index c48a7d0e49..e5781b2c8f 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -30,9 +30,11 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params/forks" "github.com/ethereum/go-ethereum/rpc" ) @@ -88,6 +90,7 @@ var caps = []string{ "engine_newPayloadV3", "engine_getPayloadBodiesByHashV1", "engine_getPayloadBodiesByRangeV1", + "engine_getClientVersionV1", } type ConsensusAPI struct { @@ -187,21 +190,21 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, pa // attributes. It supports both PayloadAttributesV1 and PayloadAttributesV2. func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { + if params.BeaconRoot != nil { + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("unexpected beacon root")) + } switch api.eth.BlockChain().Config().LatestFork(params.Timestamp) { case forks.Paris: if params.Withdrawals != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals before shanghai")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("withdrawals before shanghai")) } case forks.Shanghai: if params.Withdrawals == nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing withdrawals")) } default: return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV2 must only be called with paris and shanghai payloads")) } - if params.BeaconRoot != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("unexpected beacon root")) - } } return api.forkchoiceUpdated(update, params, engine.PayloadV2, false) } @@ -210,15 +213,11 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, pa // in the payload attributes. It supports only PayloadAttributesV3. func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { - // TODO(matt): according to https://github.com/ethereum/execution-apis/pull/498, - // payload attributes that are invalid should return error - // engine.InvalidPayloadAttributes. Once hive updates this, we should update - // on our end. if params.Withdrawals == nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing withdrawals")) } if params.BeaconRoot == nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing beacon root")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing beacon root")) } if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun { return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV3 must only be called for cancun payloads")) @@ -268,12 +267,6 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl finalized := api.remoteBlocks.get(update.FinalizedBlockHash) // Header advertised via a past newPayload request. Start syncing to it. - // Before we do however, make sure any legacy sync in switched off so we - // don't accidentally have 2 cycles running. - if merger := api.eth.Merger(); !merger.TDDReached() { - merger.ReachTTD() - api.eth.Downloader().Cancel() - } context := []interface{}{"number", header.Number, "hash", header.Hash()} if update.FinalizedBlockHash != (common.Hash{}) { if finalized == nil { @@ -335,9 +328,6 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl // If the beacon client also advertised a finalized block, mark the local // chain final and completely in PoS mode. if update.FinalizedBlockHash != (common.Hash{}) { - if merger := api.eth.Merger(); !merger.PoSFinalized() { - merger.FinalizePoS() - } // If the finalized block is not in our canonical tree, something is wrong finalBlock := api.eth.BlockChain().GetBlockByHash(update.FinalizedBlockHash) if finalBlock == nil { @@ -485,7 +475,7 @@ func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.Payl // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) { if api.eth.BlockChain().Config().IsCancun(api.eth.BlockChain().Config().LondonBlock, params.Timestamp) { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use new payload v2 post-shanghai")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use newPayloadV2 post-cancun")) } if api.eth.BlockChain().Config().LatestFork(params.Timestamp) == forks.Shanghai { if params.Withdrawals == nil { @@ -500,7 +490,7 @@ func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.Payl return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil excessBlobGas pre-cancun")) } if params.BlobGasUsed != nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil params.BlobGasUsed pre-cancun")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil blobGasUsed pre-cancun")) } return api.newPayload(params, nil, nil) } @@ -514,14 +504,14 @@ func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHas return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil excessBlobGas post-cancun")) } if params.BlobGasUsed == nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil params.BlobGasUsed post-cancun")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil blobGasUsed post-cancun")) } if versionedHashes == nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil versionedHashes post-cancun")) } if beaconRoot == nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil parentBeaconBlockRoot post-cancun")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil beaconRoot post-cancun")) } if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun { @@ -621,13 +611,6 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe return api.invalid(err, parent.Header()), nil } - // We've accepted a valid payload from the beacon client. Mark the local - // chain transitions to notify other subsystems (e.g. downloader) of the - // behavioral change. - if merger := api.eth.Merger(); !merger.TDDReached() { - merger.ReachTTD() - api.eth.Downloader().Cancel() - } hash := block.Hash() return engine.PayloadStatusV1{Status: engine.VALID, LatestValidHash: &hash}, nil } @@ -785,26 +768,23 @@ func (api *ConsensusAPI) heartbeat() { // If there have been no updates for the past while, warn the user // that the beacon client is probably offline - if api.eth.BlockChain().Config().TerminalTotalDifficultyPassed || api.eth.Merger().TDDReached() { - if time.Since(lastForkchoiceUpdate) <= beaconUpdateConsensusTimeout || time.Since(lastNewPayloadUpdate) <= beaconUpdateConsensusTimeout { - offlineLogged = time.Time{} - continue - } - - if time.Since(offlineLogged) > beaconUpdateWarnFrequency { - if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() { - if lastTransitionUpdate.IsZero() { - log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!") - } else { - log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!") - } - } else { - log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!") - } - offlineLogged = time.Now() - } + if time.Since(lastForkchoiceUpdate) <= beaconUpdateConsensusTimeout || time.Since(lastNewPayloadUpdate) <= beaconUpdateConsensusTimeout { + offlineLogged = time.Time{} continue } + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() { + if lastTransitionUpdate.IsZero() { + log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!") + } else { + log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!") + } + } else { + log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!") + } + offlineLogged = time.Now() + } + continue } } @@ -813,6 +793,23 @@ func (api *ConsensusAPI) ExchangeCapabilities([]string) []string { return caps } +// GetClientVersionV1 exchanges client version data of this node. +func (api *ConsensusAPI) GetClientVersionV1(info engine.ClientVersionV1) []engine.ClientVersionV1 { + log.Trace("Engine API request received", "method", "GetClientVersionV1", "info", info.String()) + commit := make([]byte, 4) + if vcs, ok := version.VCS(); ok { + commit = common.FromHex(vcs.Commit)[0:4] + } + return []engine.ClientVersionV1{ + { + Code: engine.ClientCode, + Name: engine.ClientName, + Version: params.VersionWithMeta, + Commit: hexutil.Encode(commit), + }, + } +} + // GetPayloadBodiesByHashV1 implements engine_getPayloadBodiesByHashV1 which allows for retrieval of a list // of block bodies by the engine api. func (api *ConsensusAPI) GetPayloadBodiesByHashV1(hashes []common.Hash) []*engine.ExecutionPayloadBodyV1 { @@ -859,8 +856,7 @@ func getBody(block *types.Block) *engine.ExecutionPayloadBodyV1 { ) for j, tx := range body.Transactions { - data, _ := tx.MarshalBinary() - txs[j] = hexutil.Bytes(data) + txs[j], _ = tx.MarshalBinary() } // Post-shanghai withdrawals MUST be set to empty slice instead of nil diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index f1d48d0dea..a88996744c 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -71,7 +71,7 @@ func generateMergeChain(n int, merged bool) (*core.Genesis, []*types.Block) { } genesis := &core.Genesis{ Config: &config, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ testAddr: {Balance: testBalance}, params.BeaconRootsStorageAddress: {Balance: common.Big0, Code: common.Hex2Bytes("3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500")}, }, @@ -262,11 +262,8 @@ func TestInvalidPayloadTimestamp(t *testing.T) { {0, true}, {parent.Time, true}, {parent.Time - 1, true}, - - // TODO (MariusVanDerWijden) following tests are currently broken, - // fixed in upcoming merge-kiln-v2 pr - //{parent.Time() + 1, false}, - //{uint64(time.Now().Unix()) + uint64(time.Minute), false}, + {parent.Time + 1, false}, + {uint64(time.Now().Unix()) + uint64(time.Minute), false}, } for i, test := range tests { @@ -450,7 +447,9 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) t.Fatal("can't create node:", err) } - ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} + mcfg := miner.DefaultConfig + mcfg.PendingFeeRecipient = testAddr + ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: mcfg} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) @@ -463,7 +462,6 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) t.Fatal("can't import test blocks:", err) } - ethservice.SetEtherbase(testAddr) ethservice.SetSynced() return n, ethservice } @@ -865,7 +863,6 @@ func TestTrickRemoteBlockCache(t *testing.T) { func TestInvalidBloom(t *testing.T) { genesis, preMergeBlocks := generateMergeChain(10, false) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() commonAncestor := ethservice.BlockChain().CurrentBlock() @@ -1047,7 +1044,6 @@ func TestWithdrawals(t *testing.T) { genesis.Config.ShanghaiTime = &time n, ethservice := startEthService(t, genesis, blocks) - ethservice.Merger().ReachTTD() defer n.Close() api := NewConsensusAPI(ethservice) @@ -1165,7 +1161,6 @@ func TestNilWithdrawals(t *testing.T) { genesis.Config.ShanghaiTime = &time n, ethservice := startEthService(t, genesis, blocks) - ethservice.Merger().ReachTTD() defer n.Close() api := NewConsensusAPI(ethservice) @@ -1592,7 +1587,6 @@ func TestParentBeaconBlockRoot(t *testing.T) { genesis.Config.CancunTime = &time n, ethservice := startEthService(t, genesis, blocks) - ethservice.Merger().ReachTTD() defer n.Close() api := NewConsensusAPI(ethservice) @@ -1663,3 +1657,26 @@ func TestParentBeaconBlockRoot(t *testing.T) { t.Fatalf("incorrect root stored: want %s, got %s", *blockParams.BeaconRoot, root) } } + +// TestGetClientVersion verifies the expected version info is returned. +func TestGetClientVersion(t *testing.T) { + genesis, preMergeBlocks := generateMergeChain(10, false) + n, ethservice := startEthService(t, genesis, preMergeBlocks) + defer n.Close() + + api := NewConsensusAPI(ethservice) + info := engine.ClientVersionV1{ + Code: "TT", + Name: "test", + Version: "1.1.1", + Commit: "0x12345678", + } + infos := api.GetClientVersionV1(info) + if len(infos) != 1 { + t.Fatalf("expected only one returned client version, got %d", len(infos)) + } + info = infos[0] + if info.Code != engine.ClientCode || info.Name != engine.ClientName || info.Version != params.VersionWithMeta { + t.Fatalf("client info does match expected, got %s", info.String()) + } +} diff --git a/eth/catalyst/blsync.go b/eth/catalyst/blsync.go new file mode 100644 index 0000000000..4877cf4c63 --- /dev/null +++ b/eth/catalyst/blsync.go @@ -0,0 +1,88 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" +) + +// Blsync tracks the head of the beacon chain through the beacon light client +// and drives the local node via ConsensusAPI. +type Blsync struct { + engine *ConsensusAPI + client Client + headCh chan types.ChainHeadEvent + headSub event.Subscription + + quitCh chan struct{} +} + +type Client interface { + SubscribeChainHeadEvent(ch chan<- types.ChainHeadEvent) event.Subscription + Start() + Stop() +} + +// NewBlsync creates a new beacon light syncer. +func NewBlsync(client Client, eth *eth.Ethereum) *Blsync { + return &Blsync{ + engine: newConsensusAPIWithoutHeartbeat(eth), + client: client, + headCh: make(chan types.ChainHeadEvent, 16), + quitCh: make(chan struct{}), + } +} + +// Start starts underlying beacon light client and the sync logic for driving +// the local node. +func (b *Blsync) Start() error { + log.Info("Beacon light sync started") + b.headSub = b.client.SubscribeChainHeadEvent(b.headCh) + go b.client.Start() + + for { + select { + case <-b.quitCh: + return nil + case head := <-b.headCh: + if _, err := b.engine.NewPayloadV2(*head.HeadBlock); err != nil { + log.Error("failed to send new payload", "err", err) + continue + } + update := engine.ForkchoiceStateV1{ + HeadBlockHash: head.HeadBlock.BlockHash, + SafeBlockHash: head.Finalized, //TODO pass finalized or empty hash here? + FinalizedBlockHash: head.Finalized, + } + if _, err := b.engine.ForkchoiceUpdatedV1(update, nil); err != nil { + log.Error("failed to send forkchoice updated", "err", err) + continue + } + } + } +} + +// Stop signals to the light client and syncer to exit. +func (b *Blsync) Stop() error { + b.client.Stop() + close(b.quitCh) + return nil +} diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index 5ad50f14c1..4ae60ed490 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -18,6 +18,7 @@ package catalyst import ( "crypto/rand" + "crypto/sha256" "errors" "math/big" "sync" @@ -25,7 +26,9 @@ import ( "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -160,14 +163,14 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u SuggestedFeeRecipient: feeRecipient, Withdrawals: withdrawals, Random: random, - }, engine.PayloadV2, true) + BeaconRoot: &common.Hash{}, + }, engine.PayloadV3, true) if err != nil { return err } if fcResponse == engine.STATUS_SYNCING { return errors.New("chain rewind prevented invocation of payload creation") } - envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID, true) if err != nil { return err @@ -185,8 +188,21 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u } } + // Independently calculate the blob hashes from sidecars. + blobHashes := make([]common.Hash, 0) + if envelope.BlobsBundle != nil { + hasher := sha256.New() + for _, commit := range envelope.BlobsBundle.Commitments { + var c kzg4844.Commitment + if len(commit) != len(c) { + return errors.New("invalid commitment length") + } + copy(c[:], commit) + blobHashes = append(blobHashes, kzg4844.CalcBlobHashV1(hasher, &c)) + } + } // Mark the payload as canon - if _, err = c.engineAPI.NewPayloadV2(*payload); err != nil { + if _, err = c.engineAPI.NewPayloadV3(*payload, blobHashes, &common.Hash{}); err != nil { return err } c.setCurrentState(payload.BlockHash, finalizedHash) @@ -263,7 +279,7 @@ func (c *SimulatedBeacon) Rollback() { // Fork sets the head to the provided hash. func (c *SimulatedBeacon) Fork(parentHash common.Hash) error { - if len(c.eth.TxPool().Pending(false)) != 0 { + if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { return errors.New("pending block dirty") } parent := c.eth.BlockChain().GetBlockByHash(parentHash) @@ -275,7 +291,7 @@ func (c *SimulatedBeacon) Fork(parentHash common.Hash) error { // AdjustTime creates a new block with an adjusted timestamp. func (c *SimulatedBeacon) AdjustTime(adjustment time.Duration) error { - if len(c.eth.TxPool().Pending(false)) != 0 { + if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { return errors.New("could not adjust time on non-empty block") } parent := c.eth.BlockChain().CurrentBlock() diff --git a/eth/catalyst/simulated_beacon_test.go b/eth/catalyst/simulated_beacon_test.go index 6fa97ad87a..df682b49d9 100644 --- a/eth/catalyst/simulated_beacon_test.go +++ b/eth/catalyst/simulated_beacon_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" @@ -48,7 +49,7 @@ func startSimulatedBeaconEthService(t *testing.T, genesis *core.Genesis) (*node. t.Fatal("can't create node:", err) } - ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} + ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: miner.DefaultConfig} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) diff --git a/eth/downloader/api.go b/eth/downloader/api.go index f09122904c..90c36afbb5 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -38,7 +38,7 @@ type DownloaderAPI struct { uninstallSyncSubscription chan *uninstallSyncSubscriptionRequest } -// NewDownloaderAPI create a new DownloaderAPI. The API has an internal event loop that +// NewDownloaderAPI creates a new DownloaderAPI. The API has an internal event loop that // listens for events from the downloader through the global event mux. In case it receives one of // these events it broadcasts it to all syncing subscriptions that are installed through the // installSyncSubscription channel. @@ -149,8 +149,6 @@ func (api *DownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error notifier.Notify(rpcSub.ID, status) case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index df8af68bc7..d3f75c8527 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -50,7 +50,8 @@ func newBeaconBackfiller(dl *Downloader, success func()) backfiller { } // suspend cancels any background downloader threads and returns the last header -// that has been successfully backfilled. +// that has been successfully backfilled (potentially in a previous run), or the +// genesis. func (b *beaconBackfiller) suspend() *types.Header { // If no filling is running, don't waste cycles b.lock.Lock() diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index f1cfa92d5d..6e7c5dcf02 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -35,7 +35,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) var ( @@ -212,7 +212,7 @@ type BlockChain interface { // TrieDB retrieves the low level trie database used for interacting // with trie nodes. - TrieDB() *trie.Database + TrieDB() *triedb.Database } // New creates a new downloader to fetch hashes and blocks from remote peers. @@ -611,6 +611,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td, ttd * if err := d.lightchain.SetHead(origin); err != nil { return err } + log.Info("Truncated excess ancient chain segment", "oldhead", frozen-1, "newhead", origin) } } // Initiate the sync using a concurrent header and content retrieval algorithm diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index e4875b959a..2468e1a980 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -69,7 +69,7 @@ func newTesterWithNotification(t *testing.T, success func()) *downloadTester { }) gspec := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + Alloc: types.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } chain, err := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) @@ -440,9 +440,6 @@ func assertOwnChain(t *testing.T, tester *downloadTester, length int) { func TestCanonicalSynchronisation68Full(t *testing.T) { testCanonSync(t, eth.ETH68, FullSync) } func TestCanonicalSynchronisation68Snap(t *testing.T) { testCanonSync(t, eth.ETH68, SnapSync) } func TestCanonicalSynchronisation68Light(t *testing.T) { testCanonSync(t, eth.ETH68, LightSync) } -func TestCanonicalSynchronisation67Full(t *testing.T) { testCanonSync(t, eth.ETH67, FullSync) } -func TestCanonicalSynchronisation67Snap(t *testing.T) { testCanonSync(t, eth.ETH67, SnapSync) } -func TestCanonicalSynchronisation67Light(t *testing.T) { testCanonSync(t, eth.ETH67, LightSync) } func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -463,8 +460,6 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // until the cached blocks are retrieved. func TestThrottling68Full(t *testing.T) { testThrottling(t, eth.ETH68, FullSync) } func TestThrottling68Snap(t *testing.T) { testThrottling(t, eth.ETH68, SnapSync) } -func TestThrottling67Full(t *testing.T) { testThrottling(t, eth.ETH67, FullSync) } -func TestThrottling67Snap(t *testing.T) { testThrottling(t, eth.ETH67, SnapSync) } func testThrottling(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -546,9 +541,6 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { func TestForkedSync68Full(t *testing.T) { testForkedSync(t, eth.ETH68, FullSync) } func TestForkedSync68Snap(t *testing.T) { testForkedSync(t, eth.ETH68, SnapSync) } func TestForkedSync68Light(t *testing.T) { testForkedSync(t, eth.ETH68, LightSync) } -func TestForkedSync67Full(t *testing.T) { testForkedSync(t, eth.ETH67, FullSync) } -func TestForkedSync67Snap(t *testing.T) { testForkedSync(t, eth.ETH67, SnapSync) } -func TestForkedSync67Light(t *testing.T) { testForkedSync(t, eth.ETH67, LightSync) } func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -576,9 +568,6 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestHeavyForkedSync68Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH68, FullSync) } func TestHeavyForkedSync68Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH68, SnapSync) } func TestHeavyForkedSync68Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH68, LightSync) } -func TestHeavyForkedSync67Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, FullSync) } -func TestHeavyForkedSync67Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, SnapSync) } -func TestHeavyForkedSync67Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, LightSync) } func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -608,9 +597,6 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestBoundedForkedSync68Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH68, FullSync) } func TestBoundedForkedSync68Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH68, SnapSync) } func TestBoundedForkedSync68Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH68, LightSync) } -func TestBoundedForkedSync67Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, FullSync) } -func TestBoundedForkedSync67Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, SnapSync) } -func TestBoundedForkedSync67Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, LightSync) } func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -645,15 +631,6 @@ func TestBoundedHeavyForkedSync68Snap(t *testing.T) { func TestBoundedHeavyForkedSync68Light(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH68, LightSync) } -func TestBoundedHeavyForkedSync67Full(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH67, FullSync) -} -func TestBoundedHeavyForkedSync67Snap(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH67, SnapSync) -} -func TestBoundedHeavyForkedSync67Light(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH67, LightSync) -} func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -681,9 +658,6 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestCancel68Full(t *testing.T) { testCancel(t, eth.ETH68, FullSync) } func TestCancel68Snap(t *testing.T) { testCancel(t, eth.ETH68, SnapSync) } func TestCancel68Light(t *testing.T) { testCancel(t, eth.ETH68, LightSync) } -func TestCancel67Full(t *testing.T) { testCancel(t, eth.ETH67, FullSync) } -func TestCancel67Snap(t *testing.T) { testCancel(t, eth.ETH67, SnapSync) } -func TestCancel67Light(t *testing.T) { testCancel(t, eth.ETH67, LightSync) } func testCancel(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -711,9 +685,6 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { func TestMultiSynchronisation68Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH68, FullSync) } func TestMultiSynchronisation68Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH68, SnapSync) } func TestMultiSynchronisation68Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH68, LightSync) } -func TestMultiSynchronisation67Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, FullSync) } -func TestMultiSynchronisation67Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, SnapSync) } -func TestMultiSynchronisation67Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, LightSync) } func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -738,9 +709,6 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { func TestMultiProtoSynchronisation68Full(t *testing.T) { testMultiProtoSync(t, eth.ETH68, FullSync) } func TestMultiProtoSynchronisation68Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH68, SnapSync) } func TestMultiProtoSynchronisation68Light(t *testing.T) { testMultiProtoSync(t, eth.ETH68, LightSync) } -func TestMultiProtoSynchronisation67Full(t *testing.T) { testMultiProtoSync(t, eth.ETH67, FullSync) } -func TestMultiProtoSynchronisation67Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH67, SnapSync) } -func TestMultiProtoSynchronisation67Light(t *testing.T) { testMultiProtoSync(t, eth.ETH67, LightSync) } func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -751,7 +719,6 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Create peers of every type tester.newPeer("peer 68", eth.ETH68, chain.blocks[1:]) - tester.newPeer("peer 67", eth.ETH67, chain.blocks[1:]) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -760,7 +727,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, len(chain.blocks)) // Check that no peers have been dropped off - for _, version := range []int{68, 67} { + for _, version := range []int{68} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -773,9 +740,6 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { func TestEmptyShortCircuit68Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, FullSync) } func TestEmptyShortCircuit68Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, SnapSync) } func TestEmptyShortCircuit68Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, LightSync) } -func TestEmptyShortCircuit67Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, FullSync) } -func TestEmptyShortCircuit67Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, SnapSync) } -func TestEmptyShortCircuit67Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, LightSync) } func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -824,9 +788,6 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { func TestMissingHeaderAttack68Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH68, FullSync) } func TestMissingHeaderAttack68Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH68, SnapSync) } func TestMissingHeaderAttack68Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH68, LightSync) } -func TestMissingHeaderAttack67Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, FullSync) } -func TestMissingHeaderAttack67Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, SnapSync) } -func TestMissingHeaderAttack67Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, LightSync) } func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -853,9 +814,6 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { func TestShiftedHeaderAttack68Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH68, FullSync) } func TestShiftedHeaderAttack68Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH68, SnapSync) } func TestShiftedHeaderAttack68Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH68, LightSync) } -func TestShiftedHeaderAttack67Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, FullSync) } -func TestShiftedHeaderAttack67Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, SnapSync) } -func TestShiftedHeaderAttack67Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, LightSync) } func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -889,15 +847,6 @@ func TestHighTDStarvationAttack68Snap(t *testing.T) { func TestHighTDStarvationAttack68Light(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH68, LightSync) } -func TestHighTDStarvationAttack67Full(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH67, FullSync) -} -func TestHighTDStarvationAttack67Snap(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH67, SnapSync) -} -func TestHighTDStarvationAttack67Light(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH67, LightSync) -} func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -912,7 +861,6 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that misbehaving peers are disconnected, whilst behaving ones are not. func TestBlockHeaderAttackerDropping68(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH68) } -func TestBlockHeaderAttackerDropping67(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH67) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Define the disconnection requirement for individual hash fetch errors @@ -963,9 +911,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { func TestSyncProgress68Full(t *testing.T) { testSyncProgress(t, eth.ETH68, FullSync) } func TestSyncProgress68Snap(t *testing.T) { testSyncProgress(t, eth.ETH68, SnapSync) } func TestSyncProgress68Light(t *testing.T) { testSyncProgress(t, eth.ETH68, LightSync) } -func TestSyncProgress67Full(t *testing.T) { testSyncProgress(t, eth.ETH67, FullSync) } -func TestSyncProgress67Snap(t *testing.T) { testSyncProgress(t, eth.ETH67, SnapSync) } -func TestSyncProgress67Light(t *testing.T) { testSyncProgress(t, eth.ETH67, LightSync) } func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1043,9 +988,6 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync func TestForkedSyncProgress68Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH68, FullSync) } func TestForkedSyncProgress68Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH68, SnapSync) } func TestForkedSyncProgress68Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH68, LightSync) } -func TestForkedSyncProgress67Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, FullSync) } -func TestForkedSyncProgress67Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, SnapSync) } -func TestForkedSyncProgress67Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, LightSync) } func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1117,9 +1059,6 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFailedSyncProgress68Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH68, FullSync) } func TestFailedSyncProgress68Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH68, SnapSync) } func TestFailedSyncProgress68Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH68, LightSync) } -func TestFailedSyncProgress67Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, FullSync) } -func TestFailedSyncProgress67Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, SnapSync) } -func TestFailedSyncProgress67Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, LightSync) } func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1186,9 +1125,6 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFakedSyncProgress68Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH68, FullSync) } func TestFakedSyncProgress68Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH68, SnapSync) } func TestFakedSyncProgress68Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH68, LightSync) } -func TestFakedSyncProgress67Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, FullSync) } -func TestFakedSyncProgress67Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, SnapSync) } -func TestFakedSyncProgress67Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, LightSync) } func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1332,8 +1268,6 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // being fast-synced from, avoiding potential cheap eclipse attacks. func TestBeaconSync68Full(t *testing.T) { testBeaconSync(t, eth.ETH68, FullSync) } func TestBeaconSync68Snap(t *testing.T) { testBeaconSync(t, eth.ETH68, SnapSync) } -func TestBeaconSync67Full(t *testing.T) { testBeaconSync(t, eth.ETH67, FullSync) } -func TestBeaconSync67Snap(t *testing.T) { testBeaconSync(t, eth.ETH67, SnapSync) } func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) { //log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index f40ca24d99..873ee950b6 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -161,7 +161,7 @@ type backfiller interface { // on initial startup. // // The method should return the last block header that has been successfully - // backfilled, or nil if the backfiller was not resumed. + // backfilled (in the current or a previous run), falling back to the genesis. suspend() *types.Header // resume requests the backfiller to start running fill or snap sync based on @@ -382,14 +382,17 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { done := make(chan struct{}) go func() { defer close(done) - if filled := s.filler.suspend(); filled != nil { - // If something was filled, try to delete stale sync helpers. If - // unsuccessful, warn the user, but not much else we can do (it's - // a programming error, just let users report an issue and don't - // choke in the meantime). - if err := s.cleanStales(filled); err != nil { - log.Error("Failed to clean stale beacon headers", "err", err) - } + filled := s.filler.suspend() + if filled == nil { + log.Error("Latest filled block is not available") + return + } + // If something was filled, try to delete stale sync helpers. If + // unsuccessful, warn the user, but not much else we can do (it's + // a programming error, just let users report an issue and don't + // choke in the meantime). + if err := s.cleanStales(filled); err != nil { + log.Error("Failed to clean stale beacon headers", "err", err) } }() // Wait for the suspend to finish, consuming head events in the meantime @@ -1120,33 +1123,46 @@ func (s *skeleton) cleanStales(filled *types.Header) error { number := filled.Number.Uint64() log.Trace("Cleaning stale beacon headers", "filled", number, "hash", filled.Hash()) - // If the filled header is below the linked subchain, something's - // corrupted internally. Report and error and refuse to do anything. - if number < s.progress.Subchains[0].Tail { + // If the filled header is below the linked subchain, something's corrupted + // internally. Report and error and refuse to do anything. + if number+1 < s.progress.Subchains[0].Tail { return fmt.Errorf("filled header below beacon header tail: %d < %d", number, s.progress.Subchains[0].Tail) } - // Subchain seems trimmable, push the tail forward up to the last - // filled header and delete everything before it - if available. In - // case we filled past the head, recreate the subchain with a new - // head to keep it consistent with the data on disk. + // If nothing in subchain is filled, don't bother to do cleanup. + if number+1 == s.progress.Subchains[0].Tail { + return nil + } var ( - start = s.progress.Subchains[0].Tail // start deleting from the first known header - end = number // delete until the requested threshold + start uint64 + end uint64 batch = s.db.NewBatch() ) - s.progress.Subchains[0].Tail = number - s.progress.Subchains[0].Next = filled.ParentHash + if number < s.progress.Subchains[0].Head { + // The skeleton chain is partially consumed, set the new tail as filled+1. + tail := rawdb.ReadSkeletonHeader(s.db, number+1) + if tail.ParentHash != filled.Hash() { + return fmt.Errorf("filled header is discontinuous with subchain: %d %s, please file an issue", number, filled.Hash()) + } + start, end = s.progress.Subchains[0].Tail, number+1 // remove headers in [tail, filled] + s.progress.Subchains[0].Tail = tail.Number.Uint64() + s.progress.Subchains[0].Next = tail.ParentHash + } else { + // The skeleton chain is fully consumed, set both head and tail as filled. + start, end = s.progress.Subchains[0].Tail, filled.Number.Uint64() // remove headers in [tail, filled) + s.progress.Subchains[0].Tail = filled.Number.Uint64() + s.progress.Subchains[0].Next = filled.ParentHash - if s.progress.Subchains[0].Head < number { - // If more headers were filled than available, push the entire - // subchain forward to keep tracking the node's block imports - end = s.progress.Subchains[0].Head + 1 // delete the entire original range, including the head - s.progress.Subchains[0].Head = number // assign a new head (tail is already assigned to this) + // If more headers were filled than available, push the entire subchain + // forward to keep tracking the node's block imports. + if number > s.progress.Subchains[0].Head { + end = s.progress.Subchains[0].Head + 1 // delete the entire original range, including the head + s.progress.Subchains[0].Head = number // assign a new head (tail is already assigned to this) - // The entire original skeleton chain was deleted and a new one - // defined. Make sure the new single-header chain gets pushed to - // disk to keep internal state consistent. - rawdb.WriteSkeletonHeader(batch, filled) + // The entire original skeleton chain was deleted and a new one + // defined. Make sure the new single-header chain gets pushed to + // disk to keep internal state consistent. + rawdb.WriteSkeletonHeader(batch, filled) + } } // Execute the trimming and the potential rewiring of the progress s.saveSyncStatus(batch) diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index aceadd00d3..2b108dfe93 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -811,7 +811,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { // Create a peer set to feed headers through peerset := newPeerSet() for _, peer := range tt.peers { - peerset.Register(newPeerConnection(peer.id, eth.ETH67, peer, log.New("id", peer.id))) + peerset.Register(newPeerConnection(peer.id, eth.ETH68, peer, log.New("id", peer.id))) } // Create a peer dropper to track malicious peers dropped := make(map[string]int) @@ -913,7 +913,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { skeleton.Sync(tt.newHead, nil, true) } if tt.newPeer != nil { - if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH67, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil { + if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH68, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil { t.Errorf("test %d: failed to register new peer: %v", i, err) } } diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go index 1bf03411d1..46f3febd8b 100644 --- a/eth/downloader/testchain_test.go +++ b/eth/downloader/testchain_test.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) // Test chain parameters. @@ -41,10 +41,10 @@ var ( testGspec = &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + Alloc: types.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - testGenesis = testGspec.MustCommit(testDB, trie.NewDatabase(testDB, trie.HashDefaults)) + testGenesis = testGspec.MustCommit(testDB, triedb.NewDatabase(testDB, triedb.HashDefaults)) ) // The common prefix of all test chains: diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index d8680c5444..fef7f29f4e 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" - "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" @@ -143,7 +142,8 @@ type Config struct { EnablePreimageRecording bool // Enables VM tracing - VMTracer vm.EVMLogger + VMTrace string + VMTraceConfig string // Miscellaneous options DocRoot string `toml:"-"` @@ -169,15 +169,14 @@ type Config struct { // Clique is allowed for now to live standalone, but ethash is forbidden and can // only exist on already merged networks. func CreateConsensusEngine(config *params.ChainConfig, db ethdb.Database) (consensus.Engine, error) { - // If proof-of-authority is requested, set it up + // Geth v1.14.0 dropped support for non-merged networks in any consensus + // mode. If such a network is requested, reject startup. + if !config.TerminalTotalDifficultyPassed { + return nil, errors.New("only PoS networks are supported, please transition old ones with Geth v1.13.x") + } + // Wrap previously supported consensus engines into their post-merge counterpart if config.Clique != nil { return beacon.New(clique.New(config.Clique, db)), nil } - // If defaulting to proof-of-work, enforce an already merged network since - // we cannot run PoW algorithms anymore, so we cannot even follow a chain - // not coordinated by a beacon node. - if !config.TerminalTotalDifficultyPassed { - return nil, errors.New("ethash is only supported as a historical component of already merged networks") - } return beacon.New(ethash.NewFaker()), nil } diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go deleted file mode 100644 index 126eaaea7f..0000000000 --- a/eth/fetcher/block_fetcher.go +++ /dev/null @@ -1,939 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package fetcher contains the announcement based header, blocks or transaction synchronisation. -package fetcher - -import ( - "errors" - "math/rand" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/prque" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/trie" -) - -const ( - lightTimeout = time.Millisecond // Time allowance before an announced header is explicitly requested - arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block/transaction is explicitly requested - gatherSlack = 100 * time.Millisecond // Interval used to collate almost-expired announces with fetches - fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block/transaction -) - -const ( - maxUncleDist = 7 // Maximum allowed backward distance from the chain head - maxQueueDist = 32 // Maximum allowed distance from the chain head to queue - hashLimit = 256 // Maximum number of unique blocks or headers a peer may have announced - blockLimit = 64 // Maximum number of unique blocks a peer may have delivered -) - -var ( - blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/in", nil) - blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/announces/out", nil) - blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/drop", nil) - blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/dos", nil) - - blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/in", nil) - blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/broadcasts/out", nil) - blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/drop", nil) - blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/dos", nil) - - headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/headers", nil) - bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/bodies", nil) - - headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/in", nil) - headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/out", nil) - bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/in", nil) - bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/out", nil) -) - -var errTerminated = errors.New("terminated") - -// HeaderRetrievalFn is a callback type for retrieving a header from the local chain. -type HeaderRetrievalFn func(common.Hash) *types.Header - -// blockRetrievalFn is a callback type for retrieving a block from the local chain. -type blockRetrievalFn func(common.Hash) *types.Block - -// headerRequesterFn is a callback type for sending a header retrieval request. -type headerRequesterFn func(common.Hash, chan *eth.Response) (*eth.Request, error) - -// bodyRequesterFn is a callback type for sending a body retrieval request. -type bodyRequesterFn func([]common.Hash, chan *eth.Response) (*eth.Request, error) - -// headerVerifierFn is a callback type to verify a block's header for fast propagation. -type headerVerifierFn func(header *types.Header) error - -// blockBroadcasterFn is a callback type for broadcasting a block to connected peers. -type blockBroadcasterFn func(block *types.Block, propagate bool) - -// chainHeightFn is a callback type to retrieve the current chain height. -type chainHeightFn func() uint64 - -// headersInsertFn is a callback type to insert a batch of headers into the local chain. -type headersInsertFn func(headers []*types.Header) (int, error) - -// chainInsertFn is a callback type to insert a batch of blocks into the local chain. -type chainInsertFn func(types.Blocks) (int, error) - -// peerDropFn is a callback type for dropping a peer detected as malicious. -type peerDropFn func(id string) - -// blockAnnounce is the hash notification of the availability of a new block in the -// network. -type blockAnnounce struct { - hash common.Hash // Hash of the block being announced - number uint64 // Number of the block being announced (0 = unknown | old protocol) - header *types.Header // Header of the block partially reassembled (new protocol) - time time.Time // Timestamp of the announcement - - origin string // Identifier of the peer originating the notification - - fetchHeader headerRequesterFn // Fetcher function to retrieve the header of an announced block - fetchBodies bodyRequesterFn // Fetcher function to retrieve the body of an announced block -} - -// headerFilterTask represents a batch of headers needing fetcher filtering. -type headerFilterTask struct { - peer string // The source peer of block headers - headers []*types.Header // Collection of headers to filter - time time.Time // Arrival time of the headers -} - -// bodyFilterTask represents a batch of block bodies (transactions and uncles) -// needing fetcher filtering. -type bodyFilterTask struct { - peer string // The source peer of block bodies - transactions [][]*types.Transaction // Collection of transactions per block bodies - uncles [][]*types.Header // Collection of uncles per block bodies - time time.Time // Arrival time of the blocks' contents -} - -// blockOrHeaderInject represents a schedules import operation. -type blockOrHeaderInject struct { - origin string - - header *types.Header // Used for light mode fetcher which only cares about header. - block *types.Block // Used for normal mode fetcher which imports full block. -} - -// number returns the block number of the injected object. -func (inject *blockOrHeaderInject) number() uint64 { - if inject.header != nil { - return inject.header.Number.Uint64() - } - return inject.block.NumberU64() -} - -// number returns the block hash of the injected object. -func (inject *blockOrHeaderInject) hash() common.Hash { - if inject.header != nil { - return inject.header.Hash() - } - return inject.block.Hash() -} - -// BlockFetcher is responsible for accumulating block announcements from various peers -// and scheduling them for retrieval. -type BlockFetcher struct { - light bool // The indicator whether it's a light fetcher or normal one. - - // Various event channels - notify chan *blockAnnounce - inject chan *blockOrHeaderInject - - headerFilter chan chan *headerFilterTask - bodyFilter chan chan *bodyFilterTask - - done chan common.Hash - quit chan struct{} - - // Announce states - announces map[string]int // Per peer blockAnnounce counts to prevent memory exhaustion - announced map[common.Hash][]*blockAnnounce // Announced blocks, scheduled for fetching - fetching map[common.Hash]*blockAnnounce // Announced blocks, currently fetching - fetched map[common.Hash][]*blockAnnounce // Blocks with headers fetched, scheduled for body retrieval - completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing - - // Block cache - queue *prque.Prque[int64, *blockOrHeaderInject] // Queue containing the import operations (block number sorted) - queues map[string]int // Per peer block counts to prevent memory exhaustion - queued map[common.Hash]*blockOrHeaderInject // Set of already queued blocks (to dedup imports) - - // Callbacks - getHeader HeaderRetrievalFn // Retrieves a header from the local chain - getBlock blockRetrievalFn // Retrieves a block from the local chain - verifyHeader headerVerifierFn // Checks if a block's headers have a valid proof of work - broadcastBlock blockBroadcasterFn // Broadcasts a block to connected peers - chainHeight chainHeightFn // Retrieves the current chain's height - insertHeaders headersInsertFn // Injects a batch of headers into the chain - insertChain chainInsertFn // Injects a batch of blocks into the chain - dropPeer peerDropFn // Drops a peer for misbehaving - - // Testing hooks - announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the blockAnnounce list - queueChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a block from the import queue - fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch - completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62) - importedHook func(*types.Header, *types.Block) // Method to call upon successful header or block import (both eth/61 and eth/62) -} - -// NewBlockFetcher creates a block fetcher to retrieve blocks based on hash announcements. -func NewBlockFetcher(light bool, getHeader HeaderRetrievalFn, getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertHeaders headersInsertFn, insertChain chainInsertFn, dropPeer peerDropFn) *BlockFetcher { - return &BlockFetcher{ - light: light, - notify: make(chan *blockAnnounce), - inject: make(chan *blockOrHeaderInject), - headerFilter: make(chan chan *headerFilterTask), - bodyFilter: make(chan chan *bodyFilterTask), - done: make(chan common.Hash), - quit: make(chan struct{}), - announces: make(map[string]int), - announced: make(map[common.Hash][]*blockAnnounce), - fetching: make(map[common.Hash]*blockAnnounce), - fetched: make(map[common.Hash][]*blockAnnounce), - completing: make(map[common.Hash]*blockAnnounce), - queue: prque.New[int64, *blockOrHeaderInject](nil), - queues: make(map[string]int), - queued: make(map[common.Hash]*blockOrHeaderInject), - getHeader: getHeader, - getBlock: getBlock, - verifyHeader: verifyHeader, - broadcastBlock: broadcastBlock, - chainHeight: chainHeight, - insertHeaders: insertHeaders, - insertChain: insertChain, - dropPeer: dropPeer, - } -} - -// Start boots up the announcement based synchroniser, accepting and processing -// hash notifications and block fetches until termination requested. -func (f *BlockFetcher) Start() { - go f.loop() -} - -// Stop terminates the announcement based synchroniser, canceling all pending -// operations. -func (f *BlockFetcher) Stop() { - close(f.quit) -} - -// Notify announces the fetcher of the potential availability of a new block in -// the network. -func (f *BlockFetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, - headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error { - block := &blockAnnounce{ - hash: hash, - number: number, - time: time, - origin: peer, - fetchHeader: headerFetcher, - fetchBodies: bodyFetcher, - } - select { - case f.notify <- block: - return nil - case <-f.quit: - return errTerminated - } -} - -// Enqueue tries to fill gaps the fetcher's future import queue. -func (f *BlockFetcher) Enqueue(peer string, block *types.Block) error { - op := &blockOrHeaderInject{ - origin: peer, - block: block, - } - select { - case f.inject <- op: - return nil - case <-f.quit: - return errTerminated - } -} - -// FilterHeaders extracts all the headers that were explicitly requested by the fetcher, -// returning those that should be handled differently. -func (f *BlockFetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header { - log.Trace("Filtering headers", "peer", peer, "headers", len(headers)) - - // Send the filter channel to the fetcher - filter := make(chan *headerFilterTask) - - select { - case f.headerFilter <- filter: - case <-f.quit: - return nil - } - // Request the filtering of the header list - select { - case filter <- &headerFilterTask{peer: peer, headers: headers, time: time}: - case <-f.quit: - return nil - } - // Retrieve the headers remaining after filtering - select { - case task := <-filter: - return task.headers - case <-f.quit: - return nil - } -} - -// FilterBodies extracts all the block bodies that were explicitly requested by -// the fetcher, returning those that should be handled differently. -func (f *BlockFetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) { - log.Trace("Filtering bodies", "peer", peer, "txs", len(transactions), "uncles", len(uncles)) - - // Send the filter channel to the fetcher - filter := make(chan *bodyFilterTask) - - select { - case f.bodyFilter <- filter: - case <-f.quit: - return nil, nil - } - // Request the filtering of the body list - select { - case filter <- &bodyFilterTask{peer: peer, transactions: transactions, uncles: uncles, time: time}: - case <-f.quit: - return nil, nil - } - // Retrieve the bodies remaining after filtering - select { - case task := <-filter: - return task.transactions, task.uncles - case <-f.quit: - return nil, nil - } -} - -// Loop is the main fetcher loop, checking and processing various notification -// events. -func (f *BlockFetcher) loop() { - // Iterate the block fetching until a quit is requested - var ( - fetchTimer = time.NewTimer(0) - completeTimer = time.NewTimer(0) - ) - <-fetchTimer.C // clear out the channel - <-completeTimer.C - defer fetchTimer.Stop() - defer completeTimer.Stop() - - for { - // Clean up any expired block fetches - for hash, announce := range f.fetching { - if time.Since(announce.time) > fetchTimeout { - f.forgetHash(hash) - } - } - // Import any queued blocks that could potentially fit - height := f.chainHeight() - for !f.queue.Empty() { - op := f.queue.PopItem() - hash := op.hash() - if f.queueChangeHook != nil { - f.queueChangeHook(hash, false) - } - // If too high up the chain or phase, continue later - number := op.number() - if number > height+1 { - f.queue.Push(op, -int64(number)) - if f.queueChangeHook != nil { - f.queueChangeHook(hash, true) - } - break - } - // Otherwise if fresh and still unknown, try and import - if (number+maxUncleDist < height) || (f.light && f.getHeader(hash) != nil) || (!f.light && f.getBlock(hash) != nil) { - f.forgetBlock(hash) - continue - } - if f.light { - f.importHeaders(op.origin, op.header) - } else { - f.importBlocks(op.origin, op.block) - } - } - // Wait for an outside event to occur - select { - case <-f.quit: - // BlockFetcher terminating, abort all operations - return - - case notification := <-f.notify: - // A block was announced, make sure the peer isn't DOSing us - blockAnnounceInMeter.Mark(1) - - count := f.announces[notification.origin] + 1 - if count > hashLimit { - log.Debug("Peer exceeded outstanding announces", "peer", notification.origin, "limit", hashLimit) - blockAnnounceDOSMeter.Mark(1) - break - } - if notification.number == 0 { - break - } - // If we have a valid block number, check that it's potentially useful - if dist := int64(notification.number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { - log.Debug("Peer discarded announcement", "peer", notification.origin, "number", notification.number, "hash", notification.hash, "distance", dist) - blockAnnounceDropMeter.Mark(1) - break - } - // All is well, schedule the announce if block's not yet downloading - if _, ok := f.fetching[notification.hash]; ok { - break - } - if _, ok := f.completing[notification.hash]; ok { - break - } - f.announces[notification.origin] = count - f.announced[notification.hash] = append(f.announced[notification.hash], notification) - if f.announceChangeHook != nil && len(f.announced[notification.hash]) == 1 { - f.announceChangeHook(notification.hash, true) - } - if len(f.announced) == 1 { - f.rescheduleFetch(fetchTimer) - } - - case op := <-f.inject: - // A direct block insertion was requested, try and fill any pending gaps - blockBroadcastInMeter.Mark(1) - - // Now only direct block injection is allowed, drop the header injection - // here silently if we receive. - if f.light { - continue - } - f.enqueue(op.origin, nil, op.block) - - case hash := <-f.done: - // A pending import finished, remove all traces of the notification - f.forgetHash(hash) - f.forgetBlock(hash) - - case <-fetchTimer.C: - // At least one block's timer ran out, check for needing retrieval - request := make(map[string][]common.Hash) - - for hash, announces := range f.announced { - // In current LES protocol(les2/les3), only header announce is - // available, no need to wait too much time for header broadcast. - timeout := arriveTimeout - gatherSlack - if f.light { - timeout = 0 - } - if time.Since(announces[0].time) > timeout { - // Pick a random peer to retrieve from, reset all others - announce := announces[rand.Intn(len(announces))] - f.forgetHash(hash) - - // If the block still didn't arrive, queue for fetching - if (f.light && f.getHeader(hash) == nil) || (!f.light && f.getBlock(hash) == nil) { - request[announce.origin] = append(request[announce.origin], hash) - f.fetching[hash] = announce - } - } - } - // Send out all block header requests - for peer, hashes := range request { - log.Trace("Fetching scheduled headers", "peer", peer, "list", hashes) - - // Create a closure of the fetch and schedule in on a new thread - fetchHeader, hashes := f.fetching[hashes[0]].fetchHeader, hashes - go func(peer string) { - if f.fetchingHook != nil { - f.fetchingHook(hashes) - } - for _, hash := range hashes { - headerFetchMeter.Mark(1) - go func(hash common.Hash) { - resCh := make(chan *eth.Response) - - req, err := fetchHeader(hash, resCh) - if err != nil { - return // Legacy code, yolo - } - defer req.Close() - - timeout := time.NewTimer(2 * fetchTimeout) // 2x leeway before dropping the peer - defer timeout.Stop() - - select { - case res := <-resCh: - res.Done <- nil - f.FilterHeaders(peer, *res.Res.(*eth.BlockHeadersRequest), time.Now()) - - case <-timeout.C: - // The peer didn't respond in time. The request - // was already rescheduled at this point, we were - // waiting for a catchup. With an unresponsive - // peer however, it's a protocol violation. - f.dropPeer(peer) - } - }(hash) - } - }(peer) - } - // Schedule the next fetch if blocks are still pending - f.rescheduleFetch(fetchTimer) - - case <-completeTimer.C: - // At least one header's timer ran out, retrieve everything - request := make(map[string][]common.Hash) - - for hash, announces := range f.fetched { - // Pick a random peer to retrieve from, reset all others - announce := announces[rand.Intn(len(announces))] - f.forgetHash(hash) - - // If the block still didn't arrive, queue for completion - if f.getBlock(hash) == nil { - request[announce.origin] = append(request[announce.origin], hash) - f.completing[hash] = announce - } - } - // Send out all block body requests - for peer, hashes := range request { - log.Trace("Fetching scheduled bodies", "peer", peer, "list", hashes) - - // Create a closure of the fetch and schedule in on a new thread - if f.completingHook != nil { - f.completingHook(hashes) - } - fetchBodies := f.completing[hashes[0]].fetchBodies - bodyFetchMeter.Mark(int64(len(hashes))) - - go func(peer string, hashes []common.Hash) { - resCh := make(chan *eth.Response) - - req, err := fetchBodies(hashes, resCh) - if err != nil { - return // Legacy code, yolo - } - defer req.Close() - - timeout := time.NewTimer(2 * fetchTimeout) // 2x leeway before dropping the peer - defer timeout.Stop() - - select { - case res := <-resCh: - res.Done <- nil - // Ignoring withdrawals here, since the block fetcher is not used post-merge. - txs, uncles, _ := res.Res.(*eth.BlockBodiesResponse).Unpack() - f.FilterBodies(peer, txs, uncles, time.Now()) - - case <-timeout.C: - // The peer didn't respond in time. The request - // was already rescheduled at this point, we were - // waiting for a catchup. With an unresponsive - // peer however, it's a protocol violation. - f.dropPeer(peer) - } - }(peer, hashes) - } - // Schedule the next fetch if blocks are still pending - f.rescheduleComplete(completeTimer) - - case filter := <-f.headerFilter: - // Headers arrived from a remote peer. Extract those that were explicitly - // requested by the fetcher, and return everything else so it's delivered - // to other parts of the system. - var task *headerFilterTask - select { - case task = <-filter: - case <-f.quit: - return - } - headerFilterInMeter.Mark(int64(len(task.headers))) - - // Split the batch of headers into unknown ones (to return to the caller), - // known incomplete ones (requiring body retrievals) and completed blocks. - unknown, incomplete, complete, lightHeaders := []*types.Header{}, []*blockAnnounce{}, []*types.Block{}, []*blockAnnounce{} - for _, header := range task.headers { - hash := header.Hash() - - // Filter fetcher-requested headers from other synchronisation algorithms - if announce := f.fetching[hash]; announce != nil && announce.origin == task.peer && f.fetched[hash] == nil && f.completing[hash] == nil && f.queued[hash] == nil { - // If the delivered header does not match the promised number, drop the announcer - if header.Number.Uint64() != announce.number { - log.Trace("Invalid block number fetched", "peer", announce.origin, "hash", header.Hash(), "announced", announce.number, "provided", header.Number) - f.dropPeer(announce.origin) - f.forgetHash(hash) - continue - } - // Collect all headers only if we are running in light - // mode and the headers are not imported by other means. - if f.light { - if f.getHeader(hash) == nil { - announce.header = header - lightHeaders = append(lightHeaders, announce) - } - f.forgetHash(hash) - continue - } - // Only keep if not imported by other means - if f.getBlock(hash) == nil { - announce.header = header - announce.time = task.time - - // If the block is empty (header only), short circuit into the final import queue - if header.TxHash == types.EmptyTxsHash && header.UncleHash == types.EmptyUncleHash { - log.Trace("Block empty, skipping body retrieval", "peer", announce.origin, "number", header.Number, "hash", header.Hash()) - - block := types.NewBlockWithHeader(header) - block.ReceivedAt = task.time - - complete = append(complete, block) - f.completing[hash] = announce - continue - } - // Otherwise add to the list of blocks needing completion - incomplete = append(incomplete, announce) - } else { - log.Trace("Block already imported, discarding header", "peer", announce.origin, "number", header.Number, "hash", header.Hash()) - f.forgetHash(hash) - } - } else { - // BlockFetcher doesn't know about it, add to the return list - unknown = append(unknown, header) - } - } - headerFilterOutMeter.Mark(int64(len(unknown))) - select { - case filter <- &headerFilterTask{headers: unknown, time: task.time}: - case <-f.quit: - return - } - // Schedule the retrieved headers for body completion - for _, announce := range incomplete { - hash := announce.header.Hash() - if _, ok := f.completing[hash]; ok { - continue - } - f.fetched[hash] = append(f.fetched[hash], announce) - if len(f.fetched) == 1 { - f.rescheduleComplete(completeTimer) - } - } - // Schedule the header for light fetcher import - for _, announce := range lightHeaders { - f.enqueue(announce.origin, announce.header, nil) - } - // Schedule the header-only blocks for import - for _, block := range complete { - if announce := f.completing[block.Hash()]; announce != nil { - f.enqueue(announce.origin, nil, block) - } - } - - case filter := <-f.bodyFilter: - // Block bodies arrived, extract any explicitly requested blocks, return the rest - var task *bodyFilterTask - select { - case task = <-filter: - case <-f.quit: - return - } - bodyFilterInMeter.Mark(int64(len(task.transactions))) - blocks := []*types.Block{} - // abort early if there's nothing explicitly requested - if len(f.completing) > 0 { - for i := 0; i < len(task.transactions) && i < len(task.uncles); i++ { - // Match up a body to any possible completion request - var ( - matched = false - uncleHash common.Hash // calculated lazily and reused - txnHash common.Hash // calculated lazily and reused - ) - for hash, announce := range f.completing { - if f.queued[hash] != nil || announce.origin != task.peer { - continue - } - if uncleHash == (common.Hash{}) { - uncleHash = types.CalcUncleHash(task.uncles[i]) - } - if uncleHash != announce.header.UncleHash { - continue - } - if txnHash == (common.Hash{}) { - txnHash = types.DeriveSha(types.Transactions(task.transactions[i]), trie.NewStackTrie(nil)) - } - if txnHash != announce.header.TxHash { - continue - } - // Mark the body matched, reassemble if still unknown - matched = true - if f.getBlock(hash) == nil { - block := types.NewBlockWithHeader(announce.header).WithBody(task.transactions[i], task.uncles[i]) - block.ReceivedAt = task.time - blocks = append(blocks, block) - } else { - f.forgetHash(hash) - } - } - if matched { - task.transactions = append(task.transactions[:i], task.transactions[i+1:]...) - task.uncles = append(task.uncles[:i], task.uncles[i+1:]...) - i-- - continue - } - } - } - bodyFilterOutMeter.Mark(int64(len(task.transactions))) - select { - case filter <- task: - case <-f.quit: - return - } - // Schedule the retrieved blocks for ordered import - for _, block := range blocks { - if announce := f.completing[block.Hash()]; announce != nil { - f.enqueue(announce.origin, nil, block) - } - } - } - } -} - -// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout. -func (f *BlockFetcher) rescheduleFetch(fetch *time.Timer) { - // Short circuit if no blocks are announced - if len(f.announced) == 0 { - return - } - // Schedule announcement retrieval quickly for light mode - // since server won't send any headers to client. - if f.light { - fetch.Reset(lightTimeout) - return - } - // Otherwise find the earliest expiring announcement - earliest := time.Now() - for _, announces := range f.announced { - if earliest.After(announces[0].time) { - earliest = announces[0].time - } - } - fetch.Reset(arriveTimeout - time.Since(earliest)) -} - -// rescheduleComplete resets the specified completion timer to the next fetch timeout. -func (f *BlockFetcher) rescheduleComplete(complete *time.Timer) { - // Short circuit if no headers are fetched - if len(f.fetched) == 0 { - return - } - // Otherwise find the earliest expiring announcement - earliest := time.Now() - for _, announces := range f.fetched { - if earliest.After(announces[0].time) { - earliest = announces[0].time - } - } - complete.Reset(gatherSlack - time.Since(earliest)) -} - -// enqueue schedules a new header or block import operation, if the component -// to be imported has not yet been seen. -func (f *BlockFetcher) enqueue(peer string, header *types.Header, block *types.Block) { - var ( - hash common.Hash - number uint64 - ) - if header != nil { - hash, number = header.Hash(), header.Number.Uint64() - } else { - hash, number = block.Hash(), block.NumberU64() - } - // Ensure the peer isn't DOSing us - count := f.queues[peer] + 1 - if count > blockLimit { - log.Debug("Discarded delivered header or block, exceeded allowance", "peer", peer, "number", number, "hash", hash, "limit", blockLimit) - blockBroadcastDOSMeter.Mark(1) - f.forgetHash(hash) - return - } - // Discard any past or too distant blocks - if dist := int64(number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { - log.Debug("Discarded delivered header or block, too far away", "peer", peer, "number", number, "hash", hash, "distance", dist) - blockBroadcastDropMeter.Mark(1) - f.forgetHash(hash) - return - } - // Schedule the block for future importing - if _, ok := f.queued[hash]; !ok { - op := &blockOrHeaderInject{origin: peer} - if header != nil { - op.header = header - } else { - op.block = block - } - f.queues[peer] = count - f.queued[hash] = op - f.queue.Push(op, -int64(number)) - if f.queueChangeHook != nil { - f.queueChangeHook(hash, true) - } - log.Debug("Queued delivered header or block", "peer", peer, "number", number, "hash", hash, "queued", f.queue.Size()) - } -} - -// importHeaders spawns a new goroutine to run a header insertion into the chain. -// If the header's number is at the same height as the current import phase, it -// updates the phase states accordingly. -func (f *BlockFetcher) importHeaders(peer string, header *types.Header) { - hash := header.Hash() - log.Debug("Importing propagated header", "peer", peer, "number", header.Number, "hash", hash) - - go func() { - defer func() { f.done <- hash }() - // If the parent's unknown, abort insertion - parent := f.getHeader(header.ParentHash) - if parent == nil { - log.Debug("Unknown parent of propagated header", "peer", peer, "number", header.Number, "hash", hash, "parent", header.ParentHash) - return - } - // Validate the header and if something went wrong, drop the peer - if err := f.verifyHeader(header); err != nil && err != consensus.ErrFutureBlock { - log.Debug("Propagated header verification failed", "peer", peer, "number", header.Number, "hash", hash, "err", err) - f.dropPeer(peer) - return - } - // Run the actual import and log any issues - if _, err := f.insertHeaders([]*types.Header{header}); err != nil { - log.Debug("Propagated header import failed", "peer", peer, "number", header.Number, "hash", hash, "err", err) - return - } - // Invoke the testing hook if needed - if f.importedHook != nil { - f.importedHook(header, nil) - } - }() -} - -// importBlocks spawns a new goroutine to run a block insertion into the chain. If the -// block's number is at the same height as the current import phase, it updates -// the phase states accordingly. -func (f *BlockFetcher) importBlocks(peer string, block *types.Block) { - hash := block.Hash() - - // Run the import on a new thread - log.Debug("Importing propagated block", "peer", peer, "number", block.Number(), "hash", hash) - go func() { - defer func() { f.done <- hash }() - - // If the parent's unknown, abort insertion - parent := f.getBlock(block.ParentHash()) - if parent == nil { - log.Debug("Unknown parent of propagated block", "peer", peer, "number", block.Number(), "hash", hash, "parent", block.ParentHash()) - return - } - // Quickly validate the header and propagate the block if it passes - switch err := f.verifyHeader(block.Header()); err { - case nil: - // All ok, quickly propagate to our peers - blockBroadcastOutTimer.UpdateSince(block.ReceivedAt) - go f.broadcastBlock(block, true) - - case consensus.ErrFutureBlock: - // Weird future block, don't fail, but neither propagate - - default: - // Something went very wrong, drop the peer - log.Debug("Propagated block verification failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err) - f.dropPeer(peer) - return - } - // Run the actual import and log any issues - if _, err := f.insertChain(types.Blocks{block}); err != nil { - log.Debug("Propagated block import failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err) - return - } - // If import succeeded, broadcast the block - blockAnnounceOutTimer.UpdateSince(block.ReceivedAt) - go f.broadcastBlock(block, false) - - // Invoke the testing hook if needed - if f.importedHook != nil { - f.importedHook(nil, block) - } - }() -} - -// forgetHash removes all traces of a block announcement from the fetcher's -// internal state. -func (f *BlockFetcher) forgetHash(hash common.Hash) { - // Remove all pending announces and decrement DOS counters - if announceMap, ok := f.announced[hash]; ok { - for _, announce := range announceMap { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - } - delete(f.announced, hash) - if f.announceChangeHook != nil { - f.announceChangeHook(hash, false) - } - } - // Remove any pending fetches and decrement the DOS counters - if announce := f.fetching[hash]; announce != nil { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - delete(f.fetching, hash) - } - - // Remove any pending completion requests and decrement the DOS counters - for _, announce := range f.fetched[hash] { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - } - delete(f.fetched, hash) - - // Remove any pending completions and decrement the DOS counters - if announce := f.completing[hash]; announce != nil { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - delete(f.completing, hash) - } -} - -// forgetBlock removes all traces of a queued block from the fetcher's internal -// state. -func (f *BlockFetcher) forgetBlock(hash common.Hash) { - if insert := f.queued[hash]; insert != nil { - f.queues[insert.origin]-- - if f.queues[insert.origin] == 0 { - delete(f.queues, insert.origin) - } - delete(f.queued, hash) - } -} diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go deleted file mode 100644 index 6927300b1d..0000000000 --- a/eth/fetcher/block_fetcher_test.go +++ /dev/null @@ -1,948 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package fetcher - -import ( - "errors" - "math/big" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" -) - -var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - gspec = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - genesis = gspec.MustCommit(testdb, trie.NewDatabase(testdb, trie.HashDefaults)) - unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) -) - -// makeChain creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 3rd block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { - blocks, _ := core.GenerateChain(gspec.Config, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { - block.SetCoinbase(common.Address{seed}) - - // If the block number is multiple of 3, send a bonus transaction to the miner - if parent == genesis && i%3 == 0 { - signer := types.MakeSigner(params.TestChainConfig, block.Number(), block.Timestamp()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) - if err != nil { - panic(err) - } - block.AddTx(tx) - } - // If the block number is a multiple of 5, add a bonus uncle to the block - if i > 0 && i%5 == 0 { - block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 2).Hash(), Number: big.NewInt(int64(i - 1))}) - } - }) - hashes := make([]common.Hash, n+1) - hashes[len(hashes)-1] = parent.Hash() - blockm := make(map[common.Hash]*types.Block, n+1) - blockm[parent.Hash()] = parent - for i, b := range blocks { - hashes[len(hashes)-i-2] = b.Hash() - blockm[b.Hash()] = b - } - return hashes, blockm -} - -// fetcherTester is a test simulator for mocking out local block chain. -type fetcherTester struct { - fetcher *BlockFetcher - - hashes []common.Hash // Hash chain belonging to the tester - headers map[common.Hash]*types.Header // Headers belonging to the tester - blocks map[common.Hash]*types.Block // Blocks belonging to the tester - drops map[string]bool // Map of peers dropped by the fetcher - - lock sync.RWMutex -} - -// newTester creates a new fetcher test mocker. -func newTester(light bool) *fetcherTester { - tester := &fetcherTester{ - hashes: []common.Hash{genesis.Hash()}, - headers: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()}, - blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, - drops: make(map[string]bool), - } - tester.fetcher = NewBlockFetcher(light, tester.getHeader, tester.getBlock, tester.verifyHeader, tester.broadcastBlock, tester.chainHeight, tester.insertHeaders, tester.insertChain, tester.dropPeer) - tester.fetcher.Start() - - return tester -} - -// getHeader retrieves a header from the tester's block chain. -func (f *fetcherTester) getHeader(hash common.Hash) *types.Header { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.headers[hash] -} - -// getBlock retrieves a block from the tester's block chain. -func (f *fetcherTester) getBlock(hash common.Hash) *types.Block { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.blocks[hash] -} - -// verifyHeader is a nop placeholder for the block header verification. -func (f *fetcherTester) verifyHeader(header *types.Header) error { - return nil -} - -// broadcastBlock is a nop placeholder for the block broadcasting. -func (f *fetcherTester) broadcastBlock(block *types.Block, propagate bool) { -} - -// chainHeight retrieves the current height (block number) of the chain. -func (f *fetcherTester) chainHeight() uint64 { - f.lock.RLock() - defer f.lock.RUnlock() - - if f.fetcher.light { - return f.headers[f.hashes[len(f.hashes)-1]].Number.Uint64() - } - return f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() -} - -// insertChain injects a new headers into the simulated chain. -func (f *fetcherTester) insertHeaders(headers []*types.Header) (int, error) { - f.lock.Lock() - defer f.lock.Unlock() - - for i, header := range headers { - // Make sure the parent in known - if _, ok := f.headers[header.ParentHash]; !ok { - return i, errors.New("unknown parent") - } - // Discard any new blocks if the same height already exists - if header.Number.Uint64() <= f.headers[f.hashes[len(f.hashes)-1]].Number.Uint64() { - return i, nil - } - // Otherwise build our current chain - f.hashes = append(f.hashes, header.Hash()) - f.headers[header.Hash()] = header - } - return 0, nil -} - -// insertChain injects a new blocks into the simulated chain. -func (f *fetcherTester) insertChain(blocks types.Blocks) (int, error) { - f.lock.Lock() - defer f.lock.Unlock() - - for i, block := range blocks { - // Make sure the parent in known - if _, ok := f.blocks[block.ParentHash()]; !ok { - return i, errors.New("unknown parent") - } - // Discard any new blocks if the same height already exists - if block.NumberU64() <= f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() { - return i, nil - } - // Otherwise build our current chain - f.hashes = append(f.hashes, block.Hash()) - f.blocks[block.Hash()] = block - } - return 0, nil -} - -// dropPeer is an emulator for the peer removal, simply accumulating the various -// peers dropped by the fetcher. -func (f *fetcherTester) dropPeer(peer string) { - f.lock.Lock() - defer f.lock.Unlock() - - f.drops[peer] = true -} - -// makeHeaderFetcher retrieves a block header fetcher associated with a simulated peer. -func (f *fetcherTester) makeHeaderFetcher(peer string, blocks map[common.Hash]*types.Block, drift time.Duration) headerRequesterFn { - closure := make(map[common.Hash]*types.Block) - for hash, block := range blocks { - closure[hash] = block - } - // Create a function that return a header from the closure - return func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - // Gather the blocks to return - headers := make([]*types.Header, 0, 1) - if block, ok := closure[hash]; ok { - headers = append(headers, block.Header()) - } - // Return on a new thread - req := ð.Request{ - Peer: peer, - } - res := ð.Response{ - Req: req, - Res: (*eth.BlockHeadersRequest)(&headers), - Time: drift, - Done: make(chan error, 1), // Ignore the returned status - } - go func() { - sink <- res - }() - return req, nil - } -} - -// makeBodyFetcher retrieves a block body fetcher associated with a simulated peer. -func (f *fetcherTester) makeBodyFetcher(peer string, blocks map[common.Hash]*types.Block, drift time.Duration) bodyRequesterFn { - closure := make(map[common.Hash]*types.Block) - for hash, block := range blocks { - closure[hash] = block - } - // Create a function that returns blocks from the closure - return func(hashes []common.Hash, sink chan *eth.Response) (*eth.Request, error) { - // Gather the block bodies to return - transactions := make([][]*types.Transaction, 0, len(hashes)) - uncles := make([][]*types.Header, 0, len(hashes)) - - for _, hash := range hashes { - if block, ok := closure[hash]; ok { - transactions = append(transactions, block.Transactions()) - uncles = append(uncles, block.Uncles()) - } - } - // Return on a new thread - bodies := make([]*eth.BlockBody, len(transactions)) - for i, txs := range transactions { - bodies[i] = ð.BlockBody{ - Transactions: txs, - Uncles: uncles[i], - } - } - req := ð.Request{ - Peer: peer, - } - res := ð.Response{ - Req: req, - Res: (*eth.BlockBodiesResponse)(&bodies), - Time: drift, - Done: make(chan error, 1), // Ignore the returned status - } - go func() { - sink <- res - }() - return req, nil - } -} - -// verifyFetchingEvent verifies that one single event arrive on a fetching channel. -func verifyFetchingEvent(t *testing.T, fetching chan []common.Hash, arrive bool) { - t.Helper() - - if arrive { - select { - case <-fetching: - case <-time.After(time.Second): - t.Fatalf("fetching timeout") - } - } else { - select { - case <-fetching: - t.Fatalf("fetching invoked") - case <-time.After(10 * time.Millisecond): - } - } -} - -// verifyCompletingEvent verifies that one single event arrive on an completing channel. -func verifyCompletingEvent(t *testing.T, completing chan []common.Hash, arrive bool) { - t.Helper() - - if arrive { - select { - case <-completing: - case <-time.After(time.Second): - t.Fatalf("completing timeout") - } - } else { - select { - case <-completing: - t.Fatalf("completing invoked") - case <-time.After(10 * time.Millisecond): - } - } -} - -// verifyImportEvent verifies that one single event arrive on an import channel. -func verifyImportEvent(t *testing.T, imported chan interface{}, arrive bool) { - t.Helper() - - if arrive { - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("import timeout") - } - } else { - select { - case <-imported: - t.Fatalf("import invoked") - case <-time.After(20 * time.Millisecond): - } - } -} - -// verifyImportCount verifies that exactly count number of events arrive on an -// import hook channel. -func verifyImportCount(t *testing.T, imported chan interface{}, count int) { - t.Helper() - - for i := 0; i < count; i++ { - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("block %d: import timeout", i+1) - } - } - verifyImportDone(t, imported) -} - -// verifyImportDone verifies that no more events are arriving on an import channel. -func verifyImportDone(t *testing.T, imported chan interface{}) { - t.Helper() - - select { - case <-imported: - t.Fatalf("extra block imported") - case <-time.After(50 * time.Millisecond): - } -} - -// verifyChainHeight verifies the chain height is as expected. -func verifyChainHeight(t *testing.T, fetcher *fetcherTester, height uint64) { - t.Helper() - - if fetcher.chainHeight() != height { - t.Fatalf("chain height mismatch, got %d, want %d", fetcher.chainHeight(), height) - } -} - -// Tests that a fetcher accepts block/header announcements and initiates retrievals -// for them, successfully importing into the local chain. -func TestFullSequentialAnnouncements(t *testing.T) { testSequentialAnnouncements(t, false) } -func TestLightSequentialAnnouncements(t *testing.T) { testSequentialAnnouncements(t, true) } - -func testSequentialAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - tester := newTester(light) - defer tester.fetcher.Stop() - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks until all are imported - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that if blocks are announced by multiple peers (or even the same buggy -// peer), they will only get downloaded at most once. -func TestFullConcurrentAnnouncements(t *testing.T) { testConcurrentAnnouncements(t, false) } -func TestLightConcurrentAnnouncements(t *testing.T) { testConcurrentAnnouncements(t, true) } - -func testConcurrentAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - // Assemble a tester with a built in counter for the requests - tester := newTester(light) - firstHeaderFetcher := tester.makeHeaderFetcher("first", blocks, -gatherSlack) - firstBodyFetcher := tester.makeBodyFetcher("first", blocks, 0) - secondHeaderFetcher := tester.makeHeaderFetcher("second", blocks, -gatherSlack) - secondBodyFetcher := tester.makeBodyFetcher("second", blocks, 0) - - var counter atomic.Uint32 - firstHeaderWrapper := func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - counter.Add(1) - return firstHeaderFetcher(hash, sink) - } - secondHeaderWrapper := func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - counter.Add(1) - return secondHeaderFetcher(hash, sink) - } - // Iteratively announce blocks until all are imported - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), firstHeaderWrapper, firstBodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), secondHeaderWrapper, secondBodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), secondHeaderWrapper, secondBodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) - - // Make sure no blocks were retrieved twice - if c := int(counter.Load()); c != targetBlocks { - t.Fatalf("retrieval count mismatch: have %v, want %v", c, targetBlocks) - } - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that announcements arriving while a previous is being fetched still -// results in a valid import. -func TestFullOverlappingAnnouncements(t *testing.T) { testOverlappingAnnouncements(t, false) } -func TestLightOverlappingAnnouncements(t *testing.T) { testOverlappingAnnouncements(t, true) } - -func testOverlappingAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, but overlap them continuously - overlap := 16 - imported := make(chan interface{}, len(hashes)-1) - for i := 0; i < overlap; i++ { - imported <- nil - } - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("block %d: import timeout", len(hashes)-i) - } - } - // Wait for all the imports to complete and check count - verifyImportCount(t, imported, overlap) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that announces already being retrieved will not be duplicated. -func TestFullPendingDeduplication(t *testing.T) { testPendingDeduplication(t, false) } -func TestLightPendingDeduplication(t *testing.T) { testPendingDeduplication(t, true) } - -func testPendingDeduplication(t *testing.T, light bool) { - // Create a hash and corresponding block - hashes, blocks := makeChain(1, 0, genesis) - - // Assemble a tester with a built in counter and delayed fetcher - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("repeater", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("repeater", blocks, 0) - - delay := 50 * time.Millisecond - var counter atomic.Uint32 - headerWrapper := func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - counter.Add(1) - - // Simulate a long running fetch - resink := make(chan *eth.Response) - req, err := headerFetcher(hash, resink) - if err == nil { - go func() { - res := <-resink - time.Sleep(delay) - sink <- res - }() - } - return req, err - } - checkNonExist := func() bool { - return tester.getBlock(hashes[0]) == nil - } - if light { - checkNonExist = func() bool { - return tester.getHeader(hashes[0]) == nil - } - } - // Announce the same block many times until it's fetched (wait for any pending ops) - for checkNonExist() { - tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), headerWrapper, bodyFetcher) - time.Sleep(time.Millisecond) - } - time.Sleep(delay) - - // Check that all blocks were imported and none fetched twice - if c := counter.Load(); c != 1 { - t.Fatalf("retrieval count mismatch: have %v, want %v", c, 1) - } - verifyChainHeight(t, tester, 1) -} - -// Tests that announcements retrieved in a random order are cached and eventually -// imported when all the gaps are filled in. -func TestFullRandomArrivalImport(t *testing.T) { testRandomArrivalImport(t, false) } -func TestLightRandomArrivalImport(t *testing.T) { testRandomArrivalImport(t, true) } - -func testRandomArrivalImport(t *testing.T, light bool) { - // Create a chain of blocks to import, and choose one to delay - targetBlocks := maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - skip := targetBlocks / 2 - - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, skipping one entry - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 1; i >= 0; i-- { - if i != skip { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - time.Sleep(time.Millisecond) - } - } - // Finally announce the skipped entry and check full import - tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - verifyImportCount(t, imported, len(hashes)-1) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that direct block enqueues (due to block propagation vs. hash announce) -// are correctly schedule, filling and import queue gaps. -func TestQueueGapFill(t *testing.T) { - // Create a chain of blocks to import, and choose one to not announce at all - targetBlocks := maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - skip := targetBlocks / 2 - - tester := newTester(false) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, skipping one entry - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - - for i := len(hashes) - 1; i >= 0; i-- { - if i != skip { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - time.Sleep(time.Millisecond) - } - } - // Fill the missing block directly as if propagated - tester.fetcher.Enqueue("valid", blocks[hashes[skip]]) - verifyImportCount(t, imported, len(hashes)-1) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that blocks arriving from various sources (multiple propagations, hash -// announces, etc) do not get scheduled for import multiple times. -func TestImportDeduplication(t *testing.T) { - // Create two blocks to import (one for duplication, the other for stalling) - hashes, blocks := makeChain(2, 0, genesis) - - // Create the tester and wrap the importer with a counter - tester := newTester(false) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - var counter atomic.Uint32 - tester.fetcher.insertChain = func(blocks types.Blocks) (int, error) { - counter.Add(uint32(len(blocks))) - return tester.insertChain(blocks) - } - // Instrument the fetching and imported events - fetching := make(chan []common.Hash) - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - - // Announce the duplicating block, wait for retrieval, and also propagate directly - tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - <-fetching - - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - - // Fill the missing block directly as if propagated, and check import uniqueness - tester.fetcher.Enqueue("valid", blocks[hashes[1]]) - verifyImportCount(t, imported, 2) - - if c := counter.Load(); c != 2 { - t.Fatalf("import invocation count mismatch: have %v, want %v", c, 2) - } -} - -// Tests that blocks with numbers much lower or higher than out current head get -// discarded to prevent wasting resources on useless blocks from faulty peers. -func TestDistantPropagationDiscarding(t *testing.T) { - // Create a long chain to import and define the discard boundaries - hashes, blocks := makeChain(3*maxQueueDist, 0, genesis) - head := hashes[len(hashes)/2] - - low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1 - - // Create a tester and simulate a head block being the middle of the above chain - tester := newTester(false) - - tester.lock.Lock() - tester.hashes = []common.Hash{head} - tester.blocks = map[common.Hash]*types.Block{head: blocks[head]} - tester.lock.Unlock() - - // Ensure that a block with a lower number than the threshold is discarded - tester.fetcher.Enqueue("lower", blocks[hashes[low]]) - time.Sleep(10 * time.Millisecond) - if !tester.fetcher.queue.Empty() { - t.Fatalf("fetcher queued stale block") - } - // Ensure that a block with a higher number than the threshold is discarded - tester.fetcher.Enqueue("higher", blocks[hashes[high]]) - time.Sleep(10 * time.Millisecond) - if !tester.fetcher.queue.Empty() { - t.Fatalf("fetcher queued future block") - } -} - -// Tests that announcements with numbers much lower or higher than out current -// head get discarded to prevent wasting resources on useless blocks from faulty -// peers. -func TestFullDistantAnnouncementDiscarding(t *testing.T) { testDistantAnnouncementDiscarding(t, false) } -func TestLightDistantAnnouncementDiscarding(t *testing.T) { testDistantAnnouncementDiscarding(t, true) } - -func testDistantAnnouncementDiscarding(t *testing.T, light bool) { - // Create a long chain to import and define the discard boundaries - hashes, blocks := makeChain(3*maxQueueDist, 0, genesis) - head := hashes[len(hashes)/2] - - low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1 - - // Create a tester and simulate a head block being the middle of the above chain - tester := newTester(light) - - tester.lock.Lock() - tester.hashes = []common.Hash{head} - tester.headers = map[common.Hash]*types.Header{head: blocks[head].Header()} - tester.blocks = map[common.Hash]*types.Block{head: blocks[head]} - tester.lock.Unlock() - - headerFetcher := tester.makeHeaderFetcher("lower", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("lower", blocks, 0) - - fetching := make(chan struct{}, 2) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- struct{}{} } - - // Ensure that a block with a lower number than the threshold is discarded - tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-time.After(50 * time.Millisecond): - case <-fetching: - t.Fatalf("fetcher requested stale header") - } - // Ensure that a block with a higher number than the threshold is discarded - tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-time.After(50 * time.Millisecond): - case <-fetching: - t.Fatalf("fetcher requested future header") - } -} - -// Tests that peers announcing blocks with invalid numbers (i.e. not matching -// the headers provided afterwards) get dropped as malicious. -func TestFullInvalidNumberAnnouncement(t *testing.T) { testInvalidNumberAnnouncement(t, false) } -func TestLightInvalidNumberAnnouncement(t *testing.T) { testInvalidNumberAnnouncement(t, true) } - -func testInvalidNumberAnnouncement(t *testing.T, light bool) { - // Create a single block to import and check numbers against - hashes, blocks := makeChain(1, 0, genesis) - - tester := newTester(light) - badHeaderFetcher := tester.makeHeaderFetcher("bad", blocks, -gatherSlack) - badBodyFetcher := tester.makeBodyFetcher("bad", blocks, 0) - - imported := make(chan interface{}) - announced := make(chan interface{}, 2) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - // Announce a block with a bad number, check for immediate drop - tester.fetcher.announceChangeHook = func(hash common.Hash, b bool) { - announced <- nil - } - tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), badHeaderFetcher, badBodyFetcher) - verifyAnnounce := func() { - for i := 0; i < 2; i++ { - select { - case <-announced: - continue - case <-time.After(1 * time.Second): - t.Fatal("announce timeout") - return - } - } - } - verifyAnnounce() - verifyImportEvent(t, imported, false) - tester.lock.RLock() - dropped := tester.drops["bad"] - tester.lock.RUnlock() - - if !dropped { - t.Fatalf("peer with invalid numbered announcement not dropped") - } - goodHeaderFetcher := tester.makeHeaderFetcher("good", blocks, -gatherSlack) - goodBodyFetcher := tester.makeBodyFetcher("good", blocks, 0) - // Make sure a good announcement passes without a drop - tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), goodHeaderFetcher, goodBodyFetcher) - verifyAnnounce() - verifyImportEvent(t, imported, true) - - tester.lock.RLock() - dropped = tester.drops["good"] - tester.lock.RUnlock() - - if dropped { - t.Fatalf("peer with valid numbered announcement dropped") - } - verifyImportDone(t, imported) -} - -// Tests that if a block is empty (i.e. header only), no body request should be -// made, and instead the header should be assembled into a whole block in itself. -func TestEmptyBlockShortCircuit(t *testing.T) { - // Create a chain of blocks to import - hashes, blocks := makeChain(32, 0, genesis) - - tester := newTester(false) - defer tester.fetcher.Stop() - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Add a monitoring hook for all internal events - fetching := make(chan []common.Hash) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } - - completing := make(chan []common.Hash) - tester.fetcher.completingHook = func(hashes []common.Hash) { completing <- hashes } - - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - // Iteratively announce blocks until all are imported - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - - // All announces should fetch the header - verifyFetchingEvent(t, fetching, true) - - // Only blocks with data contents should request bodies - verifyCompletingEvent(t, completing, len(blocks[hashes[i]].Transactions()) > 0 || len(blocks[hashes[i]].Uncles()) > 0) - - // Irrelevant of the construct, import should succeed - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} - -// Tests that a peer is unable to use unbounded memory with sending infinite -// block announcements to a node, but that even in the face of such an attack, -// the fetcher remains operational. -func TestHashMemoryExhaustionAttack(t *testing.T) { - // Create a tester with instrumented import hooks - tester := newTester(false) - - imported, announces := make(chan interface{}), atomic.Int32{} - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - tester.fetcher.announceChangeHook = func(hash common.Hash, added bool) { - if added { - announces.Add(1) - } else { - announces.Add(-1) - } - } - // Create a valid chain and an infinite junk chain - targetBlocks := hashLimit + 2*maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - validHeaderFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - validBodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - attack, _ := makeChain(targetBlocks, 0, unknownBlock) - attackerHeaderFetcher := tester.makeHeaderFetcher("attacker", nil, -gatherSlack) - attackerBodyFetcher := tester.makeBodyFetcher("attacker", nil, 0) - - // Feed the tester a huge hashset from the attacker, and a limited from the valid peer - for i := 0; i < len(attack); i++ { - if i < maxQueueDist { - tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), validHeaderFetcher, validBodyFetcher) - } - tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), attackerHeaderFetcher, attackerBodyFetcher) - } - if count := announces.Load(); count != hashLimit+maxQueueDist { - t.Fatalf("queued announce count mismatch: have %d, want %d", count, hashLimit+maxQueueDist) - } - // Wait for fetches to complete - verifyImportCount(t, imported, maxQueueDist) - - // Feed the remaining valid hashes to ensure DOS protection state remains clean - for i := len(hashes) - maxQueueDist - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), validHeaderFetcher, validBodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} - -// Tests that blocks sent to the fetcher (either through propagation or via hash -// announces and retrievals) don't pile up indefinitely, exhausting available -// system memory. -func TestBlockMemoryExhaustionAttack(t *testing.T) { - // Create a tester with instrumented import hooks - tester := newTester(false) - - imported, enqueued := make(chan interface{}), atomic.Int32{} - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - tester.fetcher.queueChangeHook = func(hash common.Hash, added bool) { - if added { - enqueued.Add(1) - } else { - enqueued.Add(-1) - } - } - // Create a valid chain and a batch of dangling (but in range) blocks - targetBlocks := hashLimit + 2*maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - attack := make(map[common.Hash]*types.Block) - for i := byte(0); len(attack) < blockLimit+2*maxQueueDist; i++ { - hashes, blocks := makeChain(maxQueueDist-1, i, unknownBlock) - for _, hash := range hashes[:maxQueueDist-2] { - attack[hash] = blocks[hash] - } - } - // Try to feed all the attacker blocks make sure only a limited batch is accepted - for _, block := range attack { - tester.fetcher.Enqueue("attacker", block) - } - time.Sleep(200 * time.Millisecond) - if queued := enqueued.Load(); queued != blockLimit { - t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit) - } - // Queue up a batch of valid blocks, and check that a new peer is allowed to do so - for i := 0; i < maxQueueDist-1; i++ { - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-3-i]]) - } - time.Sleep(100 * time.Millisecond) - if queued := enqueued.Load(); queued != blockLimit+maxQueueDist-1 { - t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit+maxQueueDist-1) - } - // Insert the missing piece (and sanity check the import) - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2]]) - verifyImportCount(t, imported, maxQueueDist) - - // Insert the remaining blocks in chunks to ensure clean DOS protection - for i := maxQueueDist; i < len(hashes)-1; i++ { - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2-i]]) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index ea7892d8d8..18c5ff007a 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -107,6 +107,8 @@ var ( txFetcherFetchingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/fetching/hashes", nil) ) +var errTerminated = errors.New("terminated") + // txAnnounce is the notification of the availability of a batch // of new transactions in the network. type txAnnounce struct { @@ -783,7 +785,7 @@ func (f *TxFetcher) loop() { // rescheduleWait iterates over all the transactions currently in the waitlist // and schedules the movement into the fetcher for the earliest. // -// The method has a granularity of 'gatherSlack', since there's not much point in +// The method has a granularity of 'txGatherSlack', since there's not much point in // spinning over all the transactions just to maybe find one that should trigger // a few ms earlier. func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { @@ -796,7 +798,7 @@ func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { for _, instance := range f.waittime { if earliest > instance { earliest = instance - if txArriveTimeout-time.Duration(now-earliest) < gatherSlack { + if txArriveTimeout-time.Duration(now-earliest) < txGatherSlack { break } } @@ -809,7 +811,7 @@ func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { // rescheduleTimeout iterates over all the transactions currently in flight and // schedules a cleanup run when the first would trigger. // -// The method has a granularity of 'gatherSlack', since there's not much point in +// The method has a granularity of 'txGatherSlack', since there's not much point in // spinning over all the transactions just to maybe find one that should trigger // a few ms earlier. // @@ -834,7 +836,7 @@ func (f *TxFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{} } if earliest > req.time { earliest = req.time - if txFetchTimeout-time.Duration(now-earliest) < gatherSlack { + if txFetchTimeout-time.Duration(now-earliest) < txGatherSlack { break } } diff --git a/eth/filters/api.go b/eth/filters/api.go index 8cf701ec57..59103ac03c 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -179,8 +179,6 @@ func (api *FilterAPI) NewPendingTransactions(ctx context.Context, fullTx *bool) } case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() @@ -241,8 +239,6 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { notifier.Notify(rpcSub.ID, h) case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() @@ -278,8 +274,6 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc } case <-rpcSub.Err(): // client send an unsubscribe request return - case <-notifier.Closed(): // connection dropped - return } } }() diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 83e3284a2b..f2b92d5a99 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -333,7 +333,7 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) ([]*typ // pendingLogs returns the logs matching the filter criteria within the pending block. func (f *Filter) pendingLogs() []*types.Log { - block, receipts := f.sys.backend.PendingBlockAndReceipts() + block, receipts, _ := f.sys.backend.Pending() if block == nil || receipts == nil { return nil } diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index f98a1f84ce..c32b837eb4 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -62,7 +63,7 @@ type Backend interface { GetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) - PendingBlockAndReceipts() (*types.Block, types.Receipts) + Pending() (*types.Block, types.Receipts, *state.StateDB) CurrentHeader() *types.Header ChainConfig() *params.ChainConfig @@ -70,7 +71,6 @@ type Backend interface { SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription BloomStatus() (uint64, uint64) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) @@ -198,20 +198,18 @@ type EventSystem struct { lastHead *types.Header // Subscriptions - txsSub event.Subscription // Subscription for new transaction event - logsSub event.Subscription // Subscription for new log event - rmLogsSub event.Subscription // Subscription for removed log event - pendingLogsSub event.Subscription // Subscription for pending log event - chainSub event.Subscription // Subscription for new chain event + txsSub event.Subscription // Subscription for new transaction event + logsSub event.Subscription // Subscription for new log event + rmLogsSub event.Subscription // Subscription for removed log event + chainSub event.Subscription // Subscription for new chain event // Channels - install chan *subscription // install filter for event notification - uninstall chan *subscription // remove filter for event notification - txsCh chan core.NewTxsEvent // Channel to receive new transactions event - logsCh chan []*types.Log // Channel to receive new log event - pendingLogsCh chan []*types.Log // Channel to receive new log event - rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event - chainCh chan core.ChainEvent // Channel to receive new chain event + install chan *subscription // install filter for event notification + uninstall chan *subscription // remove filter for event notification + txsCh chan core.NewTxsEvent // Channel to receive new transactions event + logsCh chan []*types.Log // Channel to receive new log event + rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event + chainCh chan core.ChainEvent // Channel to receive new chain event } // NewEventSystem creates a new manager that listens for event on the given mux, @@ -222,16 +220,15 @@ type EventSystem struct { // or by stopping the given mux. func NewEventSystem(sys *FilterSystem, lightMode bool) *EventSystem { m := &EventSystem{ - sys: sys, - backend: sys.backend, - lightMode: lightMode, - install: make(chan *subscription), - uninstall: make(chan *subscription), - txsCh: make(chan core.NewTxsEvent, txChanSize), - logsCh: make(chan []*types.Log, logsChanSize), - rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), - pendingLogsCh: make(chan []*types.Log, logsChanSize), - chainCh: make(chan core.ChainEvent, chainEvChanSize), + sys: sys, + backend: sys.backend, + lightMode: lightMode, + install: make(chan *subscription), + uninstall: make(chan *subscription), + txsCh: make(chan core.NewTxsEvent, txChanSize), + logsCh: make(chan []*types.Log, logsChanSize), + rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), + chainCh: make(chan core.ChainEvent, chainEvChanSize), } // Subscribe events @@ -239,10 +236,9 @@ func NewEventSystem(sys *FilterSystem, lightMode bool) *EventSystem { m.logsSub = m.backend.SubscribeLogsEvent(m.logsCh) m.rmLogsSub = m.backend.SubscribeRemovedLogsEvent(m.rmLogsCh) m.chainSub = m.backend.SubscribeChainEvent(m.chainCh) - m.pendingLogsSub = m.backend.SubscribePendingLogsEvent(m.pendingLogsCh) // Make sure none of the subscriptions are empty - if m.txsSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil || m.pendingLogsSub == nil { + if m.txsSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil { log.Crit("Subscribe for event system failed") } @@ -434,12 +430,12 @@ func (es *EventSystem) handleLogs(filters filterIndex, ev []*types.Log) { } } -func (es *EventSystem) handlePendingLogs(filters filterIndex, ev []*types.Log) { - if len(ev) == 0 { +func (es *EventSystem) handlePendingLogs(filters filterIndex, logs []*types.Log) { + if len(logs) == 0 { return } for _, f := range filters[PendingLogsSubscription] { - matchedLogs := filterLogs(ev, nil, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics) + matchedLogs := filterLogs(logs, nil, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics) if len(matchedLogs) > 0 { f.logs <- matchedLogs } @@ -550,7 +546,6 @@ func (es *EventSystem) eventLoop() { es.txsSub.Unsubscribe() es.logsSub.Unsubscribe() es.rmLogsSub.Unsubscribe() - es.pendingLogsSub.Unsubscribe() es.chainSub.Unsubscribe() }() @@ -567,10 +562,29 @@ func (es *EventSystem) eventLoop() { es.handleLogs(index, ev) case ev := <-es.rmLogsCh: es.handleLogs(index, ev.Logs) - case ev := <-es.pendingLogsCh: - es.handlePendingLogs(index, ev) case ev := <-es.chainCh: es.handleChainEvent(index, ev) + // If we have no pending log subscription, + // we don't need to collect any pending logs. + if len(index[PendingLogsSubscription]) == 0 { + continue + } + + // Pull the pending logs if there is a new chain head. + pendingBlock, pendingReceipts, _ := es.backend.Pending() + if pendingBlock == nil || pendingReceipts == nil { + continue + } + if pendingBlock.ParentHash() != ev.Block.Hash() { + continue + } + var logs []*types.Log + for _, receipt := range pendingReceipts { + if len(receipt.Logs) > 0 { + logs = append(logs, receipt.Logs...) + } + } + es.handlePendingLogs(index, logs) case f := <-es.install: if f.typ == MinedAndPendingLogsSubscription { diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 27cad8826a..6238c97735 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -48,7 +49,6 @@ type testBackend struct { txFeed event.Feed logsFeed event.Feed rmLogsFeed event.Feed - pendingLogsFeed event.Feed chainFeed event.Feed pendingBlock *types.Block pendingReceipts types.Receipts @@ -125,8 +125,8 @@ func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash, number uint return logs, nil } -func (b *testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { - return b.pendingBlock, b.pendingReceipts +func (b *testBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { + return b.pendingBlock, b.pendingReceipts, nil } func (b *testBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { @@ -141,10 +141,6 @@ func (b *testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscript return b.logsFeed.Subscribe(ch) } -func (b *testBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - return b.pendingLogsFeed.Subscribe(ch) -} - func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return b.chainFeed.Subscribe(ch) } @@ -180,6 +176,20 @@ func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.Matc }() } +func (b *testBackend) setPending(block *types.Block, receipts types.Receipts) { + b.pendingBlock = block + b.pendingReceipts = receipts +} + +func (b *testBackend) notifyPending(logs []*types.Log) { + genesis := &core.Genesis{ + Config: params.TestChainConfig, + } + _, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 2, func(i int, b *core.BlockGen) {}) + b.setPending(blocks[1], []*types.Receipt{{Logs: logs}}) + b.chainFeed.Send(core.ChainEvent{Block: blocks[0]}) +} + func newTestFilterSystem(t testing.TB, db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { backend := &testBackend{db: db} sys := NewFilterSystem(backend, cfg) @@ -203,7 +213,7 @@ func TestBlockSubscription(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), } _, chain, _ = core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 10, func(i int, gen *core.BlockGen) {}) - chainEvents = []core.ChainEvent{} + chainEvents []core.ChainEvent ) for _, blk := range chain { @@ -386,7 +396,7 @@ func TestLogFilterCreation(t *testing.T) { {FilterCriteria{FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(100)}, false}, // from block "higher" than to block {FilterCriteria{FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, false}, - // topics more then 4 + // topics more than 4 {FilterCriteria{Topics: [][]common.Hash{{}, {}, {}, {}, {}}}, false}, } ) @@ -546,9 +556,9 @@ func TestLogFilter(t *testing.T) { if nsend := backend.logsFeed.Send(allLogs); nsend == 0 { t.Fatal("Logs event not delivered") } - if nsend := backend.pendingLogsFeed.Send(allLogs); nsend == 0 { - t.Fatal("Pending logs event not delivered") - } + + // set pending logs + backend.notifyPending(allLogs) for i, tt := range testCases { var fetched []*types.Log @@ -754,10 +764,12 @@ func TestPendingLogsSubscription(t *testing.T) { }() } - // raise events - for _, ev := range allLogs { - backend.pendingLogsFeed.Send(ev) + // set pending logs + var flattenLogs []*types.Log + for _, logs := range allLogs { + flattenLogs = append(flattenLogs, logs...) } + backend.notifyPending(flattenLogs) for i := range testCases { err := <-testCases[i].err @@ -820,7 +832,7 @@ func TestLightFilterLogs(t *testing.T) { key, _ = crypto.GenerateKey() addr = crypto.PubkeyToAddress(key.PublicKey) genesis = &core.Genesis{Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr: {Balance: big.NewInt(params.Ether)}, }, } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 4250e3a9bf..48aaa584db 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -34,7 +34,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) func makeReceipt(addr common.Address) *types.Receipt { @@ -57,7 +57,7 @@ func BenchmarkFilters(b *testing.B) { addr4 = common.BytesToAddress([]byte("random addresses please")) gspec = &core.Genesis{ - Alloc: core.GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, + Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), Config: params.TestChainConfig, } @@ -86,7 +86,7 @@ func BenchmarkFilters(b *testing.B) { // The test txs are not properly signed, can't simply create a chain // and then import blocks. TODO(rjl493456442) try to get rid of the // manual database writes. - gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) + gspec.MustCommit(db, triedb.NewDatabase(db, triedb.HashDefaults)) for i, block := range chain { rawdb.WriteBlock(db, block) @@ -109,8 +109,8 @@ func BenchmarkFilters(b *testing.B) { func TestFilters(t *testing.T) { var ( - db = rawdb.NewMemoryDatabase() - _, sys = newTestFilterSystem(t, db, Config{}) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) // Sender account key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) @@ -165,7 +165,7 @@ func TestFilters(t *testing.T) { gspec = &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr: {Balance: big.NewInt(0).Mul(big.NewInt(100), big.NewInt(params.Ether))}, contract: {Balance: big.NewInt(0), Code: bytecode}, contract2: {Balance: big.NewInt(0), Code: bytecode}, @@ -181,7 +181,7 @@ func TestFilters(t *testing.T) { // Hack: GenerateChainWithGenesis creates a new db. // Commit the genesis manually and use GenerateChain. - _, err = gspec.Commit(db, trie.NewDatabase(db, nil)) + _, err = gspec.Commit(db, triedb.NewDatabase(db, nil)) if err != nil { t.Fatal(err) } @@ -277,8 +277,7 @@ func TestFilters(t *testing.T) { }), signer, key1) gen.AddTx(tx) }) - sys.backend.(*testBackend).pendingBlock = pchain[0] - sys.backend.(*testBackend).pendingReceipts = preceipts[0] + backend.setPending(pchain[0], preceipts[0]) for i, tc := range []struct { f *Filter diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 226991b24b..8ab57294b7 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -160,7 +160,7 @@ func (oracle *Oracle) resolveBlockRange(ctx context.Context, reqEnd rpc.BlockNum ) switch reqEnd { case rpc.PendingBlockNumber: - if pendingBlock, pendingReceipts = oracle.backend.PendingBlockAndReceipts(); pendingBlock != nil { + if pendingBlock, pendingReceipts, _ = oracle.backend.Pending(); pendingBlock != nil { resolved = pendingBlock.Header() } else { // Pending block not supported by backend, process only until latest block. @@ -227,8 +227,8 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks uint64, unresolvedL if p < 0 || p > 100 { return common.Big0, nil, nil, nil, fmt.Errorf("%w: %f", errInvalidPercentile, p) } - if i > 0 && p < rewardPercentiles[i-1] { - return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f > #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p) + if i > 0 && p <= rewardPercentiles[i-1] { + return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f >= #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p) } } var ( diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index b719649811..3fa70e41a0 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" @@ -54,7 +55,7 @@ type OracleBackend interface { HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) - PendingBlockAndReceipts() (*types.Block, types.Receipts) + Pending() (*types.Block, types.Receipts, *state.StateDB) ChainConfig() *params.ChainConfig SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription } diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 4ee5a0d1b2..1d2e02cde6 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -97,12 +98,13 @@ func (b *testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types. return b.chain.GetReceiptsByHash(hash), nil } -func (b *testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { +func (b *testBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { if b.pending { block := b.chain.GetBlockByNumber(testHead + 1) - return block, b.chain.GetReceiptsByHash(block.Hash()) + state, _ := b.chain.StateAt(block.Root()) + return block, b.chain.GetReceiptsByHash(block.Hash()), state } - return nil, nil + return nil, nil, nil } func (b *testBackend) ChainConfig() *params.ChainConfig { @@ -126,7 +128,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke config = *params.TestChainConfig // needs copy because it is modified below gspec = &core.Genesis{ Config: &config, - Alloc: core.GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, + Alloc: types.GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, } signer = types.LatestSigner(gspec.Config) ) diff --git a/eth/handler.go b/eth/handler.go index a327af6113..0d27e061c4 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -25,13 +25,12 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/fetcher" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -41,7 +40,9 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/triedb/pathdb" + "golang.org/x/crypto/sha3" ) const ( @@ -73,7 +74,7 @@ type txPool interface { // Pending should return pending transactions. // The slice should be modifiable by the caller. - Pending(enforceTips bool) map[common.Address][]*txpool.LazyTransaction + Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions @@ -84,10 +85,10 @@ type txPool interface { // handlerConfig is the collection of initialization parameters to create a full // node network handler. type handlerConfig struct { + NodeID enode.ID // P2P node ID used for tx propagation topology Database ethdb.Database // Database for direct sync insertions Chain *core.BlockChain // Blockchain to serve data from TxPool txPool // Transaction pool to propagate from - Merger *consensus.Merger // The manager for eth1/2 transition Network uint64 // Network identifier to advertise Sync downloader.SyncMode // Whether to snap or full sync BloomCache uint64 // Megabytes to alloc for snap sync bloom @@ -96,6 +97,7 @@ type handlerConfig struct { } type handler struct { + nodeID enode.ID networkID uint64 forkFilter forkid.Filter // Fork ID filter, constant across the lifetime of the node @@ -107,24 +109,20 @@ type handler struct { chain *core.BlockChain maxPeers int - downloader *downloader.Downloader - blockFetcher *fetcher.BlockFetcher - txFetcher *fetcher.TxFetcher - peers *peerSet - merger *consensus.Merger + downloader *downloader.Downloader + txFetcher *fetcher.TxFetcher + peers *peerSet - eventMux *event.TypeMux - txsCh chan core.NewTxsEvent - txsSub event.Subscription - minedBlockSub *event.TypeMuxSubscription + eventMux *event.TypeMux + txsCh chan core.NewTxsEvent + txsSub event.Subscription requiredBlocks map[uint64]common.Hash // channels for fetcher, syncer, txsyncLoop quitSync chan struct{} - chainSync *chainSyncer - wg sync.WaitGroup + wg sync.WaitGroup handlerStartCh chan struct{} handlerDoneCh chan struct{} @@ -137,6 +135,7 @@ func newHandler(config *handlerConfig) (*handler, error) { config.EventMux = new(event.TypeMux) // Nicety initialization for tests } h := &handler{ + nodeID: config.NodeID, networkID: config.Network, forkFilter: forkid.NewFilter(config.Chain), eventMux: config.EventMux, @@ -144,7 +143,6 @@ func newHandler(config *handlerConfig) (*handler, error) { txpool: config.TxPool, chain: config.Chain, peers: newPeerSet(), - merger: config.Merger, requiredBlocks: config.RequiredBlocks, quitSync: make(chan struct{}), handlerDoneCh: make(chan struct{}), @@ -184,92 +182,6 @@ func newHandler(config *handlerConfig) (*handler, error) { } // Construct the downloader (long sync) h.downloader = downloader.New(config.Database, h.eventMux, h.chain, nil, h.removePeer, h.enableSyncedFeatures) - if ttd := h.chain.Config().TerminalTotalDifficulty; ttd != nil { - if h.chain.Config().TerminalTotalDifficultyPassed { - log.Info("Chain post-merge, sync via beacon client") - } else { - head := h.chain.CurrentBlock() - if td := h.chain.GetTd(head.Hash(), head.Number.Uint64()); td.Cmp(ttd) >= 0 { - log.Info("Chain post-TTD, sync via beacon client") - } else { - log.Warn("Chain pre-merge, sync via PoW (ensure beacon client is ready)") - } - } - } else if h.chain.Config().TerminalTotalDifficultyPassed { - log.Error("Chain configured post-merge, but without TTD. Are you debugging sync?") - } - // Construct the fetcher (short sync) - validator := func(header *types.Header) error { - // All the block fetcher activities should be disabled - // after the transition. Print the warning log. - if h.merger.PoSFinalized() { - log.Warn("Unexpected validation activity", "hash", header.Hash(), "number", header.Number) - return errors.New("unexpected behavior after transition") - } - // Reject all the PoS style headers in the first place. No matter - // the chain has finished the transition or not, the PoS headers - // should only come from the trusted consensus layer instead of - // p2p network. - if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok { - if beacon.IsPoSHeader(header) { - return errors.New("unexpected post-merge header") - } - } - return h.chain.Engine().VerifyHeader(h.chain, header) - } - heighter := func() uint64 { - return h.chain.CurrentBlock().Number.Uint64() - } - inserter := func(blocks types.Blocks) (int, error) { - // All the block fetcher activities should be disabled - // after the transition. Print the warning log. - if h.merger.PoSFinalized() { - var ctx []interface{} - ctx = append(ctx, "blocks", len(blocks)) - if len(blocks) > 0 { - ctx = append(ctx, "firsthash", blocks[0].Hash()) - ctx = append(ctx, "firstnumber", blocks[0].Number()) - ctx = append(ctx, "lasthash", blocks[len(blocks)-1].Hash()) - ctx = append(ctx, "lastnumber", blocks[len(blocks)-1].Number()) - } - log.Warn("Unexpected insertion activity", ctx...) - return 0, errors.New("unexpected behavior after transition") - } - // If snap sync is running, deny importing weird blocks. This is a problematic - // clause when starting up a new network, because snap-syncing miners might not - // accept each others' blocks until a restart. Unfortunately we haven't figured - // out a way yet where nodes can decide unilaterally whether the network is new - // or not. This should be fixed if we figure out a solution. - if !h.synced.Load() { - log.Warn("Syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) - return 0, nil - } - if h.merger.TDDReached() { - // The blocks from the p2p network is regarded as untrusted - // after the transition. In theory block gossip should be disabled - // entirely whenever the transition is started. But in order to - // handle the transition boundary reorg in the consensus-layer, - // the legacy blocks are still accepted, but only for the terminal - // pow blocks. Spec: https://github.com/ethereum/EIPs/blob/master/EIPS/eip-3675.md#halt-the-importing-of-pow-blocks - for i, block := range blocks { - ptd := h.chain.GetTd(block.ParentHash(), block.NumberU64()-1) - if ptd == nil { - return 0, nil - } - td := new(big.Int).Add(ptd, block.Difficulty()) - if !h.chain.Config().IsTerminalPoWBlock(ptd, td) { - log.Info("Filtered out non-terminal pow block", "number", block.NumberU64(), "hash", block.Hash()) - return 0, nil - } - if err := h.chain.InsertBlockWithoutSetHead(block); err != nil { - return i, err - } - } - return 0, nil - } - return h.chain.InsertChain(blocks) - } - h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, h.BroadcastBlock, heighter, nil, inserter, h.removePeer) fetchTx := func(peer string, hashes []common.Hash) error { p := h.peers.peer(peer) @@ -282,7 +194,6 @@ func newHandler(config *handlerConfig) (*handler, error) { return h.txpool.Add(txs, false, false) } h.txFetcher = fetcher.NewTxFetcher(h.txpool.Has, addTxs, fetchTx, h.removePeer) - h.chainSync = newChainSyncer(h) return h, nil } @@ -392,8 +303,6 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { return err } } - h.chainSync.handlePeerEvent() - // Propagate existing transactions. new transactions appearing // after this will be sent via broadcasts. h.syncTransactions(peer) @@ -520,14 +429,8 @@ func (h *handler) Start(maxPeers int) { h.txsSub = h.txpool.SubscribeTransactions(h.txsCh, false) go h.txBroadcastLoop() - // broadcast mined blocks - h.wg.Add(1) - h.minedBlockSub = h.eventMux.Subscribe(core.NewMinedBlockEvent{}) - go h.minedBroadcastLoop() - // start sync handlers - h.wg.Add(1) - go h.chainSync.loop() + h.txFetcher.Start() // start peer handler tracker h.wg.Add(1) @@ -535,8 +438,9 @@ func (h *handler) Start(maxPeers int) { } func (h *handler) Stop() { - h.txsSub.Unsubscribe() // quits txBroadcastLoop - h.minedBlockSub.Unsubscribe() // quits blockBroadcastLoop + h.txsSub.Unsubscribe() // quits txBroadcastLoop + h.txFetcher.Stop() + h.downloader.Terminate() // Quit chainSync and txsync64. // After this is done, no new peers will be accepted. @@ -552,50 +456,6 @@ func (h *handler) Stop() { log.Info("Ethereum protocol stopped") } -// BroadcastBlock will either propagate a block to a subset of its peers, or -// will only announce its availability (depending what's requested). -func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { - // Disable the block propagation if the chain has already entered the PoS - // stage. The block propagation is delegated to the consensus layer. - if h.merger.PoSFinalized() { - return - } - // Disable the block propagation if it's the post-merge block. - if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok { - if beacon.IsPoSHeader(block.Header()) { - return - } - } - hash := block.Hash() - peers := h.peers.peersWithoutBlock(hash) - - // If propagation is requested, send to a subset of the peer - if propagate { - // Calculate the TD of the block (it's not imported yet, so block.Td is not valid) - var td *big.Int - if parent := h.chain.GetBlock(block.ParentHash(), block.NumberU64()-1); parent != nil { - td = new(big.Int).Add(block.Difficulty(), h.chain.GetTd(block.ParentHash(), block.NumberU64()-1)) - } else { - log.Error("Propagating dangling block", "number", block.Number(), "hash", hash) - return - } - // Send the block to a subset of our peers - transfer := peers[:int(math.Sqrt(float64(len(peers))))] - for _, peer := range transfer { - peer.AsyncSendNewBlock(block, td) - } - log.Trace("Propagated block", "hash", hash, "recipients", len(transfer), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) - return - } - // Otherwise if the block is indeed in out own chain, announce it - if h.chain.HasBlock(hash, block.NumberU64()) { - for _, peer := range peers { - peer.AsyncSendNewBlockHash(block) - } - log.Trace("Announced block", "hash", hash, "recipients", len(peers), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) - } -} - // BroadcastTransactions will propagate a batch of transactions // - To a square root of all peers for non-blob transactions // - And, separately, as announcements to all peers which are not known to @@ -614,25 +474,54 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { annos = make(map[*ethPeer][]common.Hash) // Set peer->hash to announce ) // Broadcast transactions to a batch of peers not knowing about it - for _, tx := range txs { - peers := h.peers.peersWithoutTransaction(tx.Hash()) + direct := big.NewInt(int64(math.Sqrt(float64(h.peers.len())))) // Approximate number of peers to broadcast to + if direct.BitLen() == 0 { + direct = big.NewInt(1) + } + total := new(big.Int).Exp(direct, big.NewInt(2), nil) // Stabilise total peer count a bit based on sqrt peers - var numDirect int + var ( + signer = types.LatestSignerForChainID(h.chain.Config().ChainID) // Don't care about chain status, we just need *a* sender + hasher = sha3.NewLegacyKeccak256().(crypto.KeccakState) + hash = make([]byte, 32) + ) + for _, tx := range txs { + var maybeDirect bool switch { case tx.Type() == types.BlobTxType: blobTxs++ case tx.Size() > txMaxBroadcastSize: largeTxs++ default: - numDirect = int(math.Sqrt(float64(len(peers)))) + maybeDirect = true } - // Send the tx unconditionally to a subset of our peers - for _, peer := range peers[:numDirect] { - txset[peer] = append(txset[peer], tx.Hash()) - } - // For the remaining peers, send announcement only - for _, peer := range peers[numDirect:] { - annos[peer] = append(annos[peer], tx.Hash()) + // Send the transaction (if it's small enough) directly to a subset of + // the peers that have not received it yet, ensuring that the flow of + // transactions is grouped by account to (try and) avoid nonce gaps. + // + // To do this, we hash the local enode IW with together with a peer's + // enode ID together with the transaction sender and broadcast if + // `sha(self, peer, sender) mod peers < sqrt(peers)`. + for _, peer := range h.peers.peersWithoutTransaction(tx.Hash()) { + var broadcast bool + if maybeDirect { + hasher.Reset() + hasher.Write(h.nodeID.Bytes()) + hasher.Write(peer.Node().ID().Bytes()) + + from, _ := types.Sender(signer, tx) // Ignore error, we only use the addr as a propagation target splitter + hasher.Write(from.Bytes()) + + hasher.Read(hash) + if new(big.Int).Mod(new(big.Int).SetBytes(hash), total).Cmp(direct) < 0 { + broadcast = true + } + } + if broadcast { + txset[peer] = append(txset[peer], tx.Hash()) + } else { + annos[peer] = append(annos[peer], tx.Hash()) + } } } for peer, hashes := range txset { @@ -649,18 +538,6 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { "bcastpeers", directPeers, "bcastcount", directCount, "annpeers", annPeers, "anncount", annCount) } -// minedBroadcastLoop sends mined blocks to connected peers. -func (h *handler) minedBroadcastLoop() { - defer h.wg.Done() - - for obj := range h.minedBlockSub.Chan() { - if ev, ok := obj.Data.(core.NewMinedBlockEvent); ok { - h.BroadcastBlock(ev.Block, true) // First propagate block to peers - h.BroadcastBlock(ev.Block, false) // Only then announce to the rest - } - } -} - // txBroadcastLoop announces new transactions to connected peers. func (h *handler) txBroadcastLoop() { defer h.wg.Done() diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 2a839f615f..b2cd52a221 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -19,10 +19,7 @@ package eth import ( "errors" "fmt" - "math/big" - "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -60,17 +57,7 @@ func (h *ethHandler) AcceptTxs() bool { func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Consume any broadcasts and announces, forwarding the rest to the downloader switch packet := packet.(type) { - case *eth.NewBlockHashesPacket: - hashes, numbers := packet.Unpack() - return h.handleBlockAnnounces(peer, hashes, numbers) - - case *eth.NewBlockPacket: - return h.handleBlockBroadcast(peer, packet.Block, packet.TD) - - case *eth.NewPooledTransactionHashesPacket67: - return h.txFetcher.Notify(peer.ID(), nil, nil, *packet) - - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes) case *eth.TransactionsPacket: @@ -88,55 +75,3 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { return fmt.Errorf("unexpected eth packet type: %T", packet) } } - -// handleBlockAnnounces is invoked from a peer's message handler when it transmits a -// batch of block announcements for the local node to process. -func (h *ethHandler) handleBlockAnnounces(peer *eth.Peer, hashes []common.Hash, numbers []uint64) error { - // Drop all incoming block announces from the p2p network if - // the chain already entered the pos stage and disconnect the - // remote peer. - if h.merger.PoSFinalized() { - return errors.New("disallowed block announcement") - } - // Schedule all the unknown hashes for retrieval - var ( - unknownHashes = make([]common.Hash, 0, len(hashes)) - unknownNumbers = make([]uint64, 0, len(numbers)) - ) - for i := 0; i < len(hashes); i++ { - if !h.chain.HasBlock(hashes[i], numbers[i]) { - unknownHashes = append(unknownHashes, hashes[i]) - unknownNumbers = append(unknownNumbers, numbers[i]) - } - } - for i := 0; i < len(unknownHashes); i++ { - h.blockFetcher.Notify(peer.ID(), unknownHashes[i], unknownNumbers[i], time.Now(), peer.RequestOneHeader, peer.RequestBodies) - } - return nil -} - -// handleBlockBroadcast is invoked from a peer's message handler when it transmits a -// block broadcast for the local node to process. -func (h *ethHandler) handleBlockBroadcast(peer *eth.Peer, block *types.Block, td *big.Int) error { - // Drop all incoming block announces from the p2p network if - // the chain already entered the pos stage and disconnect the - // remote peer. - if h.merger.PoSFinalized() { - return errors.New("disallowed block broadcast") - } - // Schedule the block for import - h.blockFetcher.Enqueue(peer.ID(), block) - - // Assuming the block is importable by the peer, but possibly not yet done so, - // calculate the head hash and TD that the peer truly must have. - var ( - trueHead = block.ParentHash() - trueTD = new(big.Int).Sub(td, block.Difficulty()) - ) - // Update the peer's total difficulty if better than the previous - if _, td := peer.Head(); trueTD.Cmp(td) > 0 { - peer.SetHead(trueHead, trueTD) - h.chainSync.handlePeerEvent() - } - return nil -} diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index bb342acc18..297a6bf154 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" @@ -58,11 +57,7 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { h.blockBroadcasts.Send(packet.Block) return nil - case *eth.NewPooledTransactionHashesPacket67: - h.txAnnounces.Send(([]common.Hash)(*packet)) - return nil - - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: h.txAnnounces.Send(packet.Hashes) return nil @@ -81,7 +76,6 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. -func TestForkIDSplit67(t *testing.T) { testForkIDSplit(t, eth.ETH67) } func TestForkIDSplit68(t *testing.T) { testForkIDSplit(t, eth.ETH68) } func testForkIDSplit(t *testing.T, protocol uint) { @@ -114,7 +108,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { Database: dbNoFork, Chain: chainNoFork, TxPool: newTestTxPool(), - Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FullSync, BloomCache: 1, @@ -123,7 +116,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { Database: dbProFork, Chain: chainProFork, TxPool: newTestTxPool(), - Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FullSync, BloomCache: 1, @@ -236,7 +228,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. -func TestRecvTransactions67(t *testing.T) { testRecvTransactions(t, eth.ETH67) } func TestRecvTransactions68(t *testing.T) { testRecvTransactions(t, eth.ETH68) } func testRecvTransactions(t *testing.T, protocol uint) { @@ -294,7 +285,6 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. -func TestSendTransactions67(t *testing.T) { testSendTransactions(t, eth.ETH67) } func TestSendTransactions68(t *testing.T) { testSendTransactions(t, eth.ETH68) } func testSendTransactions(t *testing.T, protocol uint) { @@ -353,7 +343,7 @@ func testSendTransactions(t *testing.T, protocol uint) { seen := make(map[common.Hash]struct{}) for len(seen) < len(insert) { switch protocol { - case 67, 68: + case 68: select { case hashes := <-anns: for _, hash := range hashes { @@ -379,7 +369,6 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. -func TestTransactionPropagation67(t *testing.T) { testTransactionPropagation(t, eth.ETH67) } func TestTransactionPropagation68(t *testing.T) { testTransactionPropagation(t, eth.ETH68) } func testTransactionPropagation(t *testing.T, protocol uint) { @@ -449,160 +438,3 @@ func testTransactionPropagation(t *testing.T, protocol uint) { } } } - -// Tests that blocks are broadcast to a sqrt number of peers only. -func TestBroadcastBlock1Peer(t *testing.T) { testBroadcastBlock(t, 1, 1) } -func TestBroadcastBlock2Peers(t *testing.T) { testBroadcastBlock(t, 2, 1) } -func TestBroadcastBlock3Peers(t *testing.T) { testBroadcastBlock(t, 3, 1) } -func TestBroadcastBlock4Peers(t *testing.T) { testBroadcastBlock(t, 4, 2) } -func TestBroadcastBlock5Peers(t *testing.T) { testBroadcastBlock(t, 5, 2) } -func TestBroadcastBlock8Peers(t *testing.T) { testBroadcastBlock(t, 9, 3) } -func TestBroadcastBlock12Peers(t *testing.T) { testBroadcastBlock(t, 12, 3) } -func TestBroadcastBlock16Peers(t *testing.T) { testBroadcastBlock(t, 16, 4) } -func TestBroadcastBloc26Peers(t *testing.T) { testBroadcastBlock(t, 26, 5) } -func TestBroadcastBlock100Peers(t *testing.T) { testBroadcastBlock(t, 100, 10) } - -func testBroadcastBlock(t *testing.T, peers, bcasts int) { - t.Parallel() - - // Create a source handler to broadcast blocks from and a number of sinks - // to receive them. - source := newTestHandlerWithBlocks(1) - defer source.close() - - sinks := make([]*testEthHandler, peers) - for i := 0; i < len(sinks); i++ { - sinks[i] = new(testEthHandler) - } - // Interconnect all the sink handlers with the source handler - var ( - genesis = source.chain.Genesis() - td = source.chain.GetTd(genesis.Hash(), genesis.NumberU64()) - ) - for i, sink := range sinks { - sink := sink // Closure for gorotuine below - - sourcePipe, sinkPipe := p2p.MsgPipe() - defer sourcePipe.Close() - defer sinkPipe.Close() - - sourcePeer := eth.NewPeer(eth.ETH67, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH67, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) - defer sourcePeer.Close() - defer sinkPeer.Close() - - go source.handler.runEthPeer(sourcePeer, func(peer *eth.Peer) error { - return eth.Handle((*ethHandler)(source.handler), peer) - }) - if err := sinkPeer.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain)); err != nil { - t.Fatalf("failed to run protocol handshake") - } - go eth.Handle(sink, sinkPeer) - } - // Subscribe to all the transaction pools - blockChs := make([]chan *types.Block, len(sinks)) - for i := 0; i < len(sinks); i++ { - blockChs[i] = make(chan *types.Block, 1) - defer close(blockChs[i]) - - sub := sinks[i].blockBroadcasts.Subscribe(blockChs[i]) - defer sub.Unsubscribe() - } - // Initiate a block propagation across the peers - time.Sleep(100 * time.Millisecond) - header := source.chain.CurrentBlock() - source.handler.BroadcastBlock(source.chain.GetBlock(header.Hash(), header.Number.Uint64()), true) - - // Iterate through all the sinks and ensure the correct number got the block - done := make(chan struct{}, peers) - for _, ch := range blockChs { - ch := ch - go func() { - <-ch - done <- struct{}{} - }() - } - var received int - for { - select { - case <-done: - received++ - - case <-time.After(100 * time.Millisecond): - if received != bcasts { - t.Errorf("broadcast count mismatch: have %d, want %d", received, bcasts) - } - return - } - } -} - -// Tests that a propagated malformed block (uncles or transactions don't match -// with the hashes in the header) gets discarded and not broadcast forward. -func TestBroadcastMalformedBlock67(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH67) } -func TestBroadcastMalformedBlock68(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH68) } - -func testBroadcastMalformedBlock(t *testing.T, protocol uint) { - t.Parallel() - - // Create a source handler to broadcast blocks from and a number of sinks - // to receive them. - source := newTestHandlerWithBlocks(1) - defer source.close() - - // Create a source handler to send messages through and a sink peer to receive them - p2pSrc, p2pSink := p2p.MsgPipe() - defer p2pSrc.Close() - defer p2pSink.Close() - - src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, source.txpool) - sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, source.txpool) - defer src.Close() - defer sink.Close() - - go source.handler.runEthPeer(src, func(peer *eth.Peer) error { - return eth.Handle((*ethHandler)(source.handler), peer) - }) - // Run the handshake locally to avoid spinning up a sink handler - var ( - genesis = source.chain.Genesis() - td = source.chain.GetTd(genesis.Hash(), genesis.NumberU64()) - ) - if err := sink.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain)); err != nil { - t.Fatalf("failed to run protocol handshake") - } - // After the handshake completes, the source handler should stream the sink - // the blocks, subscribe to inbound network events - backend := new(testEthHandler) - - blocks := make(chan *types.Block, 1) - sub := backend.blockBroadcasts.Subscribe(blocks) - defer sub.Unsubscribe() - - go eth.Handle(backend, sink) - - // Create various combinations of malformed blocks - head := source.chain.CurrentBlock() - block := source.chain.GetBlock(head.Hash(), head.Number.Uint64()) - - malformedUncles := head - malformedUncles.UncleHash[0]++ - malformedTransactions := head - malformedTransactions.TxHash[0]++ - malformedEverything := head - malformedEverything.UncleHash[0]++ - malformedEverything.TxHash[0]++ - - // Try to broadcast all malformations and ensure they all get discarded - for _, header := range []*types.Header{malformedUncles, malformedTransactions, malformedEverything} { - block := types.NewBlockWithHeader(header).WithBody(block.Transactions(), block.Uncles()) - if err := src.SendNewBlock(block, big.NewInt(131136)); err != nil { - t.Fatalf("failed to broadcast block: %v", err) - } - select { - case <-blocks: - t.Fatalf("malformed block forwarded") - case <-time.After(100 * time.Millisecond): - } - } -} diff --git a/eth/handler_test.go b/eth/handler_test.go index 6d6132ee4c..bcc8ea30e4 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -22,7 +22,6 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -34,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) var ( @@ -92,7 +92,7 @@ func (p *testTxPool) Add(txs []*types.Transaction, local bool, sync bool) []erro } // Pending returns all the transactions known to the pool -func (p *testTxPool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTransaction { +func (p *testTxPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { p.lock.RLock() defer p.lock.RUnlock() @@ -111,8 +111,8 @@ func (p *testTxPool) Pending(enforceTips bool) map[common.Address][]*txpool.Lazy Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) @@ -149,7 +149,7 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { db := rawdb.NewMemoryDatabase() gspec := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, + Alloc: types.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, } chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) @@ -163,7 +163,6 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { Database: db, Chain: chain, TxPool: txpool, - Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.SnapSync, BloomCache: 1, diff --git a/eth/peerset.go b/eth/peerset.go index c0c11e3e85..6b0aff226c 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -19,7 +19,6 @@ package eth import ( "errors" "fmt" - "math/big" "sync" "github.com/ethereum/go-ethereum/common" @@ -100,7 +99,7 @@ func (ps *peerSet) registerSnapExtension(peer *snap.Peer) error { return nil } -// waitExtensions blocks until all satellite protocols are connected and tracked +// waitSnapExtension blocks until all satellite protocols are connected and tracked // by the peerset. func (ps *peerSet) waitSnapExtension(peer *eth.Peer) (*snap.Peer, error) { // If the peer does not support a compatible `snap`, don't wait @@ -192,21 +191,6 @@ func (ps *peerSet) peer(id string) *ethPeer { return ps.peers[id] } -// peersWithoutBlock retrieves a list of peers that do not have a given block in -// their set of known hashes so it might be propagated to them. -func (ps *peerSet) peersWithoutBlock(hash common.Hash) []*ethPeer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - list := make([]*ethPeer, 0, len(ps.peers)) - for _, p := range ps.peers { - if !p.KnownBlock(hash) { - list = append(list, p) - } - } - return list -} - // peersWithoutTransaction retrieves a list of peers that do not have a given // transaction in their set of known hashes. func (ps *peerSet) peersWithoutTransaction(hash common.Hash) []*ethPeer { @@ -240,24 +224,6 @@ func (ps *peerSet) snapLen() int { return ps.snapPeers } -// peerWithHighestTD retrieves the known peer with the currently highest total -// difficulty, but below the given PoS switchover threshold. -func (ps *peerSet) peerWithHighestTD() *eth.Peer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - var ( - bestPeer *eth.Peer - bestTd *big.Int - ) - for _, p := range ps.peers { - if _, td := p.Head(); bestPeer == nil || td.Cmp(bestTd) > 0 { - bestPeer, bestTd = p.Peer, td - } - } - return bestPeer -} - // close disconnects all peers. func (ps *peerSet) close() { ps.lock.Lock() diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index 3045303f22..f0ed1d6bc9 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -17,8 +17,6 @@ package eth import ( - "math/big" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" ) @@ -29,37 +27,6 @@ const ( maxTxPacketSize = 100 * 1024 ) -// blockPropagation is a block propagation event, waiting for its turn in the -// broadcast queue. -type blockPropagation struct { - block *types.Block - td *big.Int -} - -// broadcastBlocks is a write loop that multiplexes blocks and block announcements -// to the remote peer. The goal is to have an async writer that does not lock up -// node internals and at the same time rate limits queued data. -func (p *Peer) broadcastBlocks() { - for { - select { - case prop := <-p.queuedBlocks: - if err := p.SendNewBlock(prop.block, prop.td); err != nil { - return - } - p.Log().Trace("Propagated block", "number", prop.block.Number(), "hash", prop.block.Hash(), "td", prop.td) - - case block := <-p.queuedBlockAnns: - if err := p.SendNewBlockHashes([]common.Hash{block.Hash()}, []uint64{block.NumberU64()}); err != nil { - return - } - p.Log().Trace("Announced block", "number", block.Number(), "hash", block.Hash()) - - case <-p.term: - return - } - } -} - // broadcastTransactions is a write loop that schedules transaction broadcasts // to the remote peer. The goal is to have an async writer that does not lock up // node internals and at the same time rate limits queued data. @@ -163,16 +130,9 @@ func (p *Peer) announceTransactions() { if len(pending) > 0 { done = make(chan struct{}) go func() { - if p.version >= ETH68 { - if err := p.sendPooledTransactionHashes68(pending, pendingTypes, pendingSizes); err != nil { - fail <- err - return - } - } else { - if err := p.sendPooledTransactionHashes66(pending); err != nil { - fail <- err - return - } + if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes); err != nil { + fail <- err + return } close(done) p.Log().Trace("Sent transaction announcements", "count", len(pending)) diff --git a/eth/protocols/eth/dispatcher.go b/eth/protocols/eth/dispatcher.go index ae98820cd6..146eec3f60 100644 --- a/eth/protocols/eth/dispatcher.go +++ b/eth/protocols/eth/dispatcher.go @@ -136,7 +136,7 @@ func (p *Peer) dispatchRequest(req *Request) error { } } -// dispatchRequest fulfils a pending request and delivers it to the requested +// dispatchResponse fulfils a pending request and delivers it to the requested // sink. func (p *Peer) dispatchResponse(res *Response, metadata func() interface{}) error { resOp := &response{ diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 42d0412a12..2d69ecdc83 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -93,10 +93,6 @@ type TxPool interface { func MakeProtocols(backend Backend, network uint64, dnsdisc enode.Iterator) []p2p.Protocol { protocols := make([]p2p.Protocol, 0, len(ProtocolVersions)) for _, version := range ProtocolVersions { - // Blob transactions require eth/68 announcements, disable everything else - if version <= ETH67 && backend.Chain().Config().CancunTime != nil { - continue - } version := version // Closure protocols = append(protocols, p2p.Protocol{ @@ -166,26 +162,11 @@ type Decoder interface { Time() time.Time } -var eth67 = map[uint64]msgHandler{ - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes67, - GetBlockHeadersMsg: handleGetBlockHeaders, - BlockHeadersMsg: handleBlockHeaders, - GetBlockBodiesMsg: handleGetBlockBodies, - BlockBodiesMsg: handleBlockBodies, - GetReceiptsMsg: handleGetReceipts, - ReceiptsMsg: handleReceipts, - GetPooledTransactionsMsg: handleGetPooledTransactions, - PooledTransactionsMsg: handlePooledTransactions, -} - var eth68 = map[uint64]msgHandler{ NewBlockHashesMsg: handleNewBlockhashes, NewBlockMsg: handleNewBlock, TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes68, + NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, GetBlockHeadersMsg: handleGetBlockHeaders, BlockHeadersMsg: handleBlockHeaders, GetBlockBodiesMsg: handleGetBlockBodies, @@ -209,10 +190,8 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() - var handlers = eth67 - if peer.Version() >= ETH68 { - handlers = eth68 - } + var handlers = eth68 + // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 41e18bfb3e..fdf551ef21 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -102,7 +102,7 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, generator func(int, gspec := &core.Genesis{ Config: config, - Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(100_000_000_000_000_000)}}, + Alloc: types.GenesisAlloc{testAddr: {Balance: big.NewInt(100_000_000_000_000_000)}}, } chain, _ := core.NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) @@ -117,7 +117,7 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, generator func(int, txconfig.Journal = "" // Don't litter the disk with test journals pool := legacypool.New(txconfig, chain) - txpool, _ := txpool.New(new(big.Int).SetUint64(txconfig.PriceLimit), chain, []txpool.SubPool{pool}) + txpool, _ := txpool.New(txconfig.PriceLimit, chain, []txpool.SubPool{pool}) return &testBackend{ db: db, @@ -150,7 +150,6 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. -func TestGetBlockHeaders67(t *testing.T) { testGetBlockHeaders(t, ETH67) } func TestGetBlockHeaders68(t *testing.T) { testGetBlockHeaders(t, ETH68) } func testGetBlockHeaders(t *testing.T, protocol uint) { @@ -336,7 +335,6 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. -func TestGetBlockBodies67(t *testing.T) { testGetBlockBodies(t, ETH67) } func TestGetBlockBodies68(t *testing.T) { testGetBlockBodies(t, ETH68) } func testGetBlockBodies(t *testing.T, protocol uint) { @@ -431,7 +429,6 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } // Tests that the transaction receipts can be retrieved based on hashes. -func TestGetBlockReceipts67(t *testing.T) { testGetBlockReceipts(t, ETH67) } func TestGetBlockReceipts68(t *testing.T) { testGetBlockReceipts(t, ETH68) } func testGetBlockReceipts(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 069e92dadf..96656afb1b 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -18,6 +18,7 @@ package eth import ( "encoding/json" + "errors" "fmt" "github.com/ethereum/go-ethereum/common" @@ -274,43 +275,11 @@ func ServiceGetReceiptsQuery(chain *core.BlockChain, query GetReceiptsRequest) [ } func handleNewBlockhashes(backend Backend, msg Decoder, peer *Peer) error { - // A batch of new block announcements just arrived - ann := new(NewBlockHashesPacket) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Mark the hashes as present at the remote node - for _, block := range *ann { - peer.markBlock(block.Hash) - } - // Deliver them all to the backend for queuing - return backend.Handle(peer, ann) + return errors.New("block announcements disallowed") // We dropped support for non-merge networks } func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { - // Retrieve and decode the propagated block - ann := new(NewBlockPacket) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - if err := ann.sanityCheck(); err != nil { - return err - } - if hash := types.CalcUncleHash(ann.Block.Uncles()); hash != ann.Block.UncleHash() { - log.Warn("Propagated block has invalid uncles", "have", hash, "exp", ann.Block.UncleHash()) - return nil // TODO(karalabe): return error eventually, but wait a few releases - } - if hash := types.DeriveSha(ann.Block.Transactions(), trie.NewStackTrie(nil)); hash != ann.Block.TxHash() { - log.Warn("Propagated block has invalid body", "have", hash, "exp", ann.Block.TxHash()) - return nil // TODO(karalabe): return error eventually, but wait a few releases - } - ann.Block.ReceivedAt = msg.Time() - ann.Block.ReceivedFrom = peer - - // Mark the peer as owning the block - peer.markBlock(ann.Block.Hash()) - - return backend.Handle(peer, ann) + return errors.New("block broadcasts disallowed") // We dropped support for non-merge networks } func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { @@ -383,30 +352,13 @@ func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { }, metadata) } -func handleNewPooledTransactionHashes67(backend Backend, msg Decoder, peer *Peer) error { +func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) error { // New transaction announcement arrived, make sure we have // a valid and fresh chain to handle them if !backend.AcceptTxs() { return nil } - ann := new(NewPooledTransactionHashesPacket67) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Schedule all the unknown hashes for retrieval - for _, hash := range *ann { - peer.markTransaction(hash) - } - return backend.Handle(peer, ann) -} - -func handleNewPooledTransactionHashes68(backend Backend, msg Decoder, peer *Peer) error { - // New transaction announcement arrived, make sure we have - // a valid and fresh chain to handle them - if !backend.AcceptTxs() { - return nil - } - ann := new(NewPooledTransactionHashesPacket68) + ann := new(NewPooledTransactionHashesPacket) if err := msg.Decode(ann); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index d96cfc8165..b9fd13d863 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -27,7 +27,6 @@ import ( ) // Tests that handshake failures are detected and reported correctly. -func TestHandshake67(t *testing.T) { testHandshake(t, ETH67) } func TestHandshake68(t *testing.T) { testHandshake(t, ETH68) } func testHandshake(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 98ad22a8cf..94f28f240f 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -33,10 +33,6 @@ const ( // before starting to randomly evict them. maxKnownTxs = 32768 - // maxKnownBlocks is the maximum block hashes to keep in the known list - // before starting to randomly evict them. - maxKnownBlocks = 1024 - // maxQueuedTxs is the maximum number of transactions to queue up before dropping // older broadcasts. maxQueuedTxs = 4096 @@ -44,16 +40,6 @@ const ( // maxQueuedTxAnns is the maximum number of transaction announcements to queue up // before dropping older announcements. maxQueuedTxAnns = 4096 - - // maxQueuedBlocks is the maximum number of block propagations to queue up before - // dropping broadcasts. There's not much point in queueing stale blocks, so a few - // that might cover uncles should be enough. - maxQueuedBlocks = 4 - - // maxQueuedBlockAnns is the maximum number of block announcements to queue up before - // dropping broadcasts. Similarly to block propagations, there's no point to queue - // above some healthy uncle limit, so use that. - maxQueuedBlockAnns = 4 ) // max is a helper function which returns the larger of the two given integers. @@ -75,10 +61,6 @@ type Peer struct { head common.Hash // Latest advertised head block hash td *big.Int // Latest advertised head block total difficulty - knownBlocks *knownCache // Set of block hashes known to be known by this peer - queuedBlocks chan *blockPropagation // Queue of blocks to broadcast to the peer - queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer - txpool TxPool // Transaction pool used by the broadcasters for liveness checks knownTxs *knownCache // Set of transaction hashes known to be known by this peer txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests @@ -92,28 +74,24 @@ type Peer struct { lock sync.RWMutex // Mutex protecting the internal fields } -// NewPeer create a wrapper for a network connection and negotiated protocol +// NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Peer { peer := &Peer{ - id: p.ID().String(), - Peer: p, - rw: rw, - version: version, - knownTxs: newKnownCache(maxKnownTxs), - knownBlocks: newKnownCache(maxKnownBlocks), - queuedBlocks: make(chan *blockPropagation, maxQueuedBlocks), - queuedBlockAnns: make(chan *types.Block, maxQueuedBlockAnns), - txBroadcast: make(chan []common.Hash), - txAnnounce: make(chan []common.Hash), - reqDispatch: make(chan *request), - reqCancel: make(chan *cancel), - resDispatch: make(chan *response), - txpool: txpool, - term: make(chan struct{}), + id: p.ID().String(), + Peer: p, + rw: rw, + version: version, + knownTxs: newKnownCache(maxKnownTxs), + txBroadcast: make(chan []common.Hash), + txAnnounce: make(chan []common.Hash), + reqDispatch: make(chan *request), + reqCancel: make(chan *cancel), + resDispatch: make(chan *response), + txpool: txpool, + term: make(chan struct{}), } // Start up all the broadcasters - go peer.broadcastBlocks() go peer.broadcastTransactions() go peer.announceTransactions() go peer.dispatcher() @@ -156,23 +134,11 @@ func (p *Peer) SetHead(hash common.Hash, td *big.Int) { p.td.Set(td) } -// KnownBlock returns whether peer is known to already have a block. -func (p *Peer) KnownBlock(hash common.Hash) bool { - return p.knownBlocks.Contains(hash) -} - // KnownTransaction returns whether peer is known to already have a transaction. func (p *Peer) KnownTransaction(hash common.Hash) bool { return p.knownTxs.Contains(hash) } -// markBlock marks a block as known for the peer, ensuring that the block will -// never be propagated to this particular peer. -func (p *Peer) markBlock(hash common.Hash) { - // If we reached the memory allowance, drop a previously known block hash - p.knownBlocks.Add(hash) -} - // markTransaction marks a transaction as known for the peer, ensuring that it // will never be propagated to this particular peer. func (p *Peer) markTransaction(hash common.Hash) { @@ -210,29 +176,17 @@ func (p *Peer) AsyncSendTransactions(hashes []common.Hash) { } } -// sendPooledTransactionHashes66 sends transaction hashes to the peer and includes -// them in its transaction hash set for future reference. -// -// This method is a helper used by the async transaction announcer. Don't call it -// directly as the queueing (memory) and transmission (bandwidth) costs should -// not be managed directly. -func (p *Peer) sendPooledTransactionHashes66(hashes []common.Hash) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - p.knownTxs.Add(hashes...) - return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket67(hashes)) -} - -// sendPooledTransactionHashes68 sends transaction hashes (tagged with their type +// sendPooledTransactionHashes sends transaction hashes (tagged with their type // and size) to the peer and includes them in its transaction hash set for future // reference. // // This method is a helper used by the async transaction announcer. Don't call it // directly as the queueing (memory) and transmission (bandwidth) costs should // not be managed directly. -func (p *Peer) sendPooledTransactionHashes68(hashes []common.Hash, types []byte, sizes []uint32) error { +func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32) error { // Mark all the transactions as known, but ensure we don't overflow our limits p.knownTxs.Add(hashes...) - return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket68{Types: types, Sizes: sizes, Hashes: hashes}) + return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket{Types: types, Sizes: sizes, Hashes: hashes}) } // AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually @@ -260,55 +214,6 @@ func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs [ }) } -// SendNewBlockHashes announces the availability of a number of blocks through -// a hash notification. -func (p *Peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { - // Mark all the block hashes as known, but ensure we don't overflow our limits - p.knownBlocks.Add(hashes...) - - request := make(NewBlockHashesPacket, len(hashes)) - for i := 0; i < len(hashes); i++ { - request[i].Hash = hashes[i] - request[i].Number = numbers[i] - } - return p2p.Send(p.rw, NewBlockHashesMsg, request) -} - -// AsyncSendNewBlockHash queues the availability of a block for propagation to a -// remote peer. If the peer's broadcast queue is full, the event is silently -// dropped. -func (p *Peer) AsyncSendNewBlockHash(block *types.Block) { - select { - case p.queuedBlockAnns <- block: - // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) - default: - p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash()) - } -} - -// SendNewBlock propagates an entire block to a remote peer. -func (p *Peer) SendNewBlock(block *types.Block, td *big.Int) error { - // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) - return p2p.Send(p.rw, NewBlockMsg, &NewBlockPacket{ - Block: block, - TD: td, - }) -} - -// AsyncSendNewBlock queues an entire block for propagation to a remote peer. If -// the peer's broadcast queue is full, the event is silently dropped. -func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { - select { - case p.queuedBlocks <- &blockPropagation{block: block, td: td}: - // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) - default: - p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash()) - } -} - // ReplyBlockHeadersRLP is the response to GetBlockHeaders. func (p *Peer) ReplyBlockHeadersRLP(id uint64, headers []rlp.RawValue) error { return p2p.Send(p.rw, BlockHeadersMsg, &BlockHeadersRLPPacket{ diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 0f44f83de1..c5cb2dd1dc 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -30,7 +30,6 @@ import ( // Constants to match up protocol versions and messages const ( - ETH67 = 67 ETH68 = 68 ) @@ -40,11 +39,11 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH68, ETH67} +var ProtocolVersions = []uint{ETH68} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH68: 17, ETH67: 17} +var protocolLengths = map[uint]uint64{ETH68: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 @@ -190,19 +189,6 @@ type NewBlockPacket struct { TD *big.Int } -// sanityCheck verifies that the values are reasonable, as a DoS protection -func (request *NewBlockPacket) sanityCheck() error { - if err := request.Block.SanityCheck(); err != nil { - return err - } - //TD at mainnet block #7753254 is 76 bits. If it becomes 100 million times - // larger, it will still fit within 100 bits - if tdlen := request.TD.BitLen(); tdlen > 100 { - return fmt.Errorf("too large block TD: bitlen %d", tdlen) - } - return nil -} - // GetBlockBodiesRequest represents a block body query. type GetBlockBodiesRequest []common.Hash @@ -283,11 +269,8 @@ type ReceiptsRLPPacket struct { ReceiptsRLPResponse } -// NewPooledTransactionHashesPacket67 represents a transaction announcement packet on eth/67. -type NewPooledTransactionHashesPacket67 []common.Hash - -// NewPooledTransactionHashesPacket68 represents a transaction announcement packet on eth/68 and newer. -type NewPooledTransactionHashesPacket68 struct { +// NewPooledTransactionHashesPacket represents a transaction announcement packet on eth/68 and newer. +type NewPooledTransactionHashesPacket struct { Types []byte Sizes []uint32 Hashes []common.Hash @@ -346,10 +329,8 @@ func (*BlockBodiesResponse) Kind() byte { return BlockBodiesMsg } func (*NewBlockPacket) Name() string { return "NewBlock" } func (*NewBlockPacket) Kind() byte { return NewBlockMsg } -func (*NewPooledTransactionHashesPacket67) Name() string { return "NewPooledTransactionHashes" } -func (*NewPooledTransactionHashesPacket67) Kind() byte { return NewPooledTransactionHashesMsg } -func (*NewPooledTransactionHashesPacket68) Name() string { return "NewPooledTransactionHashes" } -func (*NewPooledTransactionHashesPacket68) Kind() byte { return NewPooledTransactionHashesMsg } +func (*NewPooledTransactionHashesPacket) Name() string { return "NewPooledTransactionHashes" } +func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransactionHashesMsg } func (*GetPooledTransactionsRequest) Name() string { return "GetPooledTransactions" } func (*GetPooledTransactionsRequest) Kind() byte { return GetPooledTransactionsMsg } diff --git a/eth/protocols/snap/handler_fuzzing_test.go b/eth/protocols/snap/handler_fuzzing_test.go index daed7ed44a..4e234ad21b 100644 --- a/eth/protocols/snap/handler_fuzzing_test.go +++ b/eth/protocols/snap/handler_fuzzing_test.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -89,7 +90,7 @@ func doFuzz(input []byte, obj interface{}, code int) { var trieRoot common.Hash func getChain() *core.BlockChain { - ga := make(core.GenesisAlloc, 1000) + ga := make(types.GenesisAlloc, 1000) var a = make([]byte, 20) var mkStorage = func(k, v int) (common.Hash, common.Hash) { var kB = make([]byte, 32) @@ -105,7 +106,7 @@ func getChain() *core.BlockChain { } for i := 0; i < 1000; i++ { binary.LittleEndian.PutUint64(a, uint64(i+0xff)) - acc := core.GenesisAccount{Balance: big.NewInt(int64(i))} + acc := types.Account{Balance: big.NewInt(int64(i))} if i%2 == 1 { acc.Storage = storage } diff --git a/eth/protocols/snap/metrics.go b/eth/protocols/snap/metrics.go index a7d071953f..a8dc2b5824 100644 --- a/eth/protocols/snap/metrics.go +++ b/eth/protocols/snap/metrics.go @@ -17,7 +17,7 @@ package snap import ( - metrics "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/metrics" ) var ( diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index 3db6e22cbd..c57931678c 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -33,7 +33,7 @@ type Peer struct { logger log.Logger // Contextual logger with the peer id injected } -// NewPeer create a wrapper for a network connection and negotiated protocol +// NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { id := p.ID().String() @@ -46,7 +46,7 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { } } -// NewFakePeer create a fake snap peer without a backing p2p peer, for testing purposes. +// NewFakePeer creates a fake snap peer without a backing p2p peer, for testing purposes. func NewFakePeer(version uint, id string, rw p2p.MsgReadWriter) *Peer { return &Peer{ id: id, diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 73d61c2ffd..cea83aa2bc 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -36,8 +36,9 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/testutil" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" "github.com/holiman/uint256" "golang.org/x/crypto/sha3" "golang.org/x/exp/slices" @@ -1501,10 +1502,10 @@ func getCodeByHash(hash common.Hash) []byte { return nil } -// makeAccountTrieNoStorage spits out a trie, along with the leafs +// makeAccountTrieNoStorage spits out a trie, along with the leaves func makeAccountTrieNoStorage(n int, scheme string) (string, *trie.Trie, []*kv) { var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) + db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries []*kv ) @@ -1539,7 +1540,7 @@ func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) { entries []*kv boundaries []common.Hash - db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) + db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) ) // Initialize boundaries @@ -1597,7 +1598,7 @@ func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) { // has a unique storage set. func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots int, code bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) + db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries []*kv storageRoots = make(map[common.Hash]common.Hash) @@ -1649,10 +1650,10 @@ func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots return db.Scheme(), accTrie, entries, storageTries, storageEntries } -// makeAccountTrieWithStorage spits out a trie, along with the leafs +// makeAccountTrieWithStorage spits out a trie, along with the leaves func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, boundary bool, uneven bool) (*trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) + db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries []*kv storageRoots = make(map[common.Hash]common.Hash) @@ -1725,7 +1726,7 @@ func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, bounda // makeStorageTrieWithSeed fills a storage trie with n items, returning the // not-yet-committed trie and the sorted entries. The seeds can be used to ensure // that tries are unique. -func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trienode.NodeSet, []*kv) { +func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *triedb.Database) (common.Hash, *trienode.NodeSet, []*kv) { trie, _ := trie.New(trie.StorageTrieID(types.EmptyRootHash, owner, types.EmptyRootHash), db) var entries []*kv for i := uint64(1); i <= n; i++ { @@ -1748,7 +1749,7 @@ func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Databas // makeBoundaryStorageTrie constructs a storage trie. Instead of filling // storage slots normally, this function will fill a few slots which have // boundary hash. -func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (common.Hash, *trienode.NodeSet, []*kv) { +func makeBoundaryStorageTrie(owner common.Hash, n int, db *triedb.Database) (common.Hash, *trienode.NodeSet, []*kv) { var ( entries []*kv boundaries []common.Hash @@ -1798,7 +1799,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo // makeUnevenStorageTrie constructs a storage tries will states distributed in // different range unevenly. -func makeUnevenStorageTrie(owner common.Hash, slots int, db *trie.Database) (common.Hash, *trienode.NodeSet, []*kv) { +func makeUnevenStorageTrie(owner common.Hash, slots int, db *triedb.Database) (common.Hash, *trienode.NodeSet, []*kv) { var ( entries []*kv tr, _ = trie.New(trie.StorageTrieID(types.EmptyRootHash, owner, types.EmptyRootHash), db) @@ -1830,7 +1831,7 @@ func makeUnevenStorageTrie(owner common.Hash, slots int, db *trie.Database) (com func verifyTrie(scheme string, db ethdb.KeyValueStore, root common.Hash, t *testing.T) { t.Helper() - triedb := trie.NewDatabase(rawdb.NewDatabase(db), newDbConfig(scheme)) + triedb := triedb.NewDatabase(rawdb.NewDatabase(db), newDbConfig(scheme)) accTrie, err := trie.New(trie.StateTrieID(root), triedb) if err != nil { t.Fatal(err) @@ -1967,9 +1968,9 @@ func TestSlotEstimation(t *testing.T) { } } -func newDbConfig(scheme string) *trie.Config { +func newDbConfig(scheme string) *triedb.Config { if scheme == rawdb.HashScheme { - return &trie.Config{} + return &triedb.Config{} } - return &trie.Config{PathDB: pathdb.Defaults} + return &triedb.Config{PathDB: pathdb.Defaults} } diff --git a/eth/state_accessor.go b/eth/state_accessor.go index cf00e582bc..770532cbfe 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) // noopReleaser is returned in case there is no operation expected @@ -41,7 +42,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u var ( current *types.Block database state.Database - triedb *trie.Database + tdb *triedb.Database report = true origin = block.NumberU64() ) @@ -67,14 +68,14 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u // the internal junks created by tracing will be persisted into the disk. // TODO(rjl493456442), clean cache is disabled to prevent memory leak, // please re-enable it for better performance. - database = state.NewDatabaseWithConfig(eth.chainDb, trie.HashDefaults) + database = state.NewDatabaseWithConfig(eth.chainDb, triedb.HashDefaults) if statedb, err = state.New(block.Root(), database, nil); err == nil { log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number()) return statedb, noopReleaser, nil } } // The optional base statedb is given, mark the start point as parent block - statedb, database, triedb, report = base, base.Database(), base.Database().TrieDB(), false + statedb, database, tdb, report = base, base.Database(), base.Database().TrieDB(), false current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) } else { // Otherwise, try to reexec blocks until we find a state or reach our limit @@ -84,8 +85,8 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u // the internal junks created by tracing will be persisted into the disk. // TODO(rjl493456442), clean cache is disabled to prevent memory leak, // please re-enable it for better performance. - triedb = trie.NewDatabase(eth.chainDb, trie.HashDefaults) - database = state.NewDatabaseWithNodeDB(eth.chainDb, triedb) + tdb = triedb.NewDatabase(eth.chainDb, triedb.HashDefaults) + database = state.NewDatabaseWithNodeDB(eth.chainDb, tdb) // If we didn't check the live database, do check state over ephemeral database, // otherwise we would rewind past a persisted block (specific corner case is @@ -161,17 +162,17 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u } // Hold the state reference and also drop the parent state // to prevent accumulating too many nodes in memory. - triedb.Reference(root, common.Hash{}) + tdb.Reference(root, common.Hash{}) if parent != (common.Hash{}) { - triedb.Dereference(parent) + tdb.Dereference(parent) } parent = root } if report { - _, nodes, imgs := triedb.Size() // all memory is contained within the nodes return in hashdb + _, nodes, imgs := tdb.Size() // all memory is contained within the nodes return in hashdb log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) } - return statedb, func() { triedb.Dereference(block.Root()) }, nil + return statedb, func() { tdb.Dereference(block.Root()) }, nil } func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), error) { diff --git a/eth/sync.go b/eth/sync.go index c2a0f453bf..61f2b2b376 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -17,26 +17,15 @@ package eth import ( - "errors" - "math/big" - "time" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/log" -) - -const ( - forceSyncCycle = 10 * time.Second // Time interval to force syncs, even if few peers are available - defaultMinSyncPeers = 5 // Amount of peers desired to start syncing ) // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { var hashes []common.Hash - for _, batch := range h.txpool.Pending(false) { + for _, batch := range h.txpool.Pending(txpool.PendingFilter{OnlyPlainTxs: true}) { for _, tx := range batch { hashes = append(hashes, tx.Hash) } @@ -46,206 +35,3 @@ func (h *handler) syncTransactions(p *eth.Peer) { } p.AsyncSendPooledTransactionHashes(hashes) } - -// chainSyncer coordinates blockchain sync components. -type chainSyncer struct { - handler *handler - force *time.Timer - forced bool // true when force timer fired - warned time.Time - peerEventCh chan struct{} - doneCh chan error // non-nil when sync is running -} - -// chainSyncOp is a scheduled sync operation. -type chainSyncOp struct { - mode downloader.SyncMode - peer *eth.Peer - td *big.Int - head common.Hash -} - -// newChainSyncer creates a chainSyncer. -func newChainSyncer(handler *handler) *chainSyncer { - return &chainSyncer{ - handler: handler, - peerEventCh: make(chan struct{}), - } -} - -// handlePeerEvent notifies the syncer about a change in the peer set. -// This is called for new peers and every time a peer announces a new -// chain head. -func (cs *chainSyncer) handlePeerEvent() bool { - select { - case cs.peerEventCh <- struct{}{}: - return true - case <-cs.handler.quitSync: - return false - } -} - -// loop runs in its own goroutine and launches the sync when necessary. -func (cs *chainSyncer) loop() { - defer cs.handler.wg.Done() - - cs.handler.blockFetcher.Start() - cs.handler.txFetcher.Start() - defer cs.handler.blockFetcher.Stop() - defer cs.handler.txFetcher.Stop() - defer cs.handler.downloader.Terminate() - - // The force timer lowers the peer count threshold down to one when it fires. - // This ensures we'll always start sync even if there aren't enough peers. - cs.force = time.NewTimer(forceSyncCycle) - defer cs.force.Stop() - - for { - if op := cs.nextSyncOp(); op != nil { - cs.startSync(op) - } - select { - case <-cs.peerEventCh: - // Peer information changed, recheck. - case err := <-cs.doneCh: - cs.doneCh = nil - cs.force.Reset(forceSyncCycle) - cs.forced = false - - // If we've reached the merge transition but no beacon client is available, or - // it has not yet switched us over, keep warning the user that their infra is - // potentially flaky. - if errors.Is(err, downloader.ErrMergeTransition) && time.Since(cs.warned) > 10*time.Second { - log.Warn("Local chain is post-merge, waiting for beacon client sync switch-over...") - cs.warned = time.Now() - } - case <-cs.force.C: - cs.forced = true - - case <-cs.handler.quitSync: - // Disable all insertion on the blockchain. This needs to happen before - // terminating the downloader because the downloader waits for blockchain - // inserts, and these can take a long time to finish. - cs.handler.chain.StopInsert() - cs.handler.downloader.Terminate() - if cs.doneCh != nil { - <-cs.doneCh - } - return - } - } -} - -// nextSyncOp determines whether sync is required at this time. -func (cs *chainSyncer) nextSyncOp() *chainSyncOp { - if cs.doneCh != nil { - return nil // Sync already running - } - // If a beacon client once took over control, disable the entire legacy sync - // path from here on end. Note, there is a slight "race" between reaching TTD - // and the beacon client taking over. The downloader will enforce that nothing - // above the first TTD will be delivered to the chain for import. - // - // An alternative would be to check the local chain for exceeding the TTD and - // avoid triggering a sync in that case, but that could also miss sibling or - // other family TTD block being accepted. - if cs.handler.chain.Config().TerminalTotalDifficultyPassed || cs.handler.merger.TDDReached() { - return nil - } - // Ensure we're at minimum peer count. - minPeers := defaultMinSyncPeers - if cs.forced { - minPeers = 1 - } else if minPeers > cs.handler.maxPeers { - minPeers = cs.handler.maxPeers - } - if cs.handler.peers.len() < minPeers { - return nil - } - // We have enough peers, pick the one with the highest TD, but avoid going - // over the terminal total difficulty. Above that we expect the consensus - // clients to direct the chain head to sync to. - peer := cs.handler.peers.peerWithHighestTD() - if peer == nil { - return nil - } - mode, ourTD := cs.modeAndLocalHead() - op := peerToSyncOp(mode, peer) - if op.td.Cmp(ourTD) <= 0 { - // We seem to be in sync according to the legacy rules. In the merge - // world, it can also mean we're stuck on the merge block, waiting for - // a beacon client. In the latter case, notify the user. - if ttd := cs.handler.chain.Config().TerminalTotalDifficulty; ttd != nil && ourTD.Cmp(ttd) >= 0 && time.Since(cs.warned) > 10*time.Second { - log.Warn("Local chain is post-merge, waiting for beacon client sync switch-over...") - cs.warned = time.Now() - } - return nil // We're in sync - } - return op -} - -func peerToSyncOp(mode downloader.SyncMode, p *eth.Peer) *chainSyncOp { - peerHead, peerTD := p.Head() - return &chainSyncOp{mode: mode, peer: p, td: peerTD, head: peerHead} -} - -func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) { - // If we're in snap sync mode, return that directly - if cs.handler.snapSync.Load() { - block := cs.handler.chain.CurrentSnapBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64()) - return downloader.SnapSync, td - } - // We are probably in full sync, but we might have rewound to before the - // snap sync pivot, check if we should re-enable snap sync. - head := cs.handler.chain.CurrentBlock() - if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil { - if head.Number.Uint64() < *pivot { - block := cs.handler.chain.CurrentSnapBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64()) - return downloader.SnapSync, td - } - } - // We are in a full sync, but the associated head state is missing. To complete - // the head state, forcefully rerun the snap sync. Note it doesn't mean the - // persistent state is corrupted, just mismatch with the head block. - if !cs.handler.chain.HasState(head.Root) { - block := cs.handler.chain.CurrentSnapBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64()) - log.Info("Reenabled snap sync as chain is stateless") - return downloader.SnapSync, td - } - // Nope, we're really full syncing - td := cs.handler.chain.GetTd(head.Hash(), head.Number.Uint64()) - return downloader.FullSync, td -} - -// startSync launches doSync in a new goroutine. -func (cs *chainSyncer) startSync(op *chainSyncOp) { - cs.doneCh = make(chan error, 1) - go func() { cs.doneCh <- cs.handler.doSync(op) }() -} - -// doSync synchronizes the local blockchain with a remote peer. -func (h *handler) doSync(op *chainSyncOp) error { - // Run the sync cycle, and disable snap sync if we're past the pivot block - err := h.downloader.LegacySync(op.peer.ID(), op.head, op.td, h.chain.Config().TerminalTotalDifficulty, op.mode) - if err != nil { - return err - } - h.enableSyncedFeatures() - - head := h.chain.CurrentBlock() - if head.Number.Uint64() > 0 { - // We've completed a sync cycle, notify all peers of new state. This path is - // essential in star-topology networks where a gateway node needs to notify - // all its out-of-date peers of the availability of a new block. This failure - // scenario will most often crop up in private and hackathon networks with - // degenerate connectivity, but it should be healthy for the mainnet too to - // more reliably update peers or the local TD state. - if block := h.chain.GetBlock(head.Hash(), head.Number.Uint64()); block != nil { - h.BroadcastBlock(block, false) - } - } - return nil -} diff --git a/eth/sync_test.go b/eth/sync_test.go index d26cbb66ea..7ede0a82c5 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,7 +28,6 @@ import ( ) // Tests that snap sync is disabled after a successful sync cycle. -func TestSnapSyncDisabling67(t *testing.T) { testSnapSyncDisabling(t, eth.ETH67, snap.SNAP1) } func TestSnapSyncDisabling68(t *testing.T) { testSnapSyncDisabling(t, eth.ETH68, snap.SNAP1) } // Tests that snap sync gets disabled as soon as a real block is successfully @@ -86,10 +85,11 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) { time.Sleep(250 * time.Millisecond) // Check that snap sync was disabled - op := peerToSyncOp(downloader.SnapSync, empty.handler.peers.peerWithHighestTD()) - if err := empty.handler.doSync(op); err != nil { + if err := empty.handler.downloader.BeaconSync(downloader.SnapSync, full.chain.CurrentBlock(), nil); err != nil { t.Fatal("sync failed:", err) } + empty.handler.enableSyncedFeatures() + if empty.handler.snapSync.Load() { t.Fatalf("snap sync not disabled after successful synchronisation") } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index b9c9938b63..4d9ab2b0ca 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -228,7 +228,7 @@ func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, conf } sub := notifier.CreateSubscription() - resCh := api.traceChain(from, to, config, notifier.Closed()) + resCh := api.traceChain(from, to, config, sub.Err()) go func() { for result := range resCh { notifier.Notify(sub.ID, result) @@ -242,7 +242,7 @@ func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, conf // the end block but excludes the start one. The return value will be one item per // transaction, dependent on the requested tracer. // The tracing procedure should be aborted in case the closed signal is received. -func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed <-chan interface{}) chan *blockTraceResult { +func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed <-chan error) chan *blockTraceResult { reexec := defaultTraceReexec if config != nil && config.Reexec != nil { reexec = *config.Reexec @@ -628,7 +628,6 @@ func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, stat var ( txs = block.Transactions() blockHash = block.Hash() - blockCtx = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) signer = types.MakeSigner(api.backend.ChainConfig(), block.Number(), block.Time()) results = make([]*txTraceResult, len(txs)) pend sync.WaitGroup @@ -651,6 +650,11 @@ func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, stat TxIndex: task.index, TxHash: txs[task.index].Hash(), } + // Reconstruct the block context for each transaction + // as the GetHash function of BlockContext is not safe for + // concurrent use. + // See: https://github.com/ethereum/go-ethereum/issues/29114 + blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) res, err := api.traceTx(ctx, txs[task.index], msg, txctx, blockCtx, task.statedb, config) if err != nil { results[task.index] = &txTraceResult{TxHash: txs[task.index].Hash(), Error: err.Error()} @@ -663,6 +667,7 @@ func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, stat // Feed the transactions into the tracers and return var failed error + blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) txloop: for i, tx := range txs { // Send the trace task over for execution @@ -778,16 +783,16 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block // Swap out the noop logger to the standard tracer writer = bufio.NewWriter(dump) vmConf = vm.Config{ - Tracer: logger.NewJSONLogger(&logConfig, writer), + Tracer: logger.NewJSONLogger(&logConfig, writer).Hooks(), EnablePreimageRecording: true, } } // Execute the transaction and flush any traces to disk vmenv := vm.NewEVM(vmctx, txContext, statedb, chainConfig, vmConf) statedb.SetTxContext(tx.Hash(), i) - vmConf.Tracer.CaptureTxStart(vmenv, tx, msg.From) + vmConf.Tracer.OnTxStart(vmenv.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)) - vmConf.Tracer.CaptureTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, err) + vmConf.Tracer.OnTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, err) if writer != nil { writer.Flush() } @@ -921,7 +926,7 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc config.BlockOverrides.Apply(&vmctx) } // Execute the trace - msg, err := args.ToMessage(api.backend.RPCGasCap(), block.BaseFee()) + msg, err := args.ToMessage(api.backend.RPCGasCap(), vmctx.BaseFee) if err != nil { return nil, err } @@ -941,7 +946,7 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc // be tracer dependent. func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *core.Message, txctx *directory.Context, vmctx vm.BlockContext, statedb *state.StateDB, config *TraceConfig) (interface{}, error) { var ( - tracer directory.Tracer + tracer *directory.Tracer err error timeout = defaultTraceTimeout usedGas uint64 @@ -950,15 +955,15 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor config = &TraceConfig{} } // Default tracer is the struct logger - tracer = logger.NewStructLogger(config.Config) + tracer = logger.NewStructLogger(config.Config).Tracer() if config.Tracer != nil { tracer, err = directory.DefaultDirectory.New(*config.Tracer, txctx, config.TracerConfig) if err != nil { return nil, err } } - vmenv := vm.NewEVM(vmctx, vm.TxContext{GasPrice: big.NewInt(0)}, statedb, api.backend.ChainConfig(), vm.Config{Tracer: tracer, NoBaseFee: true}) - statedb.SetLogger(tracer) + vmenv := vm.NewEVM(vmctx, vm.TxContext{GasPrice: big.NewInt(0)}, statedb, api.backend.ChainConfig(), vm.Config{Tracer: tracer.Hooks, NoBaseFee: true}) + statedb.SetLogger(tracer.Hooks) // Define a meaningful timeout of a single transaction trace if config.Timeout != nil { diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index cab1202af3..3254f4961f 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -192,7 +192,7 @@ func TestTraceCall(t *testing.T) { accounts := newAccounts(3) genesis := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, accounts[2].addr: {Balance: big.NewInt(params.Ether)}, @@ -410,7 +410,7 @@ func TestTraceTransaction(t *testing.T) { accounts := newAccounts(2) genesis := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, }, @@ -465,7 +465,7 @@ func TestTraceBlock(t *testing.T) { accounts := newAccounts(3) genesis := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, accounts[2].addr: {Balance: big.NewInt(params.Ether)}, @@ -555,7 +555,7 @@ func TestTracingWithOverrides(t *testing.T) { storageAccount := common.Address{0x13, 37} genesis := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, accounts[2].addr: {Balance: big.NewInt(params.Ether)}, @@ -923,7 +923,7 @@ func TestTraceChain(t *testing.T) { accounts := newAccounts(3) genesis := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, accounts[2].addr: {Balance: big.NewInt(params.Ether)}, diff --git a/eth/tracers/directory/live.go b/eth/tracers/directory/live.go deleted file mode 100644 index dc6927b142..0000000000 --- a/eth/tracers/directory/live.go +++ /dev/null @@ -1,30 +0,0 @@ -package directory - -import ( - "errors" - - "github.com/ethereum/go-ethereum/core" -) - -type ctorFunc func() (core.BlockchainLogger, error) - -// LiveDirectory is the collection of tracers which can be used -// during normal block import operations. -var LiveDirectory = liveDirectory{elems: make(map[string]ctorFunc)} - -type liveDirectory struct { - elems map[string]ctorFunc -} - -// Register registers a tracer constructor by name. -func (d *liveDirectory) Register(name string, f ctorFunc) { - d.elems[name] = f -} - -// New instantiates a tracer by name. -func (d *liveDirectory) New(name string) (core.BlockchainLogger, error) { - if f, ok := d.elems[name]; ok { - return f() - } - return nil, errors.New("not found") -} diff --git a/eth/tracers/directory/live/dir.go b/eth/tracers/directory/live/dir.go new file mode 100644 index 0000000000..cedf81c35e --- /dev/null +++ b/eth/tracers/directory/live/dir.go @@ -0,0 +1,31 @@ +package live + +import ( + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/core/tracing" +) + +type ctorFunc func(config json.RawMessage) (*tracing.Hooks, error) + +// Directory is the collection of tracers which can be used +// during normal block import operations. +var Directory = directory{elems: make(map[string]ctorFunc)} + +type directory struct { + elems map[string]ctorFunc +} + +// Register registers a tracer constructor by name. +func (d *directory) Register(name string, f ctorFunc) { + d.elems[name] = f +} + +// New instantiates a tracer by name. +func (d *directory) New(name string, config json.RawMessage) (*tracing.Hooks, error) { + if f, ok := d.elems[name]; ok { + return f(config) + } + return nil, errors.New("not found") +} diff --git a/eth/tracers/directory/noop.go b/eth/tracers/directory/noop.go index e784b6c846..d0edf12e21 100644 --- a/eth/tracers/directory/noop.go +++ b/eth/tracers/directory/noop.go @@ -21,9 +21,8 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" ) func init() { @@ -35,46 +34,48 @@ func init() { type NoopTracer struct{} // newNoopTracer returns a new noop tracer. -func newNoopTracer(ctx *Context, _ json.RawMessage) (Tracer, error) { - return &NoopTracer{}, nil +func newNoopTracer(ctx *Context, _ json.RawMessage) (*Tracer, error) { + t := &NoopTracer{} + return &Tracer{ + Hooks: &tracing.Hooks{ + OnTxStart: t.OnTxStart, + OnTxEnd: t.OnTxEnd, + OnEnter: t.OnEnter, + OnExit: t.OnExit, + OnOpcode: t.OnOpcode, + OnFault: t.OnFault, + OnGasChange: t.OnGasChange, + OnBalanceChange: t.OnBalanceChange, + OnNonceChange: t.OnNonceChange, + OnCodeChange: t.OnCodeChange, + OnStorageChange: t.OnStorageChange, + OnLog: t.OnLog, + }, + GetResult: t.GetResult, + Stop: t.Stop, + }, nil } -// CaptureStart implements the EVMLogger interface to initialize the tracing operation. -func (t *NoopTracer) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { +func (t *NoopTracer) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { } -// CaptureEnd is called after the call finishes to finalize the tracing. -func (t *NoopTracer) CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) { +func (t *NoopTracer) OnFault(pc uint64, op byte, gas, cost uint64, _ tracing.OpContext, depth int, err error) { } -// CaptureState implements the EVMLogger interface to trace a single step of VM execution. -func (t *NoopTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { +func (t *NoopTracer) OnGasChange(old, new uint64, reason tracing.GasChangeReason) {} + +func (t *NoopTracer) OnEnter(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { } -// CaptureFault implements the EVMLogger interface to trace an execution fault. -func (t *NoopTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, _ *vm.ScopeContext, depth int, err error) { +func (t *NoopTracer) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { } -// CaptureKeccakPreimage is called during the KECCAK256 opcode. -func (t *NoopTracer) CaptureKeccakPreimage(hash common.Hash, data []byte) {} - -// OnGasChange is called when gas is either consumed or refunded. -func (t *NoopTracer) OnGasChange(old, new uint64, reason vm.GasChangeReason) {} - -// CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). -func (t *NoopTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { +func (*NoopTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { } -// CaptureExit is called when EVM exits a scope, even if the scope didn't -// execute any code. -func (t *NoopTracer) CaptureExit(output []byte, gasUsed uint64, err error, reverted bool) { -} +func (*NoopTracer) OnTxEnd(receipt *types.Receipt, err error) {} -func (*NoopTracer) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common.Address) {} - -func (*NoopTracer) CaptureTxEnd(receipt *types.Receipt, err error) {} - -func (*NoopTracer) OnBalanceChange(a common.Address, prev, new *big.Int, reason state.BalanceChangeReason) { +func (*NoopTracer) OnBalanceChange(a common.Address, prev, new *big.Int, reason tracing.BalanceChangeReason) { } func (*NoopTracer) OnNonceChange(a common.Address, prev, new uint64) {} @@ -86,8 +87,6 @@ func (*NoopTracer) OnStorageChange(a common.Address, k, prev, new common.Hash) { func (*NoopTracer) OnLog(log *types.Log) {} -func (*NoopTracer) OnNewAccount(a common.Address, reset bool) {} - // GetResult returns an empty json object. func (t *NoopTracer) GetResult() (json.RawMessage, error) { return json.RawMessage(`{}`), nil diff --git a/eth/tracers/directory/tracers.go b/eth/tracers/directory/tracers.go index b16d9d2fe6..766278974c 100644 --- a/eth/tracers/directory/tracers.go +++ b/eth/tracers/directory/tracers.go @@ -24,8 +24,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/tracing" ) // Context contains some contextual infos for a transaction execution that is not @@ -41,16 +40,15 @@ type Context struct { // for it to be available through the RPC interface. // This involves a method to retrieve results and one to // stop tracing. -type Tracer interface { - vm.EVMLogger - state.StateLogger - GetResult() (json.RawMessage, error) +type Tracer struct { + *tracing.Hooks + GetResult func() (json.RawMessage, error) // Stop terminates execution of the tracer at the first opportune moment. - Stop(err error) + Stop func(err error) } -type ctorFn func(*Context, json.RawMessage) (Tracer, error) -type jsCtorFn func(string, *Context, json.RawMessage) (Tracer, error) +type ctorFn func(*Context, json.RawMessage) (*Tracer, error) +type jsCtorFn func(string, *Context, json.RawMessage) (*Tracer, error) type elem struct { ctor ctorFn @@ -83,7 +81,7 @@ func (d *directory) RegisterJSEval(f jsCtorFn) { // New returns a new instance of a tracer, by iterating through the // registered lookups. Name is either name of an existing tracer // or an arbitrary JS code. -func (d *directory) New(name string, ctx *Context, cfg json.RawMessage) (Tracer, error) { +func (d *directory) New(name string, ctx *Context, cfg json.RawMessage) (*Tracer, error) { if elem, ok := d.elems[name]; ok { return elem.ctor(ctx, cfg) } diff --git a/eth/tracers/directory/util.go b/eth/tracers/directory/util.go index 92de735291..f5a2cd80d3 100644 --- a/eth/tracers/directory/util.go +++ b/eth/tracers/directory/util.go @@ -19,7 +19,7 @@ import ( "errors" "fmt" - "github.com/ethereum/go-ethereum/core/vm" + "github.com/holiman/uint256" ) const ( @@ -28,20 +28,53 @@ const ( // GetMemoryCopyPadded returns offset + size as a new slice. // It zero-pads the slice if it extends beyond memory bounds. -func GetMemoryCopyPadded(m *vm.Memory, offset, size int64) ([]byte, error) { +func GetMemoryCopyPadded(m []byte, offset, size int64) ([]byte, error) { if offset < 0 || size < 0 { return nil, errors.New("offset or size must not be negative") } - if int(offset+size) < m.Len() { // slice fully inside memory - return m.GetCopy(offset, size), nil + length := int64(len(m)) + if offset+size < length { // slice fully inside memory + return memoryCopy(m, offset, size), nil } - paddingNeeded := int(offset+size) - m.Len() + paddingNeeded := offset + size - length if paddingNeeded > memoryPadLimit { return nil, fmt.Errorf("reached limit for padding memory slice: %d", paddingNeeded) } cpy := make([]byte, size) - if overlap := int64(m.Len()) - offset; overlap > 0 { - copy(cpy, m.GetPtr(offset, overlap)) + if overlap := length - offset; overlap > 0 { + copy(cpy, MemoryPtr(m, offset, overlap)) } return cpy, nil } + +func memoryCopy(m []byte, offset, size int64) (cpy []byte) { + if size == 0 { + return nil + } + + if len(m) > int(offset) { + cpy = make([]byte, size) + copy(cpy, m[offset:offset+size]) + + return + } + + return +} + +func MemoryPtr(m []byte, offset, size int64) []byte { + if size == 0 { + return nil + } + + if len(m) > int(offset) { + return m[offset : offset+size] + } + + return nil +} + +// Back returns the n'th item in stack +func StackBack(st []uint256.Int, n int) *uint256.Int { + return &st[len(st)-n-1] +} diff --git a/eth/tracers/directory/util_test.go b/eth/tracers/directory/util_test.go index c72096708d..731eeb832f 100644 --- a/eth/tracers/directory/util_test.go +++ b/eth/tracers/directory/util_test.go @@ -33,14 +33,14 @@ func TestMemCopying(t *testing.T) { {0, 100, 0, "", 0}, // No need to pad (0 size) {100, 50, 100, "", 100}, // Should pad 100-150 {100, 50, 5, "", 5}, // Wanted range fully within memory - {100, -50, 0, "offset or size must not be negative", 0}, // Errror - {0, 1, 1024*1024 + 1, "reached limit for padding memory slice: 1048578", 0}, // Errror - {10, 0, 1024*1024 + 100, "reached limit for padding memory slice: 1048666", 0}, // Errror + {100, -50, 0, "offset or size must not be negative", 0}, // Error + {0, 1, 1024*1024 + 1, "reached limit for padding memory slice: 1048578", 0}, // Error + {10, 0, 1024*1024 + 100, "reached limit for padding memory slice: 1048666", 0}, // Error } { mem := vm.NewMemory() mem.Resize(uint64(tc.memsize)) - cpy, err := GetMemoryCopyPadded(mem, tc.offset, tc.size) + cpy, err := GetMemoryCopyPadded(mem.Data(), tc.offset, tc.size) if want := tc.wantErr; want != "" { if err == nil { t.Fatalf("test %d: want '%v' have no error", i, want) diff --git a/eth/tracers/firehose.go b/eth/tracers/firehose.go index 80e98dd1bf..97a5340ad1 100644 --- a/eth/tracers/firehose.go +++ b/eth/tracers/firehose.go @@ -5,6 +5,7 @@ import ( "cmp" "encoding/base64" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -19,23 +20,21 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/tracers/directory" + "github.com/ethereum/go-ethereum/eth/tracers/directory/live" "github.com/ethereum/go-ethereum/params" pbeth "github.com/ethereum/go-ethereum/pb/sf/ethereum/type/v2" "github.com/ethereum/go-ethereum/rlp" + "github.com/holiman/uint256" "golang.org/x/exp/maps" "golang.org/x/exp/slices" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) -var _ core.BlockchainLogger = (*Firehose)(nil) - var firehoseTracerLogLevel = strings.ToLower(os.Getenv("GETH_FIREHOSE_TRACER_LOG_LEVEL")) var isFirehoseDebugEnabled = firehoseTracerLogLevel == "debug" || firehoseTracerLogLevel == "trace" var isFirehoseTracerEnabled = firehoseTracerLogLevel == "trace" @@ -46,18 +45,55 @@ var emptyCommonHash = common.Hash{} func init() { staticFirehoseChainValidationOnInit() - directory.LiveDirectory.Register("firehose", newFirehoseTracer) + live.Directory.Register("firehose", newFirehoseTracer) } -func newFirehoseTracer() (core.BlockchainLogger, error) { +func newFirehoseTracer(cfg json.RawMessage) (*tracing.Hooks, error) { firehoseDebug("New firehose tracer") - return NewFirehoseLogger(), nil + + var config FirehoseConfig + if len([]byte(cfg)) > 0 { + if err := json.Unmarshal(cfg, &config); err != nil { + return nil, fmt.Errorf("failed to parse Firehose config: %w", err) + } + } + + tracer := NewFirehose() + + return &tracing.Hooks{ + OnBlockchainInit: tracer.OnBlockchainInit, + OnGenesisBlock: tracer.OnGenesisBlock, + OnBlockStart: tracer.OnBlockStart, + OnBlockEnd: tracer.OnBlockEnd, + OnSkippedBlock: tracer.OnSkippedBlock, + + OnTxStart: tracer.OnTxStart, + OnTxEnd: tracer.OnTxEnd, + OnEnter: tracer.OnCallEnter, + OnExit: tracer.OnCallExit, + OnOpcode: tracer.OnOpcode, + OnFault: tracer.OnOpcodeFault, + + OnBalanceChange: tracer.OnBalanceChange, + OnNonceChange: tracer.OnNonceChange, + OnCodeChange: tracer.OnCodeChange, + OnStorageChange: tracer.OnStorageChange, + OnGasChange: tracer.OnGasChange, + OnLog: tracer.OnLog, + }, nil +} + +type FirehoseConfig struct { + // Nothing for now } type Firehose struct { // Global state outputBuffer *bytes.Buffer initSent *atomic.Bool + chainConfig *params.ChainConfig + hasher crypto.KeccakState // Keccak256 hasher instance shared across tracer needs (non-concurrent safe) + hasherBuf common.Hash // Keccak256 hasher result array shared across tracer needs (non-concurrent safe) // Block state block *pbeth.Block @@ -67,7 +103,7 @@ type Firehose struct { blockRules params.Rules // Transaction state - evm *vm.EVM + evm *tracing.VMContext transaction *pbeth.TransactionTrace transactionLogIndex uint32 inSystemCall bool @@ -76,16 +112,18 @@ type Firehose struct { // Call state callStack *CallStack deferredCallState *DeferredCallState - latestCallStartSuicided bool + latestCallEnterSuicided bool } const FirehoseProtocolVersion = "3.0" -func NewFirehoseLogger() *Firehose { +func NewFirehose() *Firehose { return &Firehose{ // Global state outputBuffer: bytes.NewBuffer(make([]byte, 0, 100*1024*1024)), initSent: new(atomic.Bool), + chainConfig: nil, + hasher: crypto.NewKeccakState(), // Block state blockOrdinal: &Ordinal{}, @@ -97,7 +135,7 @@ func NewFirehoseLogger() *Firehose { // Call state callStack: NewCallStack(), deferredCallState: NewDeferredCallState(), - latestCallStartSuicided: false, + latestCallEnterSuicided: false, } } @@ -119,20 +157,33 @@ func (f *Firehose) resetTransaction() { f.inSystemCall = false f.callStack.Reset() - f.latestCallStartSuicided = false + f.latestCallEnterSuicided = false f.deferredCallState.Reset() } -func (f *Firehose) OnBlockStart(b *types.Block, td *big.Int, finalized *types.Header, safe *types.Header, chainConfig *params.ChainConfig) { +func (f *Firehose) OnBlockchainInit(chainConfig *params.ChainConfig) { + f.chainConfig = chainConfig + + if wasNeverSent := f.initSent.CompareAndSwap(false, true); wasNeverSent { + printToFirehose("INIT", FirehoseProtocolVersion, "geth", params.Version) + } else { + f.panicInvalidState("The OnBlockchainInit callback was called more than once") + } +} + +func (f *Firehose) OnBlockStart(event tracing.BlockEvent) { + b := event.Block firehoseDebug("block start number=%d hash=%s", b.NumberU64(), b.Hash()) - f.blockRules = chainConfig.Rules(b.Number(), chainConfig.TerminalTotalDifficultyPassed, b.Time()) + f.ensureBlockChainInit() + + f.blockRules = f.chainConfig.Rules(b.Number(), f.chainConfig.TerminalTotalDifficultyPassed, b.Time()) f.blockIsPrecompiledAddr = getActivePrecompilesChecker(f.blockRules) f.block = &pbeth.Block{ Hash: b.Hash().Bytes(), Number: b.Number().Uint64(), - Header: newBlockHeaderFromChainHeader(b.Header(), firehoseBigIntFromNative(new(big.Int).Add(td, b.Difficulty()))), + Header: newBlockHeaderFromChainHeader(b.Header(), firehoseBigIntFromNative(new(big.Int).Add(event.TD, b.Difficulty()))), Size: b.Size(), // Known Firehose issue: If you fix all known Firehose issue for a new chain, don't forget to bump `Ver` to `4`! Ver: 3, @@ -147,7 +198,26 @@ func (f *Firehose) OnBlockStart(b *types.Block, td *big.Int, finalized *types.He f.blockBaseFee = f.block.Header.BaseFeePerGas.Native() } - f.blockFinality.populateFromChain(finalized) + f.blockFinality.populateFromChain(event.Finalized) +} + +func (f *Firehose) OnSkippedBlock(event tracing.BlockEvent) { + // Blocks that are skipped from blockchain that were knwon and should contain 0 transactions. + // It happened in the past, on Polygon if I recall right, that we missed block because some block + // went in this code path. + // + // See https: //github.com/streamingfast/go-ethereum/blob/a46903cf0cad829479ded66b369017914bf82314/core/blockchain.go#L1797-L1814 + if event.Block.Transactions().Len() > 0 { + panic(fmt.Sprintf("The tracer received an `OnSkippedBlock` block #%d (%s) with transactions (%d), this according to core/blockchain.go should never happen and is an error", + event.Block.NumberU64(), + event.Block.Hash().Hex(), + event.Block.Transactions().Len(), + )) + } + + // Trace the block as normal, worst case the Firehose system will simply discard it at some point + f.OnBlockStart(event) + f.OnBlockEnd(nil) } func getActivePrecompilesChecker(rules params.Rules) func(addr common.Address) bool { @@ -198,7 +268,7 @@ func (f *Firehose) OnBeaconBlockRootEnd() { f.resetTransaction() } -func (f *Firehose) CaptureTxStart(evm *vm.EVM, tx *types.Transaction, from common.Address) { +func (f *Firehose) OnTxStart(evm *tracing.VMContext, tx *types.Transaction, from common.Address) { firehoseDebug("trx start hash=%s type=%d gas=%d input=%s", tx.Hash(), tx.Type(), tx.Gas(), inputView(tx.Data())) f.ensureInBlockAndNotInTrxAndNotInCall() @@ -211,13 +281,13 @@ func (f *Firehose) CaptureTxStart(evm *vm.EVM, tx *types.Transaction, from commo to = *tx.To() } - f.captureTxStart(tx, tx.Hash(), from, to) + f.onTxStart(tx, tx.Hash(), from, to) } -// captureTxStart is used internally a two places, in the normal "tracer" and in the "OnGenesisBlock", +// onTxStart is used internally a two places, in the normal "tracer" and in the "OnGenesisBlock", // we manually pass some override to the `tx` because genesis block has a different way of creating // the transaction that wraps the genesis block. -func (f *Firehose) captureTxStart(tx *types.Transaction, hash common.Hash, from, to common.Address) { +func (f *Firehose) onTxStart(tx *types.Transaction, hash common.Hash, from, to common.Address) { v, r, s := tx.RawSignatureValues() var blobGas *uint64 @@ -248,7 +318,7 @@ func (f *Firehose) captureTxStart(tx *types.Transaction, hash common.Hash, from, } } -func (f *Firehose) CaptureTxEnd(receipt *types.Receipt, err error) { +func (f *Firehose) OnTxEnd(receipt *types.Receipt, err error) { firehoseDebug("trx ending") f.ensureInBlockAndInTrx() @@ -388,31 +458,88 @@ func (f *Firehose) assignOrdinalAndIndexToReceiptLogs() { } } -// CaptureStart implements the EVMLogger interface to initialize the tracing operation. -func (f *Firehose) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { - f.callStart("root", rootCallType(create), from, to, input, gas, value) +// OnCallEnter implements the EVMLogger interface to initialize the tracing operation. +func (f *Firehose) OnCallEnter(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + opCode := vm.OpCode(typ) + + var callType pbeth.CallType + if isRootCall := depth == 0; isRootCall { + callType = rootCallType(opCode == vm.CREATE) + } else { + // The invokation for vm.SELFDESTRUCT is called while already in another call and is recorded specially + // in the Geth tracer and generates `OnEnter/OnExit` callbacks. However in Firehose, self destruction + // simply sets the call as having called suicided so there is no extra call. + // + // So we ignore `OnEnter/OnExit` callbacks for `SELFDESTRUCT` opcode, we ignore it here and set + // a special sentinel variable that will tell `OnExit` to ignore itself. + if opCode == vm.SELFDESTRUCT { + f.ensureInCall() + f.callStack.Peek().Suicide = true + + // The next OnCallExit must be ignored, this variable will make the next OnCallExit to be ignored + f.latestCallEnterSuicided = true + return + } + + callType = callTypeFromOpCode(opCode) + if callType == pbeth.CallType_UNSPECIFIED { + panic(fmt.Errorf("unexpected call type, received OpCode %s but only call related opcode (CALL, CREATE, CREATE2, STATIC, DELEGATECALL and CALLCODE) or SELFDESTRUCT is accepted", opCode)) + } + } + + f.callStart(computeCallSource(depth), callType, from, to, input, gas, value) } -// CaptureEnd is called after the call finishes to finalize the tracing. -func (f *Firehose) CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) { - f.callEnd("root", output, gasUsed, err, reverted) +// OnCallExit is called after the call finishes to finalize the tracing. +func (f *Firehose) OnCallExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { + f.callEnd(computeCallSource(depth), output, gasUsed, err, reverted) } -// CaptureState implements the EVMLogger interface to trace a single step of VM execution. -func (f *Firehose) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { - firehoseTrace("capture state op=%s gas=%d cost=%d, err=%s", op, gas, cost, errorView(err)) +// OnOpcode implements the EVMLogger interface to trace a single step of VM execution. +func (f *Firehose) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { + firehoseTrace("on opcode op=%s gas=%d cost=%d, err=%s", op, gas, cost, errorView(err)) if activeCall := f.callStack.Peek(); activeCall != nil { - f.captureInterpreterStep(activeCall, pc, op, gas, cost, scope, rData, depth, err) + opCode := vm.OpCode(op) + f.captureInterpreterStep(activeCall, pc, opCode, gas, cost, scope, rData, depth, err) - if err == nil && cost > 0 { - if reason, found := opCodeToGasChangeReasonMap[op]; found { + // The rest of the logic expects that a call succeeded, nothing to do more here if the interpreter failed on this OpCode + if err != nil { + return + } + + // The gas change must come first to retain Firehose backward compatibility. Indeed, before Firehose 3.0, + // we had a specific method `OnKeccakPreimage` that was called during the KECCAK256 opcode. However, in + // the new model, we do it through `OnOpcode`. + // + // The gas change recording in the previous Firehose patch was done before calling `OnKeccakPreimage` so + // we must do the same here. + if cost > 0 { + if reason, found := opCodeToGasChangeReasonMap[opCode]; found { activeCall.GasChanges = append(activeCall.GasChanges, f.newGasChange("state", gas, gas-cost, reason)) } } + + if opCode == vm.KECCAK256 { + f.onOpcodeSha3(activeCall, scope.StackData(), Memory(scope.MemoryData())) + } } } +type Memory []byte + +func (m Memory) GetPtr(offset, size int64) []byte { + if size == 0 { + return nil + } + + if len(m) > int(offset) { + return m[offset : offset+size] + } + + return nil +} + var opCodeToGasChangeReasonMap = map[vm.OpCode]pbeth.GasChange_Reason{ vm.CREATE: pbeth.GasChange_REASON_CONTRACT_CREATION, vm.CREATE2: pbeth.GasChange_REASON_CONTRACT_CREATION2, @@ -434,14 +561,48 @@ var opCodeToGasChangeReasonMap = map[vm.OpCode]pbeth.GasChange_Reason{ vm.RETURNDATACOPY: pbeth.GasChange_REASON_RETURN_DATA_COPY, } -// CaptureFault implements the EVMLogger interface to trace an execution fault. -func (f *Firehose) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { +// onOpcodeSha3 is called during the SHA3 (a.k.a KECCAK256) opcode it's known +// in Firehose tracer as Keccak preimages. The preimage is the input data that +// was used to produce the given keccak hash. +func (f *Firehose) onOpcodeSha3(call *pbeth.Call, stack []uint256.Int, memory Memory) { + if call.KeccakPreimages == nil { + call.KeccakPreimages = make(map[string]string) + } + + offset, size := stack[0], stack[1] + preImage := memory.GetPtr(int64(offset.Uint64()), int64(size.Uint64())) + + // We should have exclusive access to the hasher, we can safely reset it. + f.hasher.Reset() + f.hasher.Write(preImage) + f.hasher.Read(f.hasherBuf[:]) + + // Known Firehose issue: It appears the old Firehose instrumentation have a bug + // where when the keccak256 preimage is empty, it is written as "." which is + // completely wrong. + // + // To keep the same behavior, we will write the preimage as a "." when the encoded + // data is an empty string. + // + // For new chain, this code should be remove so that we just keep `hex.EncodeToString(data)`. + encodedData := hex.EncodeToString(preImage) + if encodedData == "" { + encodedData = "." + } + + call.KeccakPreimages[hex.EncodeToString(f.hasherBuf[:])] = encodedData +} + +// OnOpcodeFault implements the EVMLogger interface to trace an execution fault. +func (f *Firehose) OnOpcodeFault(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, depth int, err error) { if activeCall := f.callStack.Peek(); activeCall != nil { - f.captureInterpreterStep(activeCall, pc, op, gas, cost, scope, nil, depth, err) + f.captureInterpreterStep(activeCall, pc, vm.OpCode(op), gas, cost, scope, nil, depth, err) } } -func (f *Firehose) captureInterpreterStep(activeCall *pbeth.Call, pc uint64, op vm.OpCode, gas, cost uint64, _ *vm.ScopeContext, rData []byte, depth int, err error) { +func (f *Firehose) captureInterpreterStep(activeCall *pbeth.Call, pc uint64, op vm.OpCode, gas, cost uint64, _ tracing.OpContext, rData []byte, depth int, err error) { + _, _, _, _, _, _, _ = pc, op, gas, cost, rData, depth, err + // for call, we need to process the executed code here // since in old firehose executed code calculation depends if the code exist if activeCall.CallType == pbeth.CallType_CALL && !activeCall.ExecutedCode { @@ -450,33 +611,6 @@ func (f *Firehose) captureInterpreterStep(activeCall *pbeth.Call, pc uint64, op } } -func (f *Firehose) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { - f.ensureInBlockAndInTrx() - - // The invokation for vm.SELFDESTRUCT is called while already in another call, so we must not check that we are not in a call here - if typ == vm.SELFDESTRUCT { - f.ensureInCall() - f.callStack.Peek().Suicide = true - - // The next CaptureExit must be ignored, this variable will make the next CaptureExit to be ignored - f.latestCallStartSuicided = true - return - } - - callType := callTypeFromOpCode(typ) - if callType == pbeth.CallType_UNSPECIFIED { - panic(fmt.Errorf("unexpected call type, received OpCode %s but only call related opcode (CALL, CREATE, CREATE2, STATIC, DELEGATECALL and CALLCODE) or SELFDESTRUCT is accepted", typ)) - } - - f.callStart("child", callType, from, to, input, gas, value) -} - -// CaptureExit is called when EVM exits a scope, even if the scope didn't -// execute any code. -func (f *Firehose) CaptureExit(output []byte, gasUsed uint64, err error, reverted bool) { - f.callEnd("child", output, gasUsed, err, reverted) -} - func (f *Firehose) callStart(source string, callType pbeth.CallType, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { firehoseDebug("call start source=%s index=%d type=%s input=%s", source, f.callStack.NextIndex(), callType, inputView(input)) f.ensureInBlockAndInTrx() @@ -533,7 +667,7 @@ func (f *Firehose) callStart(source string, callType pbeth.CallType, from common f.callStack.Push(call) } -func (f *Firehose) getExecutedCode(evm *vm.EVM, call *pbeth.Call) bool { +func (f *Firehose) getExecutedCode(evm *tracing.VMContext, call *pbeth.Call) bool { precompile := f.blockIsPrecompiledAddr(common.BytesToAddress(call.Address)) if evm != nil && call.CallType == pbeth.CallType_CALL { @@ -563,14 +697,14 @@ func (f *Firehose) getExecutedCode(evm *vm.EVM, call *pbeth.Call) bool { func (f *Firehose) callEnd(source string, output []byte, gasUsed uint64, err error, reverted bool) { firehoseDebug("call end source=%s index=%d output=%s gasUsed=%d err=%s reverted=%t", source, f.callStack.ActiveIndex(), outputView(output), gasUsed, errorView(err), reverted) - if f.latestCallStartSuicided { + if f.latestCallEnterSuicided { if source != "child" { panic(fmt.Errorf("unexpected source for suicided call end, expected child but got %s, suicide are always produced on a 'child' source", source)) } - // Geth native tracer does a `CaptureEnter(SELFDESTRUCT, ...)/CaptureExit(...)`, we must skip the `CaptureExit` call + // Geth native tracer does a `OnEnter(SELFDESTRUCT, ...)/OnExit(...)`, we must skip the `OnExit` call // in that case because we did not push it on our CallStack. - f.latestCallStartSuicided = false + f.latestCallEnterSuicided = false return } @@ -649,35 +783,20 @@ func (f *Firehose) callEnd(source string, output []byte, gasUsed uint64, err err f.transaction.Calls = append(f.transaction.Calls, call) } -// CaptureKeccakPreimage is called during the KECCAK256 opcode. -func (f *Firehose) CaptureKeccakPreimage(hash common.Hash, data []byte) { - f.ensureInBlockAndInTrxAndInCall() - - activeCall := f.callStack.Peek() - if activeCall.KeccakPreimages == nil { - activeCall.KeccakPreimages = make(map[string]string) +func computeCallSource(depth int) string { + if depth == 0 { + return "root" } - // Known Firehose issue: It appears the old Firehose instrumentation have a bug - // where when the keccak256 preimage is empty, it is written as "." which is - // completely wrong. - // - // To keep the same behavior, we will write the preimage as a "." when the encoded - // data is an empty string. - // - // For new chain, this code should be remove so that we just keep `hex.EncodeToString(data)`. - encodedData := hex.EncodeToString(data) - if encodedData == "" { - encodedData = "." - } - - activeCall.KeccakPreimages[hex.EncodeToString(hash.Bytes())] = encodedData + return "child" } -func (f *Firehose) OnGenesisBlock(b *types.Block, alloc core.GenesisAlloc, chainConfig *params.ChainConfig) { - f.OnBlockStart(b, big.NewInt(0), nil, nil, chainConfig) - f.captureTxStart(types.NewTx(&types.LegacyTx{}), emptyCommonHash, emptyCommonAddress, emptyCommonAddress) - f.CaptureStart(emptyCommonAddress, emptyCommonAddress, false, nil, 0, nil) +func (f *Firehose) OnGenesisBlock(b *types.Block, alloc types.GenesisAlloc) { + f.ensureBlockChainInit() + + f.OnBlockStart(tracing.BlockEvent{Block: b, TD: big.NewInt(0), Finalized: nil, Safe: nil}) + f.onTxStart(types.NewTx(&types.LegacyTx{}), emptyCommonHash, emptyCommonAddress, emptyCommonAddress) + f.OnCallEnter(0, byte(vm.CALL), emptyCommonAddress, emptyCommonAddress, nil, 0, nil) for _, addr := range sortedKeys(alloc) { account := alloc[addr] @@ -702,8 +821,8 @@ func (f *Firehose) OnGenesisBlock(b *types.Block, alloc core.GenesisAlloc, chain } } - f.CaptureEnd(nil, 0, nil, false) - f.CaptureTxEnd(&types.Receipt{ + f.OnCallExit(0, nil, 0, nil, false) + f.OnTxEnd(&types.Receipt{ PostState: b.Root().Bytes(), Status: types.ReceiptStatusSuccessful, }, nil) @@ -724,8 +843,8 @@ func sortedKeys[K bytesGetter, V any](m map[K]V) []K { return keys } -func (f *Firehose) OnBalanceChange(a common.Address, prev, new *big.Int, reason state.BalanceChangeReason) { - if reason == state.BalanceChangeUnspecified { +func (f *Firehose) OnBalanceChange(a common.Address, prev, new *big.Int, reason tracing.BalanceChangeReason) { + if reason == tracing.BalanceChangeUnspecified { // We ignore those, if they are mislabelled, too bad so particular attention needs to be ported to this return } @@ -735,7 +854,7 @@ func (f *Firehose) OnBalanceChange(a common.Address, prev, new *big.Int, reason // we were not tracking in the old Firehose instrumentation. // // New chain integration should remove this `if` statement all along. - if reason == state.BalanceDecreaseSelfdestructBurn { + if reason == tracing.BalanceDecreaseSelfdestructBurn { return } @@ -809,7 +928,7 @@ func (f *Firehose) OnCodeChange(a common.Address, prevCodeHash common.Hash, prev if f.transaction != nil { activeCall := f.callStack.Peek() if activeCall == nil { - f.panicNotInState("caller expected to be in call state but we were not, this is a bug") + f.panicInvalidState("caller expected to be in call state but we were not, this is a bug") } activeCall.CodeChanges = append(activeCall.CodeChanges, change) @@ -893,15 +1012,15 @@ func (f *Firehose) OnNewAccount(a common.Address, previousDataExists bool) { activeCall.AccountCreations = append(activeCall.AccountCreations, accountCreation) } -func (f *Firehose) OnGasChange(old, new uint64, reason vm.GasChangeReason) { +func (f *Firehose) OnGasChange(old, new uint64, reason tracing.GasChangeReason) { f.ensureInBlockAndInTrx() if old == new { return } - if reason == vm.GasChangeCallOpCode { - // We ignore those because we track OpCode gas consumption manually by tracking the gas value at `CaptureState` call + if reason == tracing.GasChangeCallOpCode { + // We ignore those because we track OpCode gas consumption manually by tracking the gas value at `OnOpcode` call return } @@ -911,11 +1030,11 @@ func (f *Firehose) OnGasChange(old, new uint64, reason vm.GasChangeReason) { // For new chain, this code should be remove so that they are included and useful to user. // // Ref eb1916a67d9bea03df16a7a3e2cfac72 - if reason == vm.GasChangeTxInitialBalance || - reason == vm.GasChangeTxRefunds || - reason == vm.GasChangeTxLeftOverReturned || - reason == vm.GasChangeCallInitialBalance || - reason == vm.GasChangeCallLeftOverReturned { + if reason == tracing.GasChangeTxInitialBalance || + reason == tracing.GasChangeTxRefunds || + reason == tracing.GasChangeTxLeftOverReturned || + reason == tracing.GasChangeCallInitialBalance || + reason == tracing.GasChangeCallLeftOverReturned { return } @@ -947,23 +1066,37 @@ func (f *Firehose) newGasChange(tag string, oldValue, newValue uint64, reason pb } } +func (f *Firehose) ensureBlockChainInit() { + if f.chainConfig == nil { + f.panicInvalidState("the OnBlockchainInit hook should have been called at this point") + } +} + func (f *Firehose) ensureInBlock() { if f.block == nil { - f.panicNotInState("caller expected to be in block state but we were not, this is a bug") + f.panicInvalidState("caller expected to be in block state but we were not, this is a bug") + } + + if f.chainConfig == nil { + f.panicInvalidState("the OnBlockchainInit hook should have been called at this point") } } func (f *Firehose) ensureNotInBlock() { if f.block != nil { - f.panicNotInState("caller expected to not be in block state but we were, this is a bug") + f.panicInvalidState("caller expected to not be in block state but we were, this is a bug") } } +// Suppress lint warning about unusued method, we keep it in the patch because it's used in other +// network which pulls this branch. +var _ = new(Firehose).ensureNotInBlock + func (f *Firehose) ensureInBlockAndInTrx() { f.ensureInBlock() if f.transaction == nil { - f.panicNotInState("caller expected to be in transaction state but we were not, this is a bug") + f.panicInvalidState("caller expected to be in transaction state but we were not, this is a bug") } } @@ -971,7 +1104,7 @@ func (f *Firehose) ensureInBlockAndNotInTrx() { f.ensureInBlock() if f.transaction != nil { - f.panicNotInState("caller expected to not be in transaction state but we were, this is a bug") + f.panicInvalidState("caller expected to not be in transaction state but we were, this is a bug") } } @@ -979,44 +1112,44 @@ func (f *Firehose) ensureInBlockAndNotInTrxAndNotInCall() { f.ensureInBlock() if f.transaction != nil { - f.panicNotInState("caller expected to not be in transaction state but we were, this is a bug") + f.panicInvalidState("caller expected to not be in transaction state but we were, this is a bug") } if f.callStack.HasActiveCall() { - f.panicNotInState("caller expected to not be in call state but we were, this is a bug") + f.panicInvalidState("caller expected to not be in call state but we were, this is a bug") } } func (f *Firehose) ensureInBlockOrTrx() { if f.transaction == nil && f.block == nil { - f.panicNotInState("caller expected to be in either block or transaction state but we were not, this is a bug") + f.panicInvalidState("caller expected to be in either block or transaction state but we were not, this is a bug") } } func (f *Firehose) ensureInBlockAndInTrxAndInCall() { if f.transaction == nil || f.block == nil { - f.panicNotInState("caller expected to be in block and in transaction but we were not, this is a bug") + f.panicInvalidState("caller expected to be in block and in transaction but we were not, this is a bug") } if !f.callStack.HasActiveCall() { - f.panicNotInState("caller expected to be in call state but we were not, this is a bug") + f.panicInvalidState("caller expected to be in call state but we were not, this is a bug") } } func (f *Firehose) ensureInCall() { if f.block == nil { - f.panicNotInState("caller expected to be in call state but we were not, this is a bug") + f.panicInvalidState("caller expected to be in call state but we were not, this is a bug") } } func (f *Firehose) ensureInSystemCall() { if !f.inSystemCall { - f.panicNotInState("call expected to be in system call state but we were not, this is a bug") + f.panicInvalidState("call expected to be in system call state but we were not, this is a bug") } } -func (f *Firehose) panicNotInState(msg string) string { - panic(fmt.Errorf("%s (inBlock=%t, inTransaction=%t, inCall=%t)", msg, f.block != nil, f.transaction != nil, f.callStack.HasActiveCall())) +func (f *Firehose) panicInvalidState(msg string) string { + panic(fmt.Errorf("%s (init=%t, inBlock=%t, inTransaction=%t, inCall=%t)", msg, f.chainConfig != nil, f.block != nil, f.transaction != nil, f.callStack.HasActiveCall())) } // printToFirehose is an easy way to print to Firehose format, it essentially @@ -1025,10 +1158,6 @@ func (f *Firehose) panicNotInState(msg string) string { // // It flushes this through [flushToFirehose] to the `os.Stdout` writer. func (f *Firehose) printBlockToFirehose(block *pbeth.Block, finalityStatus *FinalityStatus) { - if wasNeverSent := f.initSent.CompareAndSwap(false, true); wasNeverSent { - printToFirehose("INIT", FirehoseProtocolVersion, "geth", params.Version) - } - marshalled, err := proto.Marshal(block) if err != nil { panic(fmt.Errorf("failed to marshal block: %w", err)) @@ -1282,26 +1411,26 @@ func newBlobHashesFromChain(blobHashes []common.Hash) (out [][]byte) { return } -var balanceChangeReasonToPb = map[state.BalanceChangeReason]pbeth.BalanceChange_Reason{ - state.BalanceIncreaseRewardMineUncle: pbeth.BalanceChange_REASON_REWARD_MINE_UNCLE, - state.BalanceIncreaseRewardMineBlock: pbeth.BalanceChange_REASON_REWARD_MINE_BLOCK, - state.BalanceIncreaseDaoContract: pbeth.BalanceChange_REASON_DAO_REFUND_CONTRACT, - state.BalanceDecreaseDaoAccount: pbeth.BalanceChange_REASON_DAO_ADJUST_BALANCE, - state.BalanceChangeTransfer: pbeth.BalanceChange_REASON_TRANSFER, - state.BalanceIncreaseGenesisBalance: pbeth.BalanceChange_REASON_GENESIS_BALANCE, - state.BalanceDecreaseGasBuy: pbeth.BalanceChange_REASON_GAS_BUY, - state.BalanceIncreaseRewardTransactionFee: pbeth.BalanceChange_REASON_REWARD_TRANSACTION_FEE, - state.BalanceIncreaseGasReturn: pbeth.BalanceChange_REASON_GAS_REFUND, - state.BalanceChangeTouchAccount: pbeth.BalanceChange_REASON_TOUCH_ACCOUNT, - state.BalanceIncreaseSelfdestruct: pbeth.BalanceChange_REASON_SUICIDE_REFUND, - state.BalanceDecreaseSelfdestruct: pbeth.BalanceChange_REASON_SUICIDE_WITHDRAW, - state.BalanceDecreaseSelfdestructBurn: pbeth.BalanceChange_REASON_BURN, - state.BalanceIncreaseWithdrawal: pbeth.BalanceChange_REASON_WITHDRAWAL, +var balanceChangeReasonToPb = map[tracing.BalanceChangeReason]pbeth.BalanceChange_Reason{ + tracing.BalanceIncreaseRewardMineUncle: pbeth.BalanceChange_REASON_REWARD_MINE_UNCLE, + tracing.BalanceIncreaseRewardMineBlock: pbeth.BalanceChange_REASON_REWARD_MINE_BLOCK, + tracing.BalanceIncreaseDaoContract: pbeth.BalanceChange_REASON_DAO_REFUND_CONTRACT, + tracing.BalanceDecreaseDaoAccount: pbeth.BalanceChange_REASON_DAO_ADJUST_BALANCE, + tracing.BalanceChangeTransfer: pbeth.BalanceChange_REASON_TRANSFER, + tracing.BalanceIncreaseGenesisBalance: pbeth.BalanceChange_REASON_GENESIS_BALANCE, + tracing.BalanceDecreaseGasBuy: pbeth.BalanceChange_REASON_GAS_BUY, + tracing.BalanceIncreaseRewardTransactionFee: pbeth.BalanceChange_REASON_REWARD_TRANSACTION_FEE, + tracing.BalanceIncreaseGasReturn: pbeth.BalanceChange_REASON_GAS_REFUND, + tracing.BalanceChangeTouchAccount: pbeth.BalanceChange_REASON_TOUCH_ACCOUNT, + tracing.BalanceIncreaseSelfdestruct: pbeth.BalanceChange_REASON_SUICIDE_REFUND, + tracing.BalanceDecreaseSelfdestruct: pbeth.BalanceChange_REASON_SUICIDE_WITHDRAW, + tracing.BalanceDecreaseSelfdestructBurn: pbeth.BalanceChange_REASON_BURN, + tracing.BalanceIncreaseWithdrawal: pbeth.BalanceChange_REASON_WITHDRAWAL, - state.BalanceChangeUnspecified: pbeth.BalanceChange_REASON_UNKNOWN, + tracing.BalanceChangeUnspecified: pbeth.BalanceChange_REASON_UNKNOWN, } -func balanceChangeReasonFromChain(reason state.BalanceChangeReason) pbeth.BalanceChange_Reason { +func balanceChangeReasonFromChain(reason tracing.BalanceChangeReason) pbeth.BalanceChange_Reason { if r, ok := balanceChangeReasonToPb[reason]; ok { return r } @@ -1309,38 +1438,38 @@ func balanceChangeReasonFromChain(reason state.BalanceChangeReason) pbeth.Balanc panic(fmt.Errorf("unknown tracer balance change reason value '%d', check state.BalanceChangeReason so see to which constant it refers to", reason)) } -var gasChangeReasonToPb = map[vm.GasChangeReason]pbeth.GasChange_Reason{ +var gasChangeReasonToPb = map[tracing.GasChangeReason]pbeth.GasChange_Reason{ // Known Firehose issue: Those are new gas change trace that we were missing initially in our old // Firehose patch. See Known Firehose issue referenced eb1916a67d9bea03df16a7a3e2cfac72 for details // search for the id within this project to find back all links). // // New chain should uncomment the code below and remove the same assigments to UNKNOWN // - // vm.GasChangeTxInitialBalance: pbeth.GasChange_REASON_TX_INITIAL_BALANCE, - // vm.GasChangeTxRefunds: pbeth.GasChange_REASON_TX_REFUNDS, - // vm.GasChangeTxLeftOverReturned: pbeth.GasChange_REASON_TX_LEFT_OVER_RETURNED, - // vm.GasChangeCallInitialBalance: pbeth.GasChange_REASON_CALL_INITIAL_BALANCE, - // vm.GasChangeCallLeftOverReturned: pbeth.GasChange_REASON_CALL_LEFT_OVER_RETURNED, - vm.GasChangeTxInitialBalance: pbeth.GasChange_REASON_UNKNOWN, - vm.GasChangeTxRefunds: pbeth.GasChange_REASON_UNKNOWN, - vm.GasChangeTxLeftOverReturned: pbeth.GasChange_REASON_UNKNOWN, - vm.GasChangeCallInitialBalance: pbeth.GasChange_REASON_UNKNOWN, - vm.GasChangeCallLeftOverReturned: pbeth.GasChange_REASON_UNKNOWN, + // tracing.GasChangeTxInitialBalance: pbeth.GasChange_REASON_TX_INITIAL_BALANCE, + // tracing.GasChangeTxRefunds: pbeth.GasChange_REASON_TX_REFUNDS, + // tracing.GasChangeTxLeftOverReturned: pbeth.GasChange_REASON_TX_LEFT_OVER_RETURNED, + // tracing.GasChangeCallInitialBalance: pbeth.GasChange_REASON_CALL_INITIAL_BALANCE, + // tracing.GasChangeCallLeftOverReturned: pbeth.GasChange_REASON_CALL_LEFT_OVER_RETURNED, + tracing.GasChangeTxInitialBalance: pbeth.GasChange_REASON_UNKNOWN, + tracing.GasChangeTxRefunds: pbeth.GasChange_REASON_UNKNOWN, + tracing.GasChangeTxLeftOverReturned: pbeth.GasChange_REASON_UNKNOWN, + tracing.GasChangeCallInitialBalance: pbeth.GasChange_REASON_UNKNOWN, + tracing.GasChangeCallLeftOverReturned: pbeth.GasChange_REASON_UNKNOWN, - vm.GasChangeTxIntrinsicGas: pbeth.GasChange_REASON_INTRINSIC_GAS, - vm.GasChangeCallContractCreation: pbeth.GasChange_REASON_CONTRACT_CREATION, - vm.GasChangeCallContractCreation2: pbeth.GasChange_REASON_CONTRACT_CREATION2, - vm.GasChangeCallCodeStorage: pbeth.GasChange_REASON_CODE_STORAGE, - vm.GasChangeCallPrecompiledContract: pbeth.GasChange_REASON_PRECOMPILED_CONTRACT, - vm.GasChangeCallStorageColdAccess: pbeth.GasChange_REASON_STATE_COLD_ACCESS, - vm.GasChangeCallLeftOverRefunded: pbeth.GasChange_REASON_REFUND_AFTER_EXECUTION, - vm.GasChangeCallFailedExecution: pbeth.GasChange_REASON_FAILED_EXECUTION, + tracing.GasChangeTxIntrinsicGas: pbeth.GasChange_REASON_INTRINSIC_GAS, + tracing.GasChangeCallContractCreation: pbeth.GasChange_REASON_CONTRACT_CREATION, + tracing.GasChangeCallContractCreation2: pbeth.GasChange_REASON_CONTRACT_CREATION2, + tracing.GasChangeCallCodeStorage: pbeth.GasChange_REASON_CODE_STORAGE, + tracing.GasChangeCallPrecompiledContract: pbeth.GasChange_REASON_PRECOMPILED_CONTRACT, + tracing.GasChangeCallStorageColdAccess: pbeth.GasChange_REASON_STATE_COLD_ACCESS, + tracing.GasChangeCallLeftOverRefunded: pbeth.GasChange_REASON_REFUND_AFTER_EXECUTION, + tracing.GasChangeCallFailedExecution: pbeth.GasChange_REASON_FAILED_EXECUTION, // Ignored, we track them manually, newGasChange ensure that we panic if we see Unknown - vm.GasChangeCallOpCode: pbeth.GasChange_REASON_UNKNOWN, + tracing.GasChangeCallOpCode: pbeth.GasChange_REASON_UNKNOWN, } -func gasChangeReasonFromChain(reason vm.GasChangeReason) pbeth.GasChange_Reason { +func gasChangeReasonFromChain(reason tracing.GasChangeReason) pbeth.GasChange_Reason { if r, ok := gasChangeReasonToPb[reason]; ok { if r == pbeth.GasChange_REASON_UNKNOWN { panic(fmt.Errorf("tracer gas change reason value '%d' mapped to %s which is not accepted", reason, r)) diff --git a/eth/tracers/firehose_test.go b/eth/tracers/firehose_test.go index 95f8dc394c..3349cc5357 100644 --- a/eth/tracers/firehose_test.go +++ b/eth/tracers/firehose_test.go @@ -1,10 +1,19 @@ package tracers import ( + "encoding/json" + "fmt" + "math/big" + "reflect" "testing" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" pbeth "github.com/ethereum/go-ethereum/pb/sf/ethereum/type/v2" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/exp/maps" + "google.golang.org/protobuf/types/known/timestamppb" ) func TestFirehoseCallStack_Push(t *testing.T) { @@ -82,3 +91,198 @@ func Test_validateKnownTransactionTypes(t *testing.T) { }) } } + +var ignorePbFieldNames = map[string]bool{ + "Hash": true, + "TotalDifficulty": true, + "state": true, + "unknownFields": true, + "sizeCache": true, + + // This was a Polygon specific field that existed for a while and has since been + // removed. It can be safely ignored in all protocols now. + "TxDependency": true, +} + +var pbFieldNameToGethMapping = map[string]string{ + "WithdrawalsRoot": "WithdrawalsHash", + "MixHash": "MixDigest", + "BaseFeePerGas": "BaseFee", + "StateRoot": "Root", + "ExtraData": "Extra", + "Timestamp": "Time", + "ReceiptRoot": "ReceiptHash", + "TransactionsRoot": "TxHash", + "LogsBloom": "Bloom", +} + +var ( + pbHeaderType = reflect.TypeFor[pbeth.BlockHeader]() + gethHeaderType = reflect.TypeFor[types.Header]() +) + +func Test_TypesHeader_AllConsensusFieldsAreKnown(t *testing.T) { + // This exact hash varies from protocol to protocol and also sometimes from one version to the other. + // When adding support for a new hard-fork that adds new block header fields, it's normal that this value + // changes. If you are sure the two struct are the same, then you can update the expected hash below + // to the new value. + expectedHash := common.HexToHash("5341947c531e5c9cf38202784b16ac66484fe1838aa6e825436b22321b927296") + + gethHeaderValue := reflect.New(gethHeaderType) + fillAllFieldsWithNonEmptyValues(t, gethHeaderValue, reflect.VisibleFields(gethHeaderType)) + gethHeader := gethHeaderValue.Interface().(*types.Header) + + // If you hit this assertion, it means that the fields `types.Header` of go-ethereum differs now + // versus last time this test was edited. + // + // It's important to understand that in Ethereum Block Header (e.g. `*types.Header`), the `Hash` is + // actually a computed value based on the other fields in the struct, so if you change any field, + // the hash will change also. + // + // On hard-fork, it happens that new fields are added, this test serves as a way to "detect" in codde + // that the expected fields of `types.Header` changed + require.Equal(t, expectedHash, gethHeader.Hash(), + "Geth Header Hash mistmatch, got %q but expecting %q on *types.Header:\n\nGeth Header (from fillNonDefault(new(*types.Header)))\n%s", + gethHeader.Hash().Hex(), + expectedHash, + asIndentedJSON(t, gethHeader), + ) +} + +func Test_FirehoseAndGethHeaderFieldMatches(t *testing.T) { + pbFields := filter(reflect.VisibleFields(pbHeaderType), func(f reflect.StructField) bool { + return !ignorePbFieldNames[f.Name] + }) + + gethFields := reflect.VisibleFields(gethHeaderType) + + pbFieldCount := len(pbFields) + gethFieldCount := len(gethFields) + + pbFieldNames := extractStructFieldNames(pbFields) + gethFieldNames := extractStructFieldNames(gethFields) + + // If you reach this assertion, it means that the fields count in the protobuf and go-ethereum are different. + // It is super important that you properly update the mapping from pbeth.BlockHeader to go-ethereum/core/types.Header + // that is done in `codecHeaderToGethHeader` function in `executor/provider_statedb.go`. + require.Equal( + t, + pbFieldCount, + gethFieldCount, + fieldsCountMistmatchMessage(t, pbFieldNames, gethFieldNames)) + + for pbFieldName := range pbFieldNames { + pbFieldRenamedName, found := pbFieldNameToGethMapping[pbFieldName] + if !found { + pbFieldRenamedName = pbFieldName + } + + assert.Contains(t, gethFieldNames, pbFieldRenamedName, "pbField.Name=%q (original %q) not found in gethFieldNames", pbFieldRenamedName, pbFieldName) + } +} + +func fillAllFieldsWithNonEmptyValues(t *testing.T, structValue reflect.Value, fields []reflect.StructField) { + t.Helper() + + for _, field := range fields { + fieldValue := structValue.Elem().FieldByName(field.Name) + require.True(t, fieldValue.IsValid(), "field %q not found", field.Name) + + switch fieldValue.Interface().(type) { + case []byte: + fieldValue.Set(reflect.ValueOf([]byte{1})) + case uint64: + fieldValue.Set(reflect.ValueOf(uint64(1))) + case *uint64: + var mockValue uint64 = 1 + fieldValue.Set(reflect.ValueOf(&mockValue)) + case *common.Hash: + var mockValue common.Hash = common.HexToHash("0x01") + fieldValue.Set(reflect.ValueOf(&mockValue)) + case common.Hash: + fieldValue.Set(reflect.ValueOf(common.HexToHash("0x01"))) + case common.Address: + fieldValue.Set(reflect.ValueOf(common.HexToAddress("0x01"))) + case types.Bloom: + fieldValue.Set(reflect.ValueOf(types.BytesToBloom([]byte{1}))) + case types.BlockNonce: + fieldValue.Set(reflect.ValueOf(types.EncodeNonce(1))) + case *big.Int: + fieldValue.Set(reflect.ValueOf(big.NewInt(1))) + case *pbeth.BigInt: + fieldValue.Set(reflect.ValueOf(&pbeth.BigInt{Bytes: []byte{1}})) + case *timestamppb.Timestamp: + fieldValue.Set(reflect.ValueOf(×tamppb.Timestamp{Seconds: 1})) + default: + // If you reach this panic in test, simply add a case above with a sane non-default + // value for the type in question. + t.Fatalf("unsupported type %T", fieldValue.Interface()) + } + } +} + +func fieldsCountMistmatchMessage(t *testing.T, pbFieldNames map[string]bool, gethFieldNames map[string]bool) string { + t.Helper() + + pbRemappedFieldNames := make(map[string]bool, len(pbFieldNames)) + for pbFieldName := range pbFieldNames { + pbFieldRenamedName, found := pbFieldNameToGethMapping[pbFieldName] + if !found { + pbFieldRenamedName = pbFieldName + } + + pbRemappedFieldNames[pbFieldRenamedName] = true + } + + return fmt.Sprintf( + "Field count mistmatch between `pbeth.BlockHeader` (has %d fields) and `*types.Header` (has %d fields)\n\n"+ + "Fields in `pbeth.Blockheader`:\n%s\n\n"+ + "Fields in `*types.Header`:\n%s\n\n"+ + "Missing in `pbeth.BlockHeader`:\n%s\n\n"+ + "Missing in `*types.Header`:\n%s", + len(pbRemappedFieldNames), + len(gethFieldNames), + asIndentedJSON(t, maps.Keys(pbRemappedFieldNames)), + asIndentedJSON(t, maps.Keys(gethFieldNames)), + asIndentedJSON(t, missingInSet(gethFieldNames, pbRemappedFieldNames)), + asIndentedJSON(t, missingInSet(pbRemappedFieldNames, gethFieldNames)), + ) +} + +func asIndentedJSON(t *testing.T, v any) string { + t.Helper() + out, err := json.MarshalIndent(v, "", " ") + require.NoError(t, err) + + return string(out) +} + +func missingInSet(a, b map[string]bool) []string { + missing := make([]string, 0) + for name := range a { + if !b[name] { + missing = append(missing, name) + } + } + + return missing +} + +func extractStructFieldNames(fields []reflect.StructField) map[string]bool { + result := make(map[string]bool, len(fields)) + for _, field := range fields { + result[field.Name] = true + } + return result +} + +func filter[S ~[]T, T any](s S, f func(T) bool) (out S) { + out = make(S, 0, len(s)/4) + for i, v := range s { + if f(v) { + out = append(out, s[i]) + } + } + + return out +} diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index c52013c898..4930abdca0 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -135,27 +135,27 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { GasLimit: uint64(test.Context.GasLimit), BaseFee: test.Genesis.BaseFee, } - triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + state = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) ) - triedb.Close() + state.Close() tracer, err := directory.DefaultDirectory.New(tracerName, new(directory.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } - statedb.SetLogger(tracer) + state.StateDB.SetLogger(tracer.Hooks) msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } - evm := vm.NewEVM(context, core.NewEVMTxContext(msg), statedb, test.Genesis.Config, vm.Config{Tracer: tracer}) - tracer.CaptureTxStart(evm, tx, msg.From) + evm := vm.NewEVM(context, core.NewEVMTxContext(msg), state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } - tracer.CaptureTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, nil) + tracer.OnTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, nil) // Retrieve the trace result and compare against the expected. res, err := tracer.GetResult() if err != nil { @@ -241,8 +241,8 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { if err != nil { b.Fatalf("failed to prepare transaction for tracing: %v", err) } - triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) - defer triedb.Close() + state := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + defer state.Close() b.ReportAllocs() b.ResetTimer() @@ -251,8 +251,8 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { if err != nil { b.Fatalf("failed to create call tracer: %v", err) } - evm := vm.NewEVM(context, txContext, statedb, test.Genesis.Config, vm.Config{Tracer: tracer}) - snap := statedb.Snapshot() + evm := vm.NewEVM(context, txContext, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + snap := state.StateDB.Snapshot() st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if _, err = st.TransitionDb(); err != nil { b.Fatalf("failed to execute transaction: %v", err) @@ -260,7 +260,7 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { if _, err = tracer.GetResult(); err != nil { b.Fatal(err) } - statedb.RevertToSnapshot(snap) + state.StateDB.RevertToSnapshot(snap) } } @@ -283,7 +283,7 @@ func TestInternals(t *testing.T) { BaseFee: new(big.Int), } ) - mkTracer := func(name string, cfg json.RawMessage) directory.Tracer { + mkTracer := func(name string, cfg json.RawMessage) *directory.Tracer { tr, err := directory.DefaultDirectory.New(name, nil, cfg) if err != nil { t.Fatalf("failed to create call tracer: %v", err) @@ -294,7 +294,7 @@ func TestInternals(t *testing.T) { for _, tc := range []struct { name string code []byte - tracer directory.Tracer + tracer *directory.Tracer want string }{ { @@ -369,17 +369,17 @@ func TestInternals(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), - core.GenesisAlloc{ - to: core.GenesisAccount{ + state := tests.MakePreState(rawdb.NewMemoryDatabase(), + types.GenesisAlloc{ + to: types.Account{ Code: tc.code, }, - origin: core.GenesisAccount{ + origin: types.Account{ Balance: big.NewInt(500000000000000), }, }, false, rawdb.HashScheme) - defer triedb.Close() - statedb.SetLogger(tc.tracer) + defer state.Close() + state.StateDB.SetLogger(tc.tracer.Hooks) tx, err := types.SignNewTx(key, signer, &types.LegacyTx{ To: &to, Value: big.NewInt(0), @@ -393,17 +393,17 @@ func TestInternals(t *testing.T) { Origin: origin, GasPrice: tx.GasPrice(), } - evm := vm.NewEVM(context, txContext, statedb, config, vm.Config{Tracer: tc.tracer}) + evm := vm.NewEVM(context, txContext, state.StateDB, config, vm.Config{Tracer: tc.tracer.Hooks}) msg, err := core.TransactionToMessage(tx, signer, big.NewInt(0)) if err != nil { t.Fatalf("test %v: failed to create message: %v", tc.name, err) } - tc.tracer.CaptureTxStart(evm, tx, msg.From) + tc.tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { t.Fatalf("test %v: failed to execute transaction: %v", tc.name, err) } - tc.tracer.CaptureTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, nil) + tc.tracer.OnTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, nil) // Retrieve the trace result and compare against the expected res, err := tc.tracer.GetResult() if err != nil { diff --git a/eth/tracers/internal/tracetest/flat_calltrace_test.go b/eth/tracers/internal/tracetest/flat_calltrace_test.go index 21b70e5797..ede0e3e919 100644 --- a/eth/tracers/internal/tracetest/flat_calltrace_test.go +++ b/eth/tracers/internal/tracetest/flat_calltrace_test.go @@ -93,8 +93,8 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), } - triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) - defer triedb.Close() + state := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + defer state.Close() // Create the tracer, the EVM environment and run it tracer, err := directory.DefaultDirectory.New(tracerName, new(directory.Context), test.TracerConfig) @@ -102,18 +102,18 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string return fmt.Errorf("failed to create call tracer: %v", err) } - statedb.SetLogger(tracer) + state.StateDB.SetLogger(tracer.Hooks) msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { return fmt.Errorf("failed to prepare transaction for tracing: %v", err) } - evm := vm.NewEVM(context, core.NewEVMTxContext(msg), statedb, test.Genesis.Config, vm.Config{Tracer: tracer}) - tracer.CaptureTxStart(evm, tx, msg.From) + evm := vm.NewEVM(context, core.NewEVMTxContext(msg), state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { return fmt.Errorf("failed to execute transaction: %v", err) } - tracer.CaptureTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, nil) + tracer.OnTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, nil) // Retrieve the trace result and compare against the etalon res, err := tracer.GetResult() diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index c9ca57f53c..8b70d09da1 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" @@ -103,27 +104,32 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { GasLimit: uint64(test.Context.GasLimit), BaseFee: test.Genesis.BaseFee, } - triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + state = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) ) - defer triedb.Close() + defer state.Close() + + if test.Genesis.ExcessBlobGas != nil && test.Genesis.BlobGasUsed != nil { + excessBlobGas := eip4844.CalcExcessBlobGas(*test.Genesis.ExcessBlobGas, *test.Genesis.BlobGasUsed) + context.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas) + } tracer, err := directory.DefaultDirectory.New(tracerName, new(directory.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } - statedb.SetLogger(tracer) + state.StateDB.SetLogger(tracer.Hooks) msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } - evm := vm.NewEVM(context, core.NewEVMTxContext(msg), statedb, test.Genesis.Config, vm.Config{Tracer: tracer}) - tracer.CaptureTxStart(evm, tx, msg.From) + evm := vm.NewEVM(context, core.NewEVMTxContext(msg), state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } - tracer.CaptureTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, nil) + tracer.OnTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, nil) // Retrieve the trace result and compare against the expected res, err := tracer.GetResult() if err != nil { diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json new file mode 100644 index 0000000000..315481aff5 --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json @@ -0,0 +1,63 @@ +{ + "genesis": { + "baseFeePerGas": "7", + "blobGasUsed": "0", + "difficulty": "0", + "excessBlobGas": "36306944", + "extraData": "0xd983010e00846765746888676f312e32312e308664617277696e", + "gasLimit": "15639172", + "hash": "0xc682259fda061bb9ce8ccb491d5b2d436cb73daf04e1025dd116d045ce4ad28c", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0xae1a5ba939a4c9ac38aabeff361169fb55a6fc2c9511457e0be6eff9514faec0", + "nonce": "0x0000000000000000", + "number": "315", + "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "stateRoot": "0x577f42ab21ccfd946511c57869ace0bdf7c217c36f02b7cd3459df0ed1cffc1a", + "timestamp": "1709626771", + "totalDifficulty": "1", + "withdrawals": [], + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "alloc": { + "0x0000000000000000000000000000000000000000": { + "balance": "0x272e0528" + }, + "0x0c2c51a0990aee1d73c1228de158688341557508": { + "balance": "0xde0b6b3a7640000" + } + }, + "config": { + "chainId": 1337, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "shanghaiTime": 0, + "cancunTime": 0, + "terminalTotalDifficulty": 0, + "terminalTotalDifficultyPassed": true + } + }, + "context": { + "number": "316", + "difficulty": "0", + "timestamp": "1709626785", + "gasLimit": "15654443", + "miner": "0x0000000000000000000000000000000000000000" + }, + "input": "0x03f8b1820539806485174876e800825208940c2c51a0990aee1d73c1228de1586883415575088080c083020000f842a00100c9fbdf97f747e85847b4f3fff408f89c26842f77c882858bf2c89923849aa00138e3896f3c27f2389147507f8bcec52028b0efca6ee842ed83c9158873943880a0dbac3f97a532c9b00e6239b29036245a5bfbb96940b9d848634661abee98b945a03eec8525f261c2e79798f7b45a5d6ccaefa24576d53ba5023e919b86841c0675", + "result": { + "0x0000000000000000000000000000000000000000": { "balance": "0x272e0528" }, + "0x0c2c51a0990aee1d73c1228de158688341557508": { + "balance": "0xde0b6b3a7640000" + } + } +} diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer/create_post_eip158.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer/create_post_eip158.json new file mode 100644 index 0000000000..205b472dab --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer/create_post_eip158.json @@ -0,0 +1,64 @@ +{ + "genesis": { + "baseFeePerGas": "7", + "difficulty": "2", + "extraData": "0xd983010d0e846765746888676f312e32312e318664617277696e0000000000001713699f05f79a59abec177c7a87b90ceda79b72ff5edc9197dd7627a447cde45b079bbc3765a236cdf680e2d4d2247135d0e6bb6fd92b50638b92504ddb274f00", + "gasLimit": "30000000", + "hash": "0x6ad5258175c66f4e883d238a92a08428d8ebcbeac631ab7b972634cc05effab3", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "39137", + "stateRoot": "0x715f00df764dbadd4863247a215ac44b5420beafde3ec458b15db7aafa89be0c", + "timestamp": "1709022192", + "totalDifficulty": "78275", + "alloc": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": "2" + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": "64" + } + }, + "config": { + "chainId": 12345, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "clique": { + "period": 5, + "epoch": 30000 + } + } + }, + "context": { + "number": "39138", + "difficulty": "2", + "timestamp": "1709022197", + "gasLimit": "30000000", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0" + }, + "input": "0x02f902af823039408459682f008459682f088302b3538080b90254608060405234801561001057600080fd5b50610234806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806309ce9ccb1461003b5780633fb5c1cb14610059575b600080fd5b610043610075565b60405161005091906100e2565b60405180910390f35b610073600480360381019061006e919061012e565b61007b565b005b60005481565b80600081905550600a8111156100c6576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100bd906101de565b60405180910390fd5b50565b6000819050919050565b6100dc816100c9565b82525050565b60006020820190506100f760008301846100d3565b92915050565b600080fd5b61010b816100c9565b811461011657600080fd5b50565b60008135905061012881610102565b92915050565b600060208284031215610144576101436100fd565b5b600061015284828501610119565b91505092915050565b600082825260208201905092915050565b7f4e756d6265722069732067726561746572207468616e2031302c207472616e7360008201527f616374696f6e2072657665727465642e00000000000000000000000000000000602082015250565b60006101c860308361015b565b91506101d38261016c565b604082019050919050565b600060208201905081810360008301526101f7816101bb565b905091905056fea264697066735822122069018995fecf03bda91a88b6eafe41641709dee8b4a706fe301c8a569fe8c1b364736f6c63430008130033c001a0a8cf4729b7e4664687abb3e2559853d7d489eb441519be2a17493061fb4c3a03a04b5a904ba8a6e59c6c40049c4d14a73233aeb8a45b38403199f304630dc0d453", + "result": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": 2 + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": 64 + } + } +} diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_post_eip158.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_post_eip158.json new file mode 100644 index 0000000000..83266f6669 --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_post_eip158.json @@ -0,0 +1,82 @@ +{ + "genesis": { + "baseFeePerGas": "7", + "difficulty": "2", + "extraData": "0xd983010d0e846765746888676f312e32312e318664617277696e0000000000001713699f05f79a59abec177c7a87b90ceda79b72ff5edc9197dd7627a447cde45b079bbc3765a236cdf680e2d4d2247135d0e6bb6fd92b50638b92504ddb274f00", + "gasLimit": "30000000", + "hash": "0x6ad5258175c66f4e883d238a92a08428d8ebcbeac631ab7b972634cc05effab3", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "39137", + "stateRoot": "0x715f00df764dbadd4863247a215ac44b5420beafde3ec458b15db7aafa89be0c", + "timestamp": "1709022192", + "totalDifficulty": "78275", + "alloc": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": "2" + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": "64" + } + }, + "config": { + "chainId": 12345, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "clique": { + "period": 5, + "epoch": 30000 + } + } + }, + "context": { + "number": "39138", + "difficulty": "2", + "timestamp": "1709022197", + "gasLimit": "30000000", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0" + }, + "input": "0x02f902af823039408459682f008459682f088302b3538080b90254608060405234801561001057600080fd5b50610234806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806309ce9ccb1461003b5780633fb5c1cb14610059575b600080fd5b610043610075565b60405161005091906100e2565b60405180910390f35b610073600480360381019061006e919061012e565b61007b565b005b60005481565b80600081905550600a8111156100c6576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100bd906101de565b60405180910390fd5b50565b6000819050919050565b6100dc816100c9565b82525050565b60006020820190506100f760008301846100d3565b92915050565b600080fd5b61010b816100c9565b811461011657600080fd5b50565b60008135905061012881610102565b92915050565b600060208284031215610144576101436100fd565b5b600061015284828501610119565b91505092915050565b600082825260208201905092915050565b7f4e756d6265722069732067726561746572207468616e2031302c207472616e7360008201527f616374696f6e2072657665727465642e00000000000000000000000000000000602082015250565b60006101c860308361015b565b91506101d38261016c565b604082019050919050565b600060208201905081810360008301526101f7816101bb565b905091905056fea264697066735822122069018995fecf03bda91a88b6eafe41641709dee8b4a706fe301c8a569fe8c1b364736f6c63430008130033c001a0a8cf4729b7e4664687abb3e2559853d7d489eb441519be2a17493061fb4c3a03a04b5a904ba8a6e59c6c40049c4d14a73233aeb8a45b38403199f304630dc0d453", + "tracerConfig": { + "diffMode": true + }, + "result": { + "post": { + "0x1bda2f8e4735507930bd6cfe873bf0bf0f4ab1de": { + "code": "0x608060405234801561001057600080fd5b50600436106100365760003560e01c806309ce9ccb1461003b5780633fb5c1cb14610059575b600080fd5b610043610075565b60405161005091906100e2565b60405180910390f35b610073600480360381019061006e919061012e565b61007b565b005b60005481565b80600081905550600a8111156100c6576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100bd906101de565b60405180910390fd5b50565b6000819050919050565b6100dc816100c9565b82525050565b60006020820190506100f760008301846100d3565b92915050565b600080fd5b61010b816100c9565b811461011657600080fd5b50565b60008135905061012881610102565b92915050565b600060208284031215610144576101436100fd565b5b600061015284828501610119565b91505092915050565b600082825260208201905092915050565b7f4e756d6265722069732067726561746572207468616e2031302c207472616e7360008201527f616374696f6e2072657665727465642e00000000000000000000000000000000602082015250565b60006101c860308361015b565b91506101d38261016c565b604082019050919050565b600060208201905081810360008301526101f7816101bb565b905091905056fea264697066735822122069018995fecf03bda91a88b6eafe41641709dee8b4a706fe301c8a569fe8c1b364736f6c63430008130033", + "nonce": 1 + }, + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f0645688331eb5690" + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6aae9b21b6ee855", + "nonce": 65 + } + }, + "pre": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": 2 + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": 64 + } + } + } +} diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index 73b106f4ee..5c52ba1f11 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -23,8 +23,10 @@ import ( "math/big" "github.com/dop251/goja" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/tracers/directory" + "github.com/holiman/uint256" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -42,9 +44,9 @@ func init() { if err != nil { panic(err) } - type ctorFn = func(*directory.Context, json.RawMessage) (directory.Tracer, error) + type ctorFn = func(*directory.Context, json.RawMessage) (*directory.Tracer, error) lookup := func(code string) ctorFn { - return func(ctx *directory.Context, cfg json.RawMessage) (directory.Tracer, error) { + return func(ctx *directory.Context, cfg json.RawMessage) (*directory.Tracer, error) { return newJsTracer(code, ctx, cfg) } } @@ -96,10 +98,8 @@ func fromBuf(vm *goja.Runtime, bufType goja.Value, buf goja.Value, allowString b // jsTracer is an implementation of the Tracer interface which evaluates // JS functions on the relevant EVM hooks. It uses Goja as its JS engine. type jsTracer struct { - directory.NoopTracer - vm *goja.Runtime - env *vm.EVM + env *tracing.VMContext toBig toBigFn // Converts a hex string into a JS bigint toBuf toBufFn // Converts a []byte into a JS buffer fromBuf fromBufFn // Converts an array, hex string or Uint8Array to a []byte @@ -136,7 +136,7 @@ type jsTracer struct { // The methods `result` and `fault` are required to be present. // The methods `step`, `enter`, and `exit` are optional, but note that // `enter` and `exit` always go together. -func newJsTracer(code string, ctx *directory.Context, cfg json.RawMessage) (directory.Tracer, error) { +func newJsTracer(code string, ctx *directory.Context, cfg json.RawMessage) (*directory.Tracer, error) { vm := goja.New() // By default field names are exported to JS as is, i.e. capitalized. vm.SetFieldNameMapper(goja.UncapFieldNameMapper()) @@ -219,33 +219,47 @@ func newJsTracer(code string, ctx *directory.Context, cfg json.RawMessage) (dire t.frameValue = t.frame.setupObject() t.frameResultValue = t.frameResult.setupObject() t.logValue = t.log.setupObject() - return t, nil + + return &directory.Tracer{ + Hooks: &tracing.Hooks{ + OnTxStart: t.OnTxStart, + OnTxEnd: t.OnTxEnd, + OnEnter: t.OnEnter, + OnExit: t.OnExit, + OnOpcode: t.OnOpcode, + OnFault: t.OnFault, + }, + GetResult: t.GetResult, + Stop: t.Stop, + }, nil } -// CaptureTxStart implements the Tracer interface and is invoked at the beginning of +// OnTxStart implements the Tracer interface and is invoked at the beginning of // transaction processing. -func (t *jsTracer) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common.Address) { +func (t *jsTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { t.env = env // Need statedb access for db object db := &dbObj{db: env.StateDB, vm: t.vm, toBig: t.toBig, toBuf: t.toBuf, fromBuf: t.fromBuf} t.dbValue = db.setupObject() // Update list of precompiles based on current block - rules := env.ChainConfig().Rules(env.Context.BlockNumber, env.Context.Random != nil, env.Context.Time) + rules := env.ChainConfig.Rules(env.BlockNumber, env.Random != nil, env.Time) t.activePrecompiles = vm.ActivePrecompiles(rules) - t.ctx["block"] = t.vm.ToValue(t.env.Context.BlockNumber.Uint64()) + t.ctx["block"] = t.vm.ToValue(t.env.BlockNumber.Uint64()) t.ctx["gas"] = t.vm.ToValue(tx.Gas()) - gasPriceBig, err := t.toBig(t.vm, env.TxContext.GasPrice.String()) + gasPriceBig, err := t.toBig(t.vm, env.GasPrice.String()) if err != nil { t.err = err - t.env.Cancel() return } t.ctx["gasPrice"] = gasPriceBig } -// CaptureTxEnd implements the Tracer interface and is invoked at the end of +// OnTxEnd implements the Tracer interface and is invoked at the end of // transaction processing. -func (t *jsTracer) CaptureTxEnd(receipt *types.Receipt, err error) { +func (t *jsTracer) OnTxEnd(receipt *types.Receipt, err error) { + if t.err != nil { + return + } if err != nil { // Don't override vm error if _, ok := t.ctx["error"]; !ok { @@ -256,11 +270,10 @@ func (t *jsTracer) CaptureTxEnd(receipt *types.Receipt, err error) { t.ctx["gasUsed"] = t.vm.ToValue(receipt.GasUsed) } -// CaptureStart implements the Tracer interface to initialize the tracing operation. -func (t *jsTracer) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { - cancel := func(err error) { - t.err = err - t.env.Cancel() +// onStart implements the Tracer interface to initialize the tracing operation. +func (t *jsTracer) onStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { + if t.err != nil { + return } if create { t.ctx["type"] = t.vm.ToValue("CREATE") @@ -269,32 +282,32 @@ func (t *jsTracer) CaptureStart(from common.Address, to common.Address, create b } fromVal, err := t.toBuf(t.vm, from.Bytes()) if err != nil { - cancel(err) + t.err = err return } t.ctx["from"] = fromVal toVal, err := t.toBuf(t.vm, to.Bytes()) if err != nil { - cancel(err) + t.err = err return } t.ctx["to"] = toVal inputVal, err := t.toBuf(t.vm, input) if err != nil { - cancel(err) + t.err = err return } t.ctx["input"] = inputVal valueBig, err := t.toBig(t.vm, value.String()) if err != nil { - cancel(err) + t.err = err return } t.ctx["value"] = valueBig } -// CaptureState implements the Tracer interface to trace a single step of VM execution. -func (t *jsTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { +// OnOpcode implements the Tracer interface to trace a single step of VM execution. +func (t *jsTracer) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { if !t.traceStep { return } @@ -303,10 +316,10 @@ func (t *jsTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope } log := t.log - log.op.op = op - log.memory.memory = scope.Memory - log.stack.stack = scope.Stack - log.contract.contract = scope.Contract + log.op.op = vm.OpCode(op) + log.memory.memory = scope.MemoryData() + log.stack.stack = scope.StackData() + log.contract.scope = scope log.pc = pc log.gas = gas log.cost = cost @@ -318,20 +331,23 @@ func (t *jsTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope } } -// CaptureFault implements the Tracer interface to trace an execution fault -func (t *jsTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { +// OnFault implements the Tracer interface to trace an execution fault +func (t *jsTracer) OnFault(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, depth int, err error) { if t.err != nil { return } - // Other log fields have been already set as part of the last CaptureState. + // Other log fields have been already set as part of the last OnOpcode. t.log.err = err if _, err := t.fault(t.obj, t.logValue, t.dbValue); err != nil { t.onError("fault", err) } } -// CaptureEnd is called after the call finishes to finalize the tracing. -func (t *jsTracer) CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) { +// onEnd is called after the call finishes to finalize the tracing. +func (t *jsTracer) onEnd(output []byte, gasUsed uint64, err error, reverted bool) { + if t.err != nil { + return + } if err != nil { t.ctx["error"] = t.vm.ToValue(err.Error()) } @@ -343,16 +359,20 @@ func (t *jsTracer) CaptureEnd(output []byte, gasUsed uint64, err error, reverted t.ctx["output"] = outputVal } -// CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). -func (t *jsTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { - if !t.traceFrame { - return - } +// OnEnter is called when EVM enters a new scope (via call, create or selfdestruct). +func (t *jsTracer) OnEnter(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { if t.err != nil { return } + if depth == 0 { + t.onStart(from, to, vm.OpCode(typ) == vm.CREATE, input, gas, value) + return + } + if !t.traceFrame { + return + } - t.frame.typ = typ.String() + t.frame.typ = vm.OpCode(typ).String() t.frame.from = from t.frame.to = to t.frame.input = common.CopyBytes(input) @@ -367,9 +387,16 @@ func (t *jsTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Ad } } -// CaptureExit is called when EVM exits a scope, even if the scope didn't +// OnExit is called when EVM exits a scope, even if the scope didn't // execute any code. -func (t *jsTracer) CaptureExit(output []byte, gasUsed uint64, err error, reverted bool) { +func (t *jsTracer) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { + if t.err != nil { + return + } + if depth == 0 { + t.onEnd(output, gasUsed, err, reverted) + return + } if !t.traceFrame { return } @@ -385,6 +412,9 @@ func (t *jsTracer) CaptureExit(output []byte, gasUsed uint64, err error, reverte // GetResult calls the Javascript 'result' function and returns its value, or any accumulated error func (t *jsTracer) GetResult() (json.RawMessage, error) { + if t.err != nil { + return nil, t.err + } ctx := t.vm.ToValue(t.ctx) res, err := t.result(t.obj, ctx, t.dbValue) if err != nil { @@ -394,7 +424,7 @@ func (t *jsTracer) GetResult() (json.RawMessage, error) { if err != nil { return nil, err } - return json.RawMessage(encoded), t.err + return encoded, t.err } // Stop terminates execution of the tracer at the first opportune moment. @@ -407,9 +437,6 @@ func (t *jsTracer) Stop(err error) { // execution. func (t *jsTracer) onError(context string, err error) { t.err = wrapError(context, err) - // `env` is set on CaptureStart which comes before any JS execution. - // So it should be non-nil. - t.env.Cancel() } func wrapError(context string, err error) error { @@ -588,7 +615,7 @@ func (o *opObj) setupObject() *goja.Object { } type memoryObj struct { - memory *vm.Memory + memory []byte vm *goja.Runtime toBig toBigFn toBuf toBufFn @@ -639,14 +666,14 @@ func (mo *memoryObj) GetUint(addr int64) goja.Value { // getUint returns the 32 bytes at the specified address interpreted as a uint. func (mo *memoryObj) getUint(addr int64) (*big.Int, error) { - if mo.memory.Len() < int(addr)+32 || addr < 0 { - return nil, fmt.Errorf("tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), addr, 32) + if len(mo.memory) < int(addr)+32 || addr < 0 { + return nil, fmt.Errorf("tracer accessed out of bound memory: available %d, offset %d, size %d", len(mo.memory), addr, 32) } - return new(big.Int).SetBytes(mo.memory.GetPtr(addr, 32)), nil + return new(big.Int).SetBytes(directory.MemoryPtr(mo.memory, addr, 32)), nil } func (mo *memoryObj) Length() int { - return mo.memory.Len() + return len(mo.memory) } func (m *memoryObj) setupObject() *goja.Object { @@ -658,7 +685,7 @@ func (m *memoryObj) setupObject() *goja.Object { } type stackObj struct { - stack *vm.Stack + stack []uint256.Int vm *goja.Runtime toBig toBigFn } @@ -679,14 +706,14 @@ func (s *stackObj) Peek(idx int) goja.Value { // peek returns the nth-from-the-top element of the stack. func (s *stackObj) peek(idx int) (*big.Int, error) { - if len(s.stack.Data()) <= idx || idx < 0 { - return nil, fmt.Errorf("tracer accessed out of bound stack: size %d, index %d", len(s.stack.Data()), idx) + if len(s.stack) <= idx || idx < 0 { + return nil, fmt.Errorf("tracer accessed out of bound stack: size %d, index %d", len(s.stack), idx) } - return s.stack.Back(idx).ToBig(), nil + return directory.StackBack(s.stack, idx).ToBig(), nil } func (s *stackObj) Length() int { - return len(s.stack.Data()) + return len(s.stack) } func (s *stackObj) setupObject() *goja.Object { @@ -697,7 +724,7 @@ func (s *stackObj) setupObject() *goja.Object { } type dbObj struct { - db vm.StateDB + db tracing.StateDB vm *goja.Runtime toBig toBigFn toBuf toBufFn @@ -789,14 +816,14 @@ func (do *dbObj) setupObject() *goja.Object { } type contractObj struct { - contract *vm.Contract - vm *goja.Runtime - toBig toBigFn - toBuf toBufFn + scope tracing.OpContext + vm *goja.Runtime + toBig toBigFn + toBuf toBufFn } func (co *contractObj) GetCaller() goja.Value { - caller := co.contract.Caller().Bytes() + caller := co.scope.Caller().Bytes() res, err := co.toBuf(co.vm, caller) if err != nil { co.vm.Interrupt(err) @@ -806,7 +833,7 @@ func (co *contractObj) GetCaller() goja.Value { } func (co *contractObj) GetAddress() goja.Value { - addr := co.contract.Address().Bytes() + addr := co.scope.Address().Bytes() res, err := co.toBuf(co.vm, addr) if err != nil { co.vm.Interrupt(err) @@ -816,7 +843,7 @@ func (co *contractObj) GetAddress() goja.Value { } func (co *contractObj) GetValue() goja.Value { - value := co.contract.Value() + value := co.scope.CallValue() res, err := co.toBig(co.vm, value.String()) if err != nil { co.vm.Interrupt(err) @@ -826,7 +853,7 @@ func (co *contractObj) GetValue() goja.Value { } func (co *contractObj) GetInput() goja.Value { - input := common.CopyBytes(co.contract.Input) + input := common.CopyBytes(co.scope.CallInput()) res, err := co.toBuf(co.vm, input) if err != nil { co.vm.Interrupt(err) diff --git a/eth/tracers/js/internal/tracers/call_tracer_legacy.js b/eth/tracers/js/internal/tracers/call_tracer_legacy.js index 451a644b91..0760bb1e3f 100644 --- a/eth/tracers/js/internal/tracers/call_tracer_legacy.js +++ b/eth/tracers/js/internal/tracers/call_tracer_legacy.js @@ -219,7 +219,7 @@ return this.finalize(result); }, - // finalize recreates a call object using the final desired field oder for json + // finalize recreates a call object using the final desired field order for json // serialization. This is a nicety feature to pass meaningfully ordered results // to users who don't interpret it, just display it. finalize: function(call) { diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 339e834c9f..cf746085ca 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -62,9 +62,9 @@ func testCtx() *vmContext { return &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1)}, txCtx: vm.TxContext{GasPrice: big.NewInt(100000)}} } -func runTrace(tracer directory.Tracer, vmctx *vmContext, chaincfg *params.ChainConfig, contractCode []byte) (json.RawMessage, error) { +func runTrace(tracer *directory.Tracer, vmctx *vmContext, chaincfg *params.ChainConfig, contractCode []byte) (json.RawMessage, error) { var ( - env = vm.NewEVM(vmctx.blockCtx, vmctx.txCtx, &dummyStatedb{}, chaincfg, vm.Config{Tracer: tracer}) + env = vm.NewEVM(vmctx.blockCtx, vmctx.txCtx, &dummyStatedb{}, chaincfg, vm.Config{Tracer: tracer.Hooks}) gasLimit uint64 = 31000 startGas uint64 = 10000 value = uint256.NewInt(0) @@ -75,12 +75,12 @@ func runTrace(tracer directory.Tracer, vmctx *vmContext, chaincfg *params.ChainC contract.Code = contractCode } - tracer.CaptureTxStart(env, types.NewTx(&types.LegacyTx{Gas: gasLimit}), contract.Caller()) - tracer.CaptureStart(contract.Caller(), contract.Address(), false, []byte{}, startGas, value.ToBig()) + tracer.OnTxStart(env.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit}), contract.Caller()) + tracer.OnEnter(0, byte(vm.CALL), contract.Caller(), contract.Address(), []byte{}, startGas, value.ToBig()) ret, err := env.Interpreter().Run(contract, []byte{}, false) - tracer.CaptureEnd(ret, startGas-contract.Gas, err, true) + tracer.OnExit(0, ret, startGas-contract.Gas, err, true) // Rest gas assumes no refund - tracer.CaptureTxEnd(&types.Receipt{GasUsed: gasLimit - contract.Gas}, nil) + tracer.OnTxEnd(&types.Receipt{GasUsed: gasLimit - contract.Gas}, nil) if err != nil { return nil, err } @@ -185,13 +185,13 @@ func TestHaltBetweenSteps(t *testing.T) { scope := &vm.ScopeContext{ Contract: vm.NewContract(&account{}, &account{}, uint256.NewInt(0), 0), } - env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{GasPrice: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: tracer}) - tracer.CaptureTxStart(env, types.NewTx(&types.LegacyTx{}), common.Address{}) - tracer.CaptureStart(common.Address{}, common.Address{}, false, []byte{}, 0, big.NewInt(0)) - tracer.CaptureState(0, 0, 0, 0, scope, nil, 0, nil) + env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{GasPrice: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: tracer.Hooks}) + tracer.OnTxStart(env.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) + tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 0, big.NewInt(0)) + tracer.OnOpcode(0, 0, 0, 0, scope, nil, 0, nil) timeout := errors.New("stahp") tracer.Stop(timeout) - tracer.CaptureState(0, 0, 0, 0, scope, nil, 0, nil) + tracer.OnOpcode(0, 0, 0, 0, scope, nil, 0, nil) if _, err := tracer.GetResult(); !strings.Contains(err.Error(), timeout.Error()) { t.Errorf("Expected timeout error, got %v", err) @@ -207,10 +207,10 @@ func TestNoStepExec(t *testing.T) { if err != nil { t.Fatal(err) } - env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{GasPrice: big.NewInt(100)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: tracer}) - tracer.CaptureTxStart(env, types.NewTx(&types.LegacyTx{}), common.Address{}) - tracer.CaptureStart(common.Address{}, common.Address{}, false, []byte{}, 1000, big.NewInt(0)) - tracer.CaptureEnd(nil, 0, nil, false) + env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{GasPrice: big.NewInt(100)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: tracer.Hooks}) + tracer.OnTxStart(env.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) + tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 1000, big.NewInt(0)) + tracer.OnExit(0, nil, 0, nil, false) ret, err := tracer.GetResult() if err != nil { t.Fatal(err) @@ -279,8 +279,8 @@ func TestEnterExit(t *testing.T) { scope := &vm.ScopeContext{ Contract: vm.NewContract(&account{}, &account{}, uint256.NewInt(0), 0), } - tracer.CaptureEnter(vm.CALL, scope.Contract.Caller(), scope.Contract.Address(), []byte{}, 1000, new(big.Int)) - tracer.CaptureExit([]byte{}, 400, nil, false) + tracer.OnEnter(1, byte(vm.CALL), scope.Contract.Caller(), scope.Contract.Address(), []byte{}, 1000, new(big.Int)) + tracer.OnExit(1, []byte{}, 400, nil, false) have, err := tracer.GetResult() if err != nil { diff --git a/eth/tracers/live/noop.go b/eth/tracers/live/noop.go new file mode 100644 index 0000000000..2236674201 --- /dev/null +++ b/eth/tracers/live/noop.go @@ -0,0 +1,96 @@ +package live + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/tracers/directory/live" + "github.com/ethereum/go-ethereum/params" +) + +func init() { + live.Directory.Register("noop", newNoopTracer) +} + +// noop is a no-op live tracer. It's there to +// catch changes in the tracing interface, as well as +// for testing live tracing performance. Can be removed +// as soon as we have a real live tracer. +type noop struct{} + +func newNoopTracer(_ json.RawMessage) (*tracing.Hooks, error) { + t := &noop{} + return &tracing.Hooks{ + OnTxStart: t.OnTxStart, + OnTxEnd: t.OnTxEnd, + OnEnter: t.OnEnter, + OnExit: t.OnExit, + OnOpcode: t.OnOpcode, + OnFault: t.OnFault, + OnGasChange: t.OnGasChange, + OnBlockchainInit: t.OnBlockchainInit, + OnBlockStart: t.OnBlockStart, + OnBlockEnd: t.OnBlockEnd, + OnSkippedBlock: t.OnSkippedBlock, + OnGenesisBlock: t.OnGenesisBlock, + OnBalanceChange: t.OnBalanceChange, + OnNonceChange: t.OnNonceChange, + OnCodeChange: t.OnCodeChange, + OnStorageChange: t.OnStorageChange, + OnLog: t.OnLog, + }, nil +} + +func (t *noop) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { +} + +func (t *noop) OnFault(pc uint64, op byte, gas, cost uint64, _ tracing.OpContext, depth int, err error) { +} + +func (t *noop) OnEnter(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { +} + +func (t *noop) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { +} + +func (t *noop) OnTxStart(vm *tracing.VMContext, tx *types.Transaction, from common.Address) { +} + +func (t *noop) OnTxEnd(receipt *types.Receipt, err error) { +} + +func (t *noop) OnBlockStart(ev tracing.BlockEvent) { +} + +func (t *noop) OnBlockEnd(err error) { +} + +func (t *noop) OnSkippedBlock(ev tracing.BlockEvent) {} + +func (t *noop) OnBlockchainInit(chainConfig *params.ChainConfig) { +} + +func (t *noop) OnGenesisBlock(b *types.Block, alloc types.GenesisAlloc) { +} + +func (t *noop) OnBalanceChange(a common.Address, prev, new *big.Int, reason tracing.BalanceChangeReason) { +} + +func (t *noop) OnNonceChange(a common.Address, prev, new uint64) { +} + +func (t *noop) OnCodeChange(a common.Address, prevCodeHash common.Hash, prev []byte, codeHash common.Hash, code []byte) { +} + +func (t *noop) OnStorageChange(a common.Address, k, prev, new common.Hash) { +} + +func (t *noop) OnLog(l *types.Log) { + +} + +func (t *noop) OnGasChange(old, new uint64, reason tracing.GasChangeReason) { +} diff --git a/eth/tracers/live/printer.go b/eth/tracers/live/printer.go deleted file mode 100644 index a959c0cac6..0000000000 --- a/eth/tracers/live/printer.go +++ /dev/null @@ -1,134 +0,0 @@ -package live - -import ( - "encoding/json" - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/tracers/directory" - "github.com/ethereum/go-ethereum/params" -) - -func init() { - directory.LiveDirectory.Register("printer", newPrinter) -} - -type Printer struct{} - -func newPrinter() (core.BlockchainLogger, error) { - return &Printer{}, nil -} - -// CaptureStart implements the EVMLogger interface to initialize the tracing operation. -func (p *Printer) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { - fmt.Printf("CaptureStart: from=%v, to=%v, create=%v, input=%s, gas=%v, value=%v\n", from, to, create, hexutil.Bytes(input), gas, value) -} - -// CaptureEnd is called after the call finishes to finalize the tracing. -func (p *Printer) CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) { - fmt.Printf("CaptureEnd: output=%s, gasUsed=%v, err=%v\n", hexutil.Bytes(output), gasUsed, err) -} - -// CaptureState implements the EVMLogger interface to trace a single step of VM execution. -func (p *Printer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { - //fmt.Printf("CaptureState: pc=%v, op=%v, gas=%v, cost=%v, scope=%v, rData=%s, depth=%v, err=%v\n", pc, op, gas, cost, scope, hexutil.Bytes(rData), depth, err) -} - -// CaptureFault implements the EVMLogger interface to trace an execution fault. -func (p *Printer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, _ *vm.ScopeContext, depth int, err error) { - fmt.Printf("CaptureFault: pc=%v, op=%v, gas=%v, cost=%v, depth=%v, err=%v\n", pc, op, gas, cost, depth, err) -} - -// CaptureKeccakPreimage is called during the KECCAK256 opcode. -func (p *Printer) CaptureKeccakPreimage(hash common.Hash, data []byte) {} - -// CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). -func (p *Printer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { - fmt.Printf("CaptureEnter: typ=%v, from=%v, to=%v, input=%s, gas=%v, value=%v\n", typ, from, to, hexutil.Bytes(input), gas, value) -} - -// CaptureExit is called when EVM exits a scope, even if the scope didn't -// execute any code. -func (p *Printer) CaptureExit(output []byte, gasUsed uint64, err error, reverted bool) { - fmt.Printf("CaptureExit: output=%s, gasUsed=%v, err=%v\n", hexutil.Bytes(output), gasUsed, err) -} - -func (p *Printer) OnBeaconBlockRootStart(root common.Hash) {} -func (p *Printer) OnBeaconBlockRootEnd() {} - -func (p *Printer) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common.Address) { - buf, err := json.Marshal(tx) - if err != nil { - fmt.Printf("err: %v\n", err) - return - } - fmt.Printf("CaptureTxStart: tx=%s\n", buf) -} - -func (p *Printer) CaptureTxEnd(receipt *types.Receipt, err error) { - if err != nil { - fmt.Printf("CaptureTxEnd err: %v\n", err) - return - } - buf, err := json.Marshal(receipt) - if err != nil { - fmt.Printf("err: %v\n", err) - return - } - fmt.Printf("CaptureTxEnd: receipt=%s\n", buf) -} - -func (p *Printer) OnBlockStart(b *types.Block, td *big.Int, finalized, safe *types.Header, _ *params.ChainConfig) { - if finalized != nil && safe != nil { - fmt.Printf("OnBlockStart: b=%v, td=%v, finalized=%v, safe=%v\n", b.NumberU64(), td, finalized.Number.Uint64(), safe.Number.Uint64()) - } else { - fmt.Printf("OnBlockStart: b=%v, td=%v\n", b.NumberU64(), td) - } -} - -func (p *Printer) OnBlockEnd(err error) { - fmt.Printf("OnBlockEnd: err=%v\n", err) -} - -func (p *Printer) OnGenesisBlock(b *types.Block, alloc core.GenesisAlloc, chainConfig *params.ChainConfig) { - fmt.Printf("OnGenesisBlock: b=%v, allocLength=%d\n", b.NumberU64(), len(alloc)) -} - -func (p *Printer) OnBalanceChange(a common.Address, prev, new *big.Int, reason state.BalanceChangeReason) { - fmt.Printf("OnBalanceChange: a=%v, prev=%v, new=%v\n", a, prev, new) -} - -func (p *Printer) OnNonceChange(a common.Address, prev, new uint64) { - fmt.Printf("OnNonceChange: a=%v, prev=%v, new=%v\n", a, prev, new) -} - -func (p *Printer) OnCodeChange(a common.Address, prevCodeHash common.Hash, prev []byte, codeHash common.Hash, code []byte) { - fmt.Printf("OnCodeChange: a=%v, prevCodeHash=%v, prev=%s, codeHash=%v, code=%s\n", a, prevCodeHash, hexutil.Bytes(prev), codeHash, hexutil.Bytes(code)) -} - -func (p *Printer) OnStorageChange(a common.Address, k, prev, new common.Hash) { - fmt.Printf("OnStorageChange: a=%v, k=%v, prev=%v, new=%v\n", a, k, prev, new) -} - -func (p *Printer) OnLog(l *types.Log) { - buf, err := json.Marshal(l) - if err != nil { - fmt.Printf("err: %v\n", err) - return - } - fmt.Printf("OnLog: l=%s\n", buf) -} - -func (p *Printer) OnNewAccount(a common.Address, reset bool) { - fmt.Printf("OnNewAccount: a=%v\n", a) -} - -func (p *Printer) OnGasChange(old, new uint64, reason vm.GasChangeReason) { - fmt.Printf("OnGasChange: old=%v, new=%v, diff=%v\n", old, new, new-old) -} diff --git a/eth/tracers/logger/access_list_tracer.go b/eth/tracers/logger/access_list_tracer.go index 7df66066b5..fc7713b245 100644 --- a/eth/tracers/logger/access_list_tracer.go +++ b/eth/tracers/logger/access_list_tracer.go @@ -18,9 +18,9 @@ package logger import ( "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/tracers/directory" ) // accessList is an accumulator for the set of accounts and storage slots an EVM @@ -102,7 +102,6 @@ func (al accessList) accessList() types.AccessList { // AccessListTracer is a tracer that accumulates touched accounts and storage // slots into an internal set. type AccessListTracer struct { - directory.NoopTracer excl map[common.Address]struct{} // Set of account to exclude from the list list accessList // Set of accounts and storage slots touched } @@ -132,14 +131,20 @@ func NewAccessListTracer(acl types.AccessList, from, to common.Address, precompi } } -// CaptureState captures all opcodes that touch storage or addresses and adds them to the accesslist. -func (a *AccessListTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { - stack := scope.Stack - stackData := stack.Data() +func (a *AccessListTracer) Hooks() *tracing.Hooks { + return &tracing.Hooks{ + OnOpcode: a.OnOpcode, + } +} + +// OnOpcode captures all opcodes that touch storage or addresses and adds them to the accesslist. +func (a *AccessListTracer) OnOpcode(pc uint64, opcode byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { + stackData := scope.StackData() stackLen := len(stackData) + op := vm.OpCode(opcode) if (op == vm.SLOAD || op == vm.SSTORE) && stackLen >= 1 { slot := common.Hash(stackData[stackLen-1].Bytes32()) - a.list.addSlot(scope.Contract.Address(), slot) + a.list.addSlot(scope.Address(), slot) } if (op == vm.EXTCODECOPY || op == vm.EXTCODEHASH || op == vm.EXTCODESIZE || op == vm.BALANCE || op == vm.SELFDESTRUCT) && stackLen >= 1 { addr := common.Address(stackData[stackLen-1].Bytes20()) diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index c3bc494280..9d6f4c8a19 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/directory" @@ -108,9 +109,8 @@ func (s *StructLog) ErrorString() string { // a track record of modified storage which is used in reporting snapshots of the // contract their storage. type StructLogger struct { - directory.NoopTracer cfg Config - env *vm.EVM + env *tracing.VMContext storage map[common.Address]Storage logs []StructLog @@ -133,6 +133,23 @@ func NewStructLogger(cfg *Config) *StructLogger { return logger } +func (l *StructLogger) Hooks() *tracing.Hooks { + return &tracing.Hooks{ + OnTxStart: l.OnTxStart, + OnTxEnd: l.OnTxEnd, + OnExit: l.OnExit, + OnOpcode: l.OnOpcode, + } +} + +func (l *StructLogger) Tracer() *directory.Tracer { + return &directory.Tracer{ + Hooks: l.Hooks(), + GetResult: l.GetResult, + Stop: l.Stop, + } +} + // Reset clears the data held by the logger. func (l *StructLogger) Reset() { l.storage = make(map[common.Address]Storage) @@ -141,10 +158,10 @@ func (l *StructLogger) Reset() { l.err = nil } -// CaptureState logs a new structured log message and pushes it out to the environment +// OnOpcode logs a new structured log message and pushes it out to the environment // -// CaptureState also tracks SLOAD/SSTORE ops to track storage change. -func (l *StructLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { +// OnOpcode also tracks SLOAD/SSTORE ops to track storage change. +func (l *StructLogger) OnOpcode(pc uint64, opcode byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { // If tracing was interrupted, set the error and stop if l.interrupt.Load() { return @@ -154,49 +171,47 @@ func (l *StructLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, s return } - memory := scope.Memory - stack := scope.Stack - contract := scope.Contract + op := vm.OpCode(opcode) + memory := scope.MemoryData() + stack := scope.StackData() // Copy a snapshot of the current memory state to a new buffer var mem []byte if l.cfg.EnableMemory { - mem = make([]byte, len(memory.Data())) - copy(mem, memory.Data()) + mem = make([]byte, len(memory)) + copy(mem, memory) } // Copy a snapshot of the current stack state to a new buffer var stck []uint256.Int if !l.cfg.DisableStack { - stck = make([]uint256.Int, len(stack.Data())) - for i, item := range stack.Data() { - stck[i] = item - } + stck = make([]uint256.Int, len(stack)) + copy(stck, stack) } - stackData := stack.Data() - stackLen := len(stackData) + contractAddr := scope.Address() + stackLen := len(stack) // Copy a snapshot of the current storage to a new container var storage Storage if !l.cfg.DisableStorage && (op == vm.SLOAD || op == vm.SSTORE) { // initialise new changed values storage container for this contract // if not present. - if l.storage[contract.Address()] == nil { - l.storage[contract.Address()] = make(Storage) + if l.storage[contractAddr] == nil { + l.storage[contractAddr] = make(Storage) } // capture SLOAD opcodes and record the read entry in the local storage if op == vm.SLOAD && stackLen >= 1 { var ( - address = common.Hash(stackData[stackLen-1].Bytes32()) - value = l.env.StateDB.GetState(contract.Address(), address) + address = common.Hash(stack[stackLen-1].Bytes32()) + value = l.env.StateDB.GetState(contractAddr, address) ) - l.storage[contract.Address()][address] = value - storage = l.storage[contract.Address()].Copy() + l.storage[contractAddr][address] = value + storage = l.storage[contractAddr].Copy() } else if op == vm.SSTORE && stackLen >= 2 { // capture SSTORE opcodes and record the written entry in the local storage. var ( - value = common.Hash(stackData[stackLen-2].Bytes32()) - address = common.Hash(stackData[stackLen-1].Bytes32()) + value = common.Hash(stack[stackLen-2].Bytes32()) + address = common.Hash(stack[stackLen-1].Bytes32()) ) - l.storage[contract.Address()][address] = value - storage = l.storage[contract.Address()].Copy() + l.storage[contractAddr][address] = value + storage = l.storage[contractAddr].Copy() } } var rdata []byte @@ -205,12 +220,15 @@ func (l *StructLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, s copy(rdata, rData) } // create a new snapshot of the EVM. - log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, rdata, storage, depth, l.env.StateDB.GetRefund(), err} + log := StructLog{pc, op, gas, cost, mem, len(memory), stck, rdata, storage, depth, l.env.StateDB.GetRefund(), err} l.logs = append(l.logs, log) } -// CaptureEnd is called after the call finishes to finalize the tracing. -func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) { +// OnExit is called a call frame finishes processing. +func (l *StructLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { + if depth != 0 { + return + } l.output = output l.err = err if l.cfg.Debug { @@ -247,11 +265,11 @@ func (l *StructLogger) Stop(err error) { l.interrupt.Store(true) } -func (l *StructLogger) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common.Address) { +func (l *StructLogger) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { l.env = env } -func (l *StructLogger) CaptureTxEnd(receipt *types.Receipt, err error) { +func (l *StructLogger) OnTxEnd(receipt *types.Receipt, err error) { if err != nil { // Don't override vm error if l.err == nil { @@ -319,10 +337,9 @@ func WriteLogs(writer io.Writer, logs []*types.Log) { } type mdLogger struct { - directory.NoopTracer out io.Writer cfg *Config - env *vm.EVM + env *tracing.VMContext } // NewMarkdownLogger creates a logger which outputs information in a format adapted @@ -335,7 +352,25 @@ func NewMarkdownLogger(cfg *Config, writer io.Writer) *mdLogger { return l } -func (t *mdLogger) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { +func (t *mdLogger) Hooks() *tracing.Hooks { + return &tracing.Hooks{ + OnTxStart: t.OnTxStart, + OnEnter: t.OnEnter, + OnExit: t.OnExit, + OnOpcode: t.OnOpcode, + OnFault: t.OnFault, + } +} + +func (t *mdLogger) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { + t.env = env +} + +func (t *mdLogger) OnEnter(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + if depth != 0 { + return + } + create := vm.OpCode(typ) == vm.CREATE if !create { fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", from.String(), to.String(), @@ -352,15 +387,22 @@ func (t *mdLogger) CaptureStart(from common.Address, to common.Address, create b `) } -// CaptureState also tracks SLOAD/SSTORE ops to track storage change. -func (t *mdLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { - stack := scope.Stack +func (t *mdLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { + if depth == 0 { + fmt.Fprintf(t.out, "\nOutput: `%#x`\nConsumed gas: `%d`\nError: `%v`\n", + output, gasUsed, err) + } +} + +// OnOpcode also tracks SLOAD/SSTORE ops to track storage change. +func (t *mdLogger) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { + stack := scope.StackData() fmt.Fprintf(t.out, "| %4d | %10v | %3d |", pc, op, cost) if !t.cfg.DisableStack { // format stack var a []string - for _, elem := range stack.Data() { + for _, elem := range stack { a = append(a, elem.Hex()) } b := fmt.Sprintf("[%v]", strings.Join(a, ",")) @@ -373,15 +415,10 @@ func (t *mdLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope } } -func (t *mdLogger) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { +func (t *mdLogger) OnFault(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, depth int, err error) { fmt.Fprintf(t.out, "\nError: at pc=%d, op=%v: %v\n", pc, op, err) } -func (t *mdLogger) CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) { - fmt.Fprintf(t.out, "\nOutput: `%#x`\nConsumed gas: `%d`\nError: `%v`\n", - output, gasUsed, err) -} - // ExecutionResult groups all structured logs emitted by the EVM // while replaying a transaction in debug mode as well as transaction // execution status, the amount of gas used and the return value diff --git a/eth/tracers/logger/logger_json.go b/eth/tracers/logger/logger_json.go index 1bf88f8899..e9a4508969 100644 --- a/eth/tracers/logger/logger_json.go +++ b/eth/tracers/logger/logger_json.go @@ -22,16 +22,15 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/tracers/directory" ) type JSONLogger struct { - directory.NoopTracer encoder *json.Encoder cfg *Config - env *vm.EVM + env *tracing.VMContext } // NewJSONLogger creates a new EVM tracer that prints execution steps as JSON objects @@ -44,31 +43,39 @@ func NewJSONLogger(cfg *Config, writer io.Writer) *JSONLogger { return l } -func (l *JSONLogger) CaptureFault(pc uint64, op vm.OpCode, gas uint64, cost uint64, scope *vm.ScopeContext, depth int, err error) { - // TODO: Add rData to this interface as well - l.CaptureState(pc, op, gas, cost, scope, nil, depth, err) +func (l *JSONLogger) Hooks() *tracing.Hooks { + return &tracing.Hooks{ + OnTxStart: l.OnTxStart, + OnExit: l.OnExit, + OnOpcode: l.OnOpcode, + OnFault: l.OnFault, + } } -// CaptureState outputs state information on the logger. -func (l *JSONLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { - memory := scope.Memory - stack := scope.Stack +func (l *JSONLogger) OnFault(pc uint64, op byte, gas uint64, cost uint64, scope tracing.OpContext, depth int, err error) { + // TODO: Add rData to this interface as well + l.OnOpcode(pc, op, gas, cost, scope, nil, depth, err) +} + +func (l *JSONLogger) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { + memory := scope.MemoryData() + stack := scope.StackData() log := StructLog{ Pc: pc, - Op: op, + Op: vm.OpCode(op), Gas: gas, GasCost: cost, - MemorySize: memory.Len(), + MemorySize: len(memory), Depth: depth, RefundCounter: l.env.StateDB.GetRefund(), Err: err, } if l.cfg.EnableMemory { - log.Memory = memory.Data() + log.Memory = memory } if !l.cfg.DisableStack { - log.Stack = stack.Data() + log.Stack = stack } if l.cfg.EnableReturnData { log.ReturnData = rData @@ -76,8 +83,10 @@ func (l *JSONLogger) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, sco l.encoder.Encode(log) } -// CaptureEnd is triggered at end of execution. -func (l *JSONLogger) CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) { +func (l *JSONLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { + if depth > 0 { + return + } type endLog struct { Output string `json:"output"` GasUsed math.HexOrDecimal64 `json:"gasUsed"` @@ -90,6 +99,6 @@ func (l *JSONLogger) CaptureEnd(output []byte, gasUsed uint64, err error, revert l.encoder.Encode(endLog{common.Bytes2Hex(output), math.HexOrDecimal64(gasUsed), errMsg}) } -func (l *JSONLogger) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common.Address) { +func (l *JSONLogger) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { l.env = env } diff --git a/eth/tracers/logger/logger_test.go b/eth/tracers/logger/logger_test.go index 816f9522c1..137608f884 100644 --- a/eth/tracers/logger/logger_test.go +++ b/eth/tracers/logger/logger_test.go @@ -56,13 +56,12 @@ func (*dummyStatedb) SetState(_ common.Address, _ common.Hash, _ common.Hash) {} func TestStoreCapture(t *testing.T) { var ( logger = NewStructLogger(nil) - env = vm.NewEVM(vm.BlockContext{}, vm.TxContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: logger}) + env = vm.NewEVM(vm.BlockContext{}, vm.TxContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: logger.Hooks()}) contract = vm.NewContract(&dummyContractRef{}, &dummyContractRef{}, new(uint256.Int), 100000) ) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x0, byte(vm.SSTORE)} var index common.Hash - logger.CaptureTxStart(env, nil, common.Address{}) - logger.CaptureStart(common.Address{}, contract.Address(), false, nil, 0, nil) + logger.OnTxStart(env.GetVMContext(), nil, common.Address{}) _, err := env.Interpreter().Run(contract, []byte{}, false) if err != nil { t.Fatal(err) diff --git a/eth/tracers/native/4byte.go b/eth/tracers/native/4byte.go index c7c45cfaa4..550b0115a3 100644 --- a/eth/tracers/native/4byte.go +++ b/eth/tracers/native/4byte.go @@ -23,6 +23,7 @@ import ( "sync/atomic" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/directory" @@ -47,21 +48,26 @@ func init() { // 0xc281d19e-0: 1 // } type fourByteTracer struct { - directory.NoopTracer - env *vm.EVM ids map[string]int // ids aggregates the 4byte ids found interrupt atomic.Bool // Atomic flag to signal execution interruption reason error // Textual reason for the interruption - activePrecompiles []common.Address // Updated on CaptureStart based on given rules + activePrecompiles []common.Address // Updated on tx start based on given rules } // newFourByteTracer returns a native go tracer which collects // 4 byte-identifiers of a tx, and implements vm.EVMLogger. -func newFourByteTracer(ctx *directory.Context, _ json.RawMessage) (directory.Tracer, error) { +func newFourByteTracer(ctx *directory.Context, _ json.RawMessage) (*directory.Tracer, error) { t := &fourByteTracer{ ids: make(map[string]int), } - return t, nil + return &directory.Tracer{ + Hooks: &tracing.Hooks{ + OnTxStart: t.OnTxStart, + OnEnter: t.OnEnter, + }, + GetResult: t.GetResult, + Stop: t.Stop, + }, nil } // isPrecompiled returns whether the addr is a precompile. Logic borrowed from newJsTracer in eth/tracers/js/tracer.go @@ -80,23 +86,14 @@ func (t *fourByteTracer) store(id []byte, size int) { t.ids[key] += 1 } -func (t *fourByteTracer) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common.Address) { - t.env = env +func (t *fourByteTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { // Update list of precompiles based on current block - rules := t.env.ChainConfig().Rules(t.env.Context.BlockNumber, t.env.Context.Random != nil, t.env.Context.Time) + rules := env.ChainConfig.Rules(env.BlockNumber, env.Random != nil, env.Time) t.activePrecompiles = vm.ActivePrecompiles(rules) } -// CaptureStart implements the EVMLogger interface to initialize the tracing operation. -func (t *fourByteTracer) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { - // Save the outer calldata - if len(input) >= 4 { - t.store(input[0:4], len(input)-4) - } -} - -// CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). -func (t *fourByteTracer) CaptureEnter(op vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { +// OnEnter is called when EVM enters a new scope (via call, create or selfdestruct). +func (t *fourByteTracer) OnEnter(depth int, opcode byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { // Skip if tracing was interrupted if t.interrupt.Load() { return @@ -104,6 +101,7 @@ func (t *fourByteTracer) CaptureEnter(op vm.OpCode, from common.Address, to comm if len(input) < 4 { return } + op := vm.OpCode(opcode) // primarily we want to avoid CREATE/CREATE2/SELFDESTRUCT if op != vm.DELEGATECALL && op != vm.STATICCALL && op != vm.CALL && op != vm.CALLCODE { diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index 43bd0198da..83db1139f1 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/directory" @@ -104,7 +105,6 @@ type callFrameMarshaling struct { } type callTracer struct { - directory.NoopTracer callstack []callFrame config callTracerConfig gasLimit uint64 @@ -120,7 +120,25 @@ type callTracerConfig struct { // newCallTracer returns a native go tracer which tracks // call frames of a tx, and implements vm.EVMLogger. -func newCallTracer(ctx *directory.Context, cfg json.RawMessage) (directory.Tracer, error) { +func newCallTracer(ctx *directory.Context, cfg json.RawMessage) (*directory.Tracer, error) { + t, err := newCallTracerObject(ctx, cfg) + if err != nil { + return nil, err + } + return &directory.Tracer{ + Hooks: &tracing.Hooks{ + OnTxStart: t.OnTxStart, + OnTxEnd: t.OnTxEnd, + OnEnter: t.OnEnter, + OnExit: t.OnExit, + OnLog: t.OnLog, + }, + GetResult: t.GetResult, + Stop: t.Stop, + }, nil +} + +func newCallTracerObject(ctx *directory.Context, cfg json.RawMessage) (*callTracer, error) { var config callTracerConfig if cfg != nil { if err := json.Unmarshal(cfg, &config); err != nil { @@ -129,38 +147,13 @@ func newCallTracer(ctx *directory.Context, cfg json.RawMessage) (directory.Trace } // First callframe contains tx context info // and is populated on start and end. - return &callTracer{callstack: make([]callFrame, 1), config: config}, nil + return &callTracer{callstack: make([]callFrame, 0, 1), config: config}, nil } -// CaptureStart implements the EVMLogger interface to initialize the tracing operation. -func (t *callTracer) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { - toCopy := to - t.callstack[0] = callFrame{ - Type: vm.CALL, - From: from, - To: &toCopy, - Input: common.CopyBytes(input), - Gas: t.gasLimit, - Value: value, - } - if create { - t.callstack[0].Type = vm.CREATE - } -} - -// CaptureEnd is called after the call finishes to finalize the tracing. -func (t *callTracer) CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) { - t.callstack[0].processOutput(output, err, reverted) -} - -// CaptureState implements the EVMLogger interface to trace a single step of VM execution. -func (t *callTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { -} - -// CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). -func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { - t.depth++ - if t.config.OnlyTopCall { +// OnEnter is called when EVM enters a new scope (via call, create or selfdestruct). +func (t *callTracer) OnEnter(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + t.depth = depth + if t.config.OnlyTopCall && depth > 0 { return } // Skip if tracing was interrupted @@ -170,42 +163,59 @@ func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common. toCopy := to call := callFrame{ - Type: typ, + Type: vm.OpCode(typ), From: from, To: &toCopy, Input: common.CopyBytes(input), Gas: gas, Value: value, } + if depth == 0 { + call.Gas = t.gasLimit + } t.callstack = append(t.callstack, call) } -// CaptureExit is called when EVM exits a scope, even if the scope didn't +// OnExit is called when EVM exits a scope, even if the scope didn't // execute any code. -func (t *callTracer) CaptureExit(output []byte, gasUsed uint64, err error, reverted bool) { - t.depth-- +func (t *callTracer) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { + if depth == 0 { + t.captureEnd(output, gasUsed, err, reverted) + return + } + + t.depth = depth - 1 if t.config.OnlyTopCall { return } + size := len(t.callstack) if size <= 1 { return } - // pop call + // Pop call. call := t.callstack[size-1] t.callstack = t.callstack[:size-1] size -= 1 call.GasUsed = gasUsed call.processOutput(output, err, reverted) + // Nest call into parent. t.callstack[size-1].Calls = append(t.callstack[size-1].Calls, call) } -func (t *callTracer) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common.Address) { +func (t *callTracer) captureEnd(output []byte, gasUsed uint64, err error, reverted bool) { + if len(t.callstack) != 1 { + return + } + t.callstack[0].processOutput(output, err, reverted) +} + +func (t *callTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { t.gasLimit = tx.Gas() } -func (t *callTracer) CaptureTxEnd(receipt *types.Receipt, err error) { +func (t *callTracer) OnTxEnd(receipt *types.Receipt, err error) { // Error happened during tx validation. if err != nil { return diff --git a/eth/tracers/native/call_flat.go b/eth/tracers/native/call_flat.go index 26f114f44f..cd53736ed9 100644 --- a/eth/tracers/native/call_flat.go +++ b/eth/tracers/native/call_flat.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/directory" @@ -109,12 +110,11 @@ type flatCallResultMarshaling struct { // flatCallTracer reports call frame information of a tx in a flat format, i.e. // as opposed to the nested format of `callTracer`. type flatCallTracer struct { - directory.NoopTracer tracer *callTracer config flatCallTracerConfig ctx *directory.Context // Holds tracer context data reason error // Textual reason for the interruption - activePrecompiles []common.Address // Updated on CaptureStart based on given rules + activePrecompiles []common.Address // Updated on tx start based on given rules } type flatCallTracerConfig struct { @@ -123,7 +123,7 @@ type flatCallTracerConfig struct { } // newFlatCallTracer returns a new flatCallTracer. -func newFlatCallTracer(ctx *directory.Context, cfg json.RawMessage) (directory.Tracer, error) { +func newFlatCallTracer(ctx *directory.Context, cfg json.RawMessage) (*directory.Tracer, error) { var config flatCallTracerConfig if cfg != nil { if err := json.Unmarshal(cfg, &config); err != nil { @@ -133,42 +133,31 @@ func newFlatCallTracer(ctx *directory.Context, cfg json.RawMessage) (directory.T // Create inner call tracer with default configuration, don't forward // the OnlyTopCall or WithLog to inner for now - tracer, err := directory.DefaultDirectory.New("callTracer", ctx, nil) + t, err := newCallTracerObject(ctx, nil) if err != nil { return nil, err } - t, ok := tracer.(*callTracer) - if !ok { - return nil, errors.New("internal error: embedded tracer has wrong type") + + ft := &flatCallTracer{tracer: t, ctx: ctx, config: config} + return &directory.Tracer{ + Hooks: &tracing.Hooks{ + OnTxStart: ft.OnTxStart, + OnTxEnd: ft.OnTxEnd, + OnEnter: ft.OnEnter, + OnExit: ft.OnExit, + }, + Stop: ft.Stop, + GetResult: ft.GetResult, + }, nil +} + +// OnEnter is called when EVM enters a new scope (via call, create or selfdestruct). +func (t *flatCallTracer) OnEnter(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + t.tracer.OnEnter(depth, typ, from, to, input, gas, value) + + if depth == 0 { + return } - - return &flatCallTracer{tracer: t, ctx: ctx, config: config}, nil -} - -// CaptureStart implements the EVMLogger interface to initialize the tracing operation. -func (t *flatCallTracer) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { - t.tracer.CaptureStart(from, to, create, input, gas, value) -} - -// CaptureEnd is called after the call finishes to finalize the tracing. -func (t *flatCallTracer) CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) { - t.tracer.CaptureEnd(output, gasUsed, err, reverted) -} - -// CaptureState implements the EVMLogger interface to trace a single step of VM execution. -func (t *flatCallTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { - t.tracer.CaptureState(pc, op, gas, cost, scope, rData, depth, err) -} - -// CaptureFault implements the EVMLogger interface to trace an execution fault. -func (t *flatCallTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { - t.tracer.CaptureFault(pc, op, gas, cost, scope, depth, err) -} - -// CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). -func (t *flatCallTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { - t.tracer.CaptureEnter(typ, from, to, input, gas, value) - // Child calls must have a value, even if it's zero. // Practically speaking, only STATICCALL has nil value. Set it to zero. if t.tracer.callstack[len(t.tracer.callstack)-1].Value == nil && value == nil { @@ -176,11 +165,14 @@ func (t *flatCallTracer) CaptureEnter(typ vm.OpCode, from common.Address, to com } } -// CaptureExit is called when EVM exits a scope, even if the scope didn't +// OnExit is called when EVM exits a scope, even if the scope didn't // execute any code. -func (t *flatCallTracer) CaptureExit(output []byte, gasUsed uint64, err error, reverted bool) { - t.tracer.CaptureExit(output, gasUsed, err, reverted) +func (t *flatCallTracer) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { + t.tracer.OnExit(depth, output, gasUsed, err, reverted) + if depth == 0 { + return + } // Parity traces don't include CALL/STATICCALLs to precompiles. // By default we remove them from the callstack. if t.config.IncludePrecompiles { @@ -200,15 +192,15 @@ func (t *flatCallTracer) CaptureExit(output []byte, gasUsed uint64, err error, r } } -func (t *flatCallTracer) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common.Address) { - t.tracer.CaptureTxStart(env, tx, from) +func (t *flatCallTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { + t.tracer.OnTxStart(env, tx, from) // Update list of precompiles based on current block - rules := env.ChainConfig().Rules(env.Context.BlockNumber, env.Context.Random != nil, env.Context.Time) + rules := env.ChainConfig.Rules(env.BlockNumber, env.Random != nil, env.Time) t.activePrecompiles = vm.ActivePrecompiles(rules) } -func (t *flatCallTracer) CaptureTxEnd(receipt *types.Receipt, err error) { - t.tracer.CaptureTxEnd(receipt, err) +func (t *flatCallTracer) OnTxEnd(receipt *types.Receipt, err error) { + t.tracer.OnTxEnd(receipt, err) } // GetResult returns an empty json object. diff --git a/eth/tracers/native/mux.go b/eth/tracers/native/mux.go index 1175cc768a..177082ffd9 100644 --- a/eth/tracers/native/mux.go +++ b/eth/tracers/native/mux.go @@ -21,9 +21,8 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/directory" ) @@ -35,18 +34,18 @@ func init() { // runs multiple tracers in one go. type muxTracer struct { names []string - tracers []directory.Tracer + tracers []*directory.Tracer } // newMuxTracer returns a new mux tracer. -func newMuxTracer(ctx *directory.Context, cfg json.RawMessage) (directory.Tracer, error) { +func newMuxTracer(ctx *directory.Context, cfg json.RawMessage) (*directory.Tracer, error) { var config map[string]json.RawMessage if cfg != nil { if err := json.Unmarshal(cfg, &config); err != nil { return nil, err } } - objects := make([]directory.Tracer, 0, len(config)) + objects := make([]*directory.Tracer, 0, len(config)) names := make([]string, 0, len(config)) for k, v := range config { t, err := directory.DefaultDirectory.New(k, ctx, v) @@ -57,111 +56,120 @@ func newMuxTracer(ctx *directory.Context, cfg json.RawMessage) (directory.Tracer names = append(names, k) } - return &muxTracer{names: names, tracers: objects}, nil + t := &muxTracer{names: names, tracers: objects} + return &directory.Tracer{ + Hooks: &tracing.Hooks{ + OnTxStart: t.OnTxStart, + OnTxEnd: t.OnTxEnd, + OnEnter: t.OnEnter, + OnExit: t.OnExit, + OnOpcode: t.OnOpcode, + OnFault: t.OnFault, + OnGasChange: t.OnGasChange, + OnBalanceChange: t.OnBalanceChange, + OnNonceChange: t.OnNonceChange, + OnCodeChange: t.OnCodeChange, + OnStorageChange: t.OnStorageChange, + OnLog: t.OnLog, + }, + GetResult: t.GetResult, + Stop: t.Stop, + }, nil } -// CaptureStart implements the EVMLogger interface to initialize the tracing operation. -func (t *muxTracer) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { +func (t *muxTracer) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { for _, t := range t.tracers { - t.CaptureStart(from, to, create, input, gas, value) + if t.OnOpcode != nil { + t.OnOpcode(pc, op, gas, cost, scope, rData, depth, err) + } } } -// CaptureEnd is called after the call finishes to finalize the tracing. -func (t *muxTracer) CaptureEnd(output []byte, gasUsed uint64, err error, reverted bool) { +func (t *muxTracer) OnFault(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, depth int, err error) { for _, t := range t.tracers { - t.CaptureEnd(output, gasUsed, err, reverted) + if t.OnFault != nil { + t.OnFault(pc, op, gas, cost, scope, depth, err) + } } } -// CaptureState implements the EVMLogger interface to trace a single step of VM execution. -func (t *muxTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { +func (t *muxTracer) OnGasChange(old, new uint64, reason tracing.GasChangeReason) { for _, t := range t.tracers { - t.CaptureState(pc, op, gas, cost, scope, rData, depth, err) + if t.OnGasChange != nil { + t.OnGasChange(old, new, reason) + } } } -// CaptureFault implements the EVMLogger interface to trace an execution fault. -func (t *muxTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { +func (t *muxTracer) OnEnter(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { for _, t := range t.tracers { - t.CaptureFault(pc, op, gas, cost, scope, depth, err) + if t.OnEnter != nil { + t.OnEnter(depth, typ, from, to, input, gas, value) + } } } -// CaptureKeccakPreimage is called during the KECCAK256 opcode. -func (t *muxTracer) CaptureKeccakPreimage(hash common.Hash, data []byte) { +func (t *muxTracer) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { for _, t := range t.tracers { - t.CaptureKeccakPreimage(hash, data) + if t.OnExit != nil { + t.OnExit(depth, output, gasUsed, err, reverted) + } } } -// CaptureGasConsumed is called when gas is consumed. -func (t *muxTracer) OnGasChange(old, new uint64, reason vm.GasChangeReason) { +func (t *muxTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { for _, t := range t.tracers { - t.OnGasChange(old, new, reason) + if t.OnTxStart != nil { + t.OnTxStart(env, tx, from) + } } } -// CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). -func (t *muxTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { +func (t *muxTracer) OnTxEnd(receipt *types.Receipt, err error) { for _, t := range t.tracers { - t.CaptureEnter(typ, from, to, input, gas, value) + if t.OnTxEnd != nil { + t.OnTxEnd(receipt, err) + } } } -// CaptureExit is called when EVM exits a scope, even if the scope didn't -// execute any code. -func (t *muxTracer) CaptureExit(output []byte, gasUsed uint64, err error, reverted bool) { +func (t *muxTracer) OnBalanceChange(a common.Address, prev, new *big.Int, reason tracing.BalanceChangeReason) { for _, t := range t.tracers { - t.CaptureExit(output, gasUsed, err, reverted) - } -} - -func (t *muxTracer) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common.Address) { - for _, t := range t.tracers { - t.CaptureTxStart(env, tx, from) - } -} - -func (t *muxTracer) CaptureTxEnd(receipt *types.Receipt, err error) { - for _, t := range t.tracers { - t.CaptureTxEnd(receipt, err) - } -} - -func (t *muxTracer) OnBalanceChange(a common.Address, prev, new *big.Int, reason state.BalanceChangeReason) { - for _, t := range t.tracers { - t.OnBalanceChange(a, prev, new, reason) + if t.OnBalanceChange != nil { + t.OnBalanceChange(a, prev, new, reason) + } } } func (t *muxTracer) OnNonceChange(a common.Address, prev, new uint64) { for _, t := range t.tracers { - t.OnNonceChange(a, prev, new) + if t.OnNonceChange != nil { + t.OnNonceChange(a, prev, new) + } } } func (t *muxTracer) OnCodeChange(a common.Address, prevCodeHash common.Hash, prev []byte, codeHash common.Hash, code []byte) { for _, t := range t.tracers { - t.OnCodeChange(a, prevCodeHash, prev, codeHash, code) + if t.OnCodeChange != nil { + t.OnCodeChange(a, prevCodeHash, prev, codeHash, code) + } } } func (t *muxTracer) OnStorageChange(a common.Address, k, prev, new common.Hash) { for _, t := range t.tracers { - t.OnStorageChange(a, k, prev, new) + if t.OnStorageChange != nil { + t.OnStorageChange(a, k, prev, new) + } } } func (t *muxTracer) OnLog(log *types.Log) { for _, t := range t.tracers { - t.OnLog(log) - } -} - -func (t *muxTracer) OnNewAccount(a common.Address, reset bool) { - for _, t := range t.tracers { - t.OnNewAccount(a, reset) + if t.OnLog != nil { + t.OnLog(log) + } } } diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index f49b3e61fc..556544041c 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -57,8 +58,7 @@ type accountMarshaling struct { } type prestateTracer struct { - directory.NoopTracer - env *vm.EVM + env *tracing.VMContext pre stateMap post stateMap to common.Address @@ -73,24 +73,33 @@ type prestateTracerConfig struct { DiffMode bool `json:"diffMode"` // If true, this tracer will return state modifications } -func newPrestateTracer(ctx *directory.Context, cfg json.RawMessage) (directory.Tracer, error) { +func newPrestateTracer(ctx *directory.Context, cfg json.RawMessage) (*directory.Tracer, error) { var config prestateTracerConfig if cfg != nil { if err := json.Unmarshal(cfg, &config); err != nil { return nil, err } } - return &prestateTracer{ + t := &prestateTracer{ pre: stateMap{}, post: stateMap{}, config: config, created: make(map[common.Address]bool), deleted: make(map[common.Address]bool), + } + return &directory.Tracer{ + Hooks: &tracing.Hooks{ + OnTxStart: t.OnTxStart, + OnTxEnd: t.OnTxEnd, + OnOpcode: t.OnOpcode, + }, + GetResult: t.GetResult, + Stop: t.Stop, }, nil } -// CaptureState implements the EVMLogger interface to trace a single step of VM execution. -func (t *prestateTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { +// OnOpcode implements the EVMLogger interface to trace a single step of VM execution. +func (t *prestateTracer) OnOpcode(pc uint64, opcode byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { if err != nil { return } @@ -98,10 +107,10 @@ func (t *prestateTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, if t.interrupt.Load() { return } - stack := scope.Stack - stackData := stack.Data() + op := vm.OpCode(opcode) + stackData := scope.StackData() stackLen := len(stackData) - caller := scope.Contract.Address() + caller := scope.Address() switch { case stackLen >= 1 && (op == vm.SLOAD || op == vm.SSTORE): slot := common.Hash(stackData[stackLen-1].Bytes32()) @@ -123,7 +132,7 @@ func (t *prestateTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, case stackLen >= 4 && op == vm.CREATE2: offset := stackData[stackLen-2] size := stackData[stackLen-3] - init, err := directory.GetMemoryCopyPadded(scope.Memory, int64(offset.Uint64()), int64(size.Uint64())) + init, err := directory.GetMemoryCopyPadded(scope.MemoryData(), int64(offset.Uint64()), int64(size.Uint64())) if err != nil { log.Warn("failed to copy CREATE2 input", "err", err, "tracer", "prestateTracer", "offset", offset, "size", size) return @@ -136,7 +145,7 @@ func (t *prestateTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, } } -func (t *prestateTracer) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common.Address) { +func (t *prestateTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { t.env = env if tx.To() == nil { t.to = crypto.CreateAddress(from, env.StateDB.GetNonce(from)) @@ -147,10 +156,10 @@ func (t *prestateTracer) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from t.lookupAccount(from) t.lookupAccount(t.to) - t.lookupAccount(env.Context.Coinbase) + t.lookupAccount(env.Coinbase) } -func (t *prestateTracer) CaptureTxEnd(receipt *types.Receipt, err error) { +func (t *prestateTracer) OnTxEnd(receipt *types.Receipt, err error) { if err != nil { return } diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 9863d0a4c6..3cce7bffa1 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -61,7 +61,7 @@ func BenchmarkTransactionTrace(b *testing.B) { GasLimit: gas, BaseFee: big.NewInt(8), } - alloc := core.GenesisAlloc{} + alloc := types.GenesisAlloc{} // The code pushes 'deadbeef' into memory, then the other params, and calls CREATE2, then returns // the address loop := []byte{ @@ -69,18 +69,18 @@ func BenchmarkTransactionTrace(b *testing.B) { byte(vm.PUSH1), 0, // jumpdestination byte(vm.JUMP), } - alloc[common.HexToAddress("0x00000000000000000000000000000000deadbeef")] = core.GenesisAccount{ + alloc[common.HexToAddress("0x00000000000000000000000000000000deadbeef")] = types.Account{ Nonce: 1, Code: loop, Balance: big.NewInt(1), } - alloc[from] = core.GenesisAccount{ + alloc[from] = types.Account{ Nonce: 1, Code: []byte{}, Balance: big.NewInt(500000000000000), } - triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false, rawdb.HashScheme) - defer triedb.Close() + state := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false, rawdb.HashScheme) + defer state.Close() // Create the tracer, the EVM environment and run it tracer := logger.NewStructLogger(&logger.Config{ @@ -89,7 +89,7 @@ func BenchmarkTransactionTrace(b *testing.B) { //EnableMemory: false, //EnableReturnData: false, }) - evm := vm.NewEVM(context, txContext, statedb, params.AllEthashProtocolChanges, vm.Config{Tracer: tracer}) + evm := vm.NewEVM(context, txContext, state.StateDB, params.AllEthashProtocolChanges, vm.Config{Tracer: tracer.Hooks()}) msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { b.Fatalf("failed to prepare transaction for tracing: %v", err) @@ -98,13 +98,13 @@ func BenchmarkTransactionTrace(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - snap := statedb.Snapshot() + snap := state.StateDB.Snapshot() st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(tx.Gas())) _, err = st.TransitionDb() if err != nil { b.Fatal(err) } - statedb.RevertToSnapshot(snap) + state.StateDB.RevertToSnapshot(snap) if have, want := len(tracer.StructLogs()), 244752; have != want { b.Fatalf("trace wrong, want %d steps, have %d", want, have) } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 4c63b776ef..5c3cb79dd6 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -665,6 +665,12 @@ func toCallArg(msg ethereum.CallMsg) interface{} { if msg.AccessList != nil { arg["accessList"] = msg.AccessList } + if msg.BlobGasFeeCap != nil { + arg["maxFeePerBlobGas"] = (*hexutil.Big)(msg.BlobGasFeeCap) + } + if msg.BlobHashes != nil { + arg["blobVersionedHashes"] = msg.BlobHashes + } return arg } diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index fd053c1d73..2f3229cedc 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -187,7 +187,7 @@ var ( var genesis = &core.Genesis{ Config: params.AllEthashProtocolChanges, - Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, + Alloc: types.GenesisAlloc{testAddr: {Balance: testBalance}}, ExtraData: []byte("test genesis"), Timestamp: 9000, BaseFee: big.NewInt(params.InitialBaseFee), @@ -602,17 +602,22 @@ func testAtFunctions(t *testing.T, client *rpc.Client) { } // send a transaction for some interesting pending status + // and wait for the transaction to be included in the pending block sendTransaction(ec) - time.Sleep(100 * time.Millisecond) - // Check pending transaction count - pending, err := ec.PendingTransactionCount(context.Background()) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if pending != 1 { - t.Fatalf("unexpected pending, wanted 1 got: %v", pending) + // wait for the transaction to be included in the pending block + for { + // Check pending transaction count + pending, err := ec.PendingTransactionCount(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if pending == 1 { + break + } + time.Sleep(100 * time.Millisecond) } + // Query balance balance, err := ec.BalanceAt(context.Background(), testAddr, nil) if err != nil { @@ -737,7 +742,7 @@ func sendTransaction(ec *Client) error { if err != nil { return err } - nonce, err := ec.PendingNonceAt(context.Background(), testAddr) + nonce, err := ec.NonceAt(context.Background(), testAddr, nil) if err != nil { return err } diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 73d05d499e..b1678b6766 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -245,6 +245,12 @@ func toCallArg(msg ethereum.CallMsg) interface{} { if msg.AccessList != nil { arg["accessList"] = msg.AccessList } + if msg.BlobGasFeeCap != nil { + arg["maxFeePerBlobGas"] = (*hexutil.Big)(msg.BlobGasFeeCap) + } + if msg.BlobHashes != nil { + arg["blobVersionedHashes"] = msg.BlobHashes + } return arg } diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index dbe2310a62..d562bcda1f 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -81,7 +81,7 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { func generateTestChain() (*core.Genesis, []*types.Block) { genesis := &core.Genesis{ Config: params.AllEthashProtocolChanges, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ testAddr: {Balance: testBalance, Storage: map[common.Hash]common.Hash{testSlot: testValue}}, testContract: {Nonce: 1, Code: []byte{0x13, 0x37}}, testEmpty: {Balance: big.NewInt(1)}, @@ -146,7 +146,7 @@ func TestGethClient(t *testing.T) { func(t *testing.T) { testCallContractWithBlockOverrides(t, client) }, }, // The testaccesslist is a bit time-sensitive: the newTestBackend imports - // one block. The `testAcessList` fails if the miner has not yet created a + // one block. The `testAccessList` fails if the miner has not yet created a // new pending-block after the import event. // Hence: this test should be last, execute the tests serially. { diff --git a/ethclient/simulated/backend.go b/ethclient/simulated/backend.go index 6169dde61b..0c2a0b453c 100644 --- a/ethclient/simulated/backend.go +++ b/ethclient/simulated/backend.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/downloader" @@ -70,7 +71,7 @@ type Backend struct { // contract bindings in unit tests. // // A simulated backend always uses chainID 1337. -func NewBackend(alloc core.GenesisAlloc, options ...func(nodeConf *node.Config, ethConf *ethconfig.Config)) *Backend { +func NewBackend(alloc types.GenesisAlloc, options ...func(nodeConf *node.Config, ethConf *ethconfig.Config)) *Backend { // Create the default configurations for the outer node shell and the Ethereum // service to mutate with the options afterwards nodeConf := node.DefaultConfig diff --git a/ethclient/simulated/backend_test.go b/ethclient/simulated/backend_test.go index a9a8accfea..a8fd7913c3 100644 --- a/ethclient/simulated/backend_test.go +++ b/ethclient/simulated/backend_test.go @@ -26,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" @@ -41,7 +40,7 @@ var ( func simTestBackend(testAddr common.Address) *Backend { return NewBackend( - core.GenesisAlloc{ + types.GenesisAlloc{ testAddr: {Balance: big.NewInt(10000000000000000)}, }, ) @@ -52,7 +51,7 @@ func newTx(sim *Backend, key *ecdsa.PrivateKey) (*types.Transaction, error) { // create a signed transaction to send head, _ := client.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(params.GWei)) addr := crypto.PubkeyToAddress(key.PublicKey) chainid, _ := client.ChainID(context.Background()) nonce, err := client.PendingNonceAt(context.Background(), addr) @@ -62,7 +61,7 @@ func newTx(sim *Backend, key *ecdsa.PrivateKey) (*types.Transaction, error) { tx := types.NewTx(&types.DynamicFeeTx{ ChainID: chainid, Nonce: nonce, - GasTipCap: big.NewInt(1), + GasTipCap: big.NewInt(params.GWei), GasFeeCap: gasPrice, Gas: 21000, To: &addr, @@ -71,7 +70,7 @@ func newTx(sim *Backend, key *ecdsa.PrivateKey) (*types.Transaction, error) { } func TestNewBackend(t *testing.T) { - sim := NewBackend(core.GenesisAlloc{}) + sim := NewBackend(types.GenesisAlloc{}) defer sim.Close() client := sim.Client() @@ -94,7 +93,7 @@ func TestNewBackend(t *testing.T) { } func TestAdjustTime(t *testing.T) { - sim := NewBackend(core.GenesisAlloc{}) + sim := NewBackend(types.GenesisAlloc{}) defer sim.Close() client := sim.Client() diff --git a/ethclient/simulated/options.go b/ethclient/simulated/options.go index 1b2f4c090d..6db995c917 100644 --- a/ethclient/simulated/options.go +++ b/ethclient/simulated/options.go @@ -17,6 +17,8 @@ package simulated import ( + "math/big" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/node" ) @@ -37,3 +39,17 @@ func WithCallGasLimit(gaslimit uint64) func(nodeConf *node.Config, ethConf *ethc ethConf.RPCGasCap = gaslimit } } + +// WithMinerMinTip configures the simulated backend to require a specific minimum +// gas tip for a transaction to be included. +// +// 0 is not possible as a live Geth node would reject that due to DoS protection, +// so the simulated backend will replicate that behavior for consistency. +func WithMinerMinTip(tip *big.Int) func(nodeConf *node.Config, ethConf *ethconfig.Config) { + if tip == nil || tip.Cmp(new(big.Int)) <= 0 { + panic("invalid miner minimum tip") + } + return func(nodeConf *node.Config, ethConf *ethconfig.Config) { + ethConf.Miner.GasPrice = tip + } +} diff --git a/ethclient/simulated/options_test.go b/ethclient/simulated/options_test.go index d9ff3b428a..9ff2be5ff9 100644 --- a/ethclient/simulated/options_test.go +++ b/ethclient/simulated/options_test.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" ) @@ -31,7 +32,7 @@ import ( // and that it keeps the same target value. func TestWithBlockGasLimitOption(t *testing.T) { // Construct a simulator, targeting a different gas limit - sim := NewBackend(core.GenesisAlloc{}, WithBlockGasLimit(12_345_678)) + sim := NewBackend(types.GenesisAlloc{}, WithBlockGasLimit(12_345_678)) defer sim.Close() client := sim.Client() @@ -56,7 +57,7 @@ func TestWithBlockGasLimitOption(t *testing.T) { // Tests that the simulator honors the RPC call caps set by the options. func TestWithCallGasLimitOption(t *testing.T) { // Construct a simulator, targeting a different gas limit - sim := NewBackend(core.GenesisAlloc{ + sim := NewBackend(types.GenesisAlloc{ testAddr: {Balance: big.NewInt(10000000000000000)}, }, WithCallGasLimit(params.TxGas-1)) defer sim.Close() diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index af4686cf5b..57689ab04b 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -589,8 +589,8 @@ func (b *batch) Reset() { func (b *batch) Replay(w ethdb.KeyValueWriter) error { reader := b.b.Reader() for { - kind, k, v, ok := reader.Next() - if !ok { + kind, k, v, ok, err := reader.Next() + if !ok || err != nil { break } // The (k,v) slices might be overwritten if the batch is reset/reused, diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 29559991be..6e71666ec1 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -39,7 +39,6 @@ import ( ethproto "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" @@ -80,13 +79,6 @@ type fullNodeBackend interface { SuggestGasTipCap(ctx context.Context) (*big.Int, error) } -// miningNodeBackend encompasses the functionality necessary for a mining node -// reporting to ethstats -type miningNodeBackend interface { - fullNodeBackend - Miner() *miner.Miner -} - // Service implements an Ethereum netstats reporting daemon that pushes local // chain statistics up to a monitoring server. type Service struct { @@ -611,6 +603,10 @@ func (s *Service) reportBlock(conn *connWrapper, block *types.Block) error { // Gather the block details from the header or block chain details := s.assembleBlockStats(block) + // Short circuit if the block detail is not available. + if details == nil { + return nil + } // Assemble the block report and send it to the server log.Trace("Sending new block to ethstats", "number", details.Number, "hash", details.Hash) @@ -638,10 +634,16 @@ func (s *Service) assembleBlockStats(block *types.Block) *blockStats { // check if backend is a full node fullBackend, ok := s.backend.(fullNodeBackend) if ok { + // Retrieve current chain head if no block is given. if block == nil { head := fullBackend.CurrentBlock() block, _ = fullBackend.BlockByNumber(context.Background(), rpc.BlockNumber(head.Number.Uint64())) } + // Short circuit if no block is available. It might happen when + // the blockchain is reorging. + if block == nil { + return nil + } header = block.Header() td = fullBackend.GetTd(context.Background(), header.Hash()) @@ -767,30 +769,21 @@ func (s *Service) reportPending(conn *connWrapper) error { type nodeStats struct { Active bool `json:"active"` Syncing bool `json:"syncing"` - Mining bool `json:"mining"` - Hashrate int `json:"hashrate"` Peers int `json:"peers"` GasPrice int `json:"gasPrice"` Uptime int `json:"uptime"` } -// reportStats retrieves various stats about the node at the networking and -// mining layer and reports it to the stats server. +// reportStats retrieves various stats about the node at the networking layer +// and reports it to the stats server. func (s *Service) reportStats(conn *connWrapper) error { - // Gather the syncing and mining infos from the local miner instance + // Gather the syncing infos from the local miner instance var ( - mining bool - hashrate int syncing bool gasprice int ) // check if backend is a full node if fullBackend, ok := s.backend.(fullNodeBackend); ok { - if miningBackend, ok := s.backend.(miningNodeBackend); ok { - mining = miningBackend.Miner().Mining() - hashrate = int(miningBackend.Miner().Hashrate()) - } - sync := fullBackend.SyncProgress() syncing = !sync.Done() @@ -810,8 +803,6 @@ func (s *Service) reportStats(conn *connWrapper) error { "id": s.node, "stats": &nodeStats{ Active: true, - Mining: mining, - Hashrate: hashrate, Peers: s.server.PeerCount(), GasPrice: gasprice, Syncing: syncing, diff --git a/go.mod b/go.mod index 2c900b2600..0acbcd4a35 100644 --- a/go.mod +++ b/go.mod @@ -13,17 +13,19 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.2.0 github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.79.0 - github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 + github.com/cockroachdb/pebble v1.1.0 github.com/consensys/gnark-crypto v0.12.1 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 - github.com/crate-crypto/go-kzg-4844 v0.7.0 + github.com/crate-crypto/go-kzg-4844 v1.0.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.1.0 - github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 - github.com/ethereum/c-kzg-4844 v0.4.0 + github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 + github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 + github.com/ethereum/c-kzg-4844 v1.0.0 github.com/fatih/color v1.13.0 + github.com/ferranbt/fastssz v0.1.2 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e - github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 + github.com/fjl/memsize v0.0.2 github.com/fsnotify/fsnotify v1.6.0 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 @@ -36,7 +38,7 @@ require ( github.com/gorilla/websocket v1.4.2 github.com/graph-gophers/graphql-go v1.3.0 github.com/hashicorp/go-bexpr v0.1.10 - github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 + github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.2.4 github.com/huin/goupnp v1.3.0 @@ -45,7 +47,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 github.com/julienschmidt/httprouter v1.3.0 - github.com/karalabe/usb v0.0.2 + github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.17 @@ -53,6 +55,8 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7 + github.com/protolambda/zrnt v0.30.0 + github.com/protolambda/ztyp v0.2.2 github.com/rs/cors v1.7.0 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.2.0 @@ -69,7 +73,7 @@ require ( golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.15.0 - google.golang.org/protobuf v1.28.0 + google.golang.org/protobuf v1.28.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -91,10 +95,9 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cockroachdb/errors v1.8.1 // indirect - github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect - github.com/cockroachdb/redact v1.0.8 // indirect - github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect + github.com/cockroachdb/errors v1.11.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -102,11 +105,11 @@ require ( github.com/deepmap/oapi-codegen v1.6.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -115,10 +118,12 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect github.com/klauspost/compress v1.15.15 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect diff --git a/go.sum b/go.sum index 1ba08e3b37..5b01a41ba8 100644 --- a/go.sum +++ b/go.sum @@ -31,7 +31,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= @@ -44,20 +43,14 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkM github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= -github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -65,7 +58,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= @@ -92,7 +84,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwF github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -119,36 +110,27 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.79.0 h1:ErwCYDjFCYppDJlDJ/5WhsSmzegAUe2+K9qgFyQDg3M= github.com/cloudflare/cloudflare-go v0.79.0/go.mod h1:gkHQf9xEubaQPEuerBuoinR9P8bf8a05Lq0X6WKy1Oc= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= -github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= -github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= -github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= -github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= -github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= -github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4= +github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= -github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= -github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -162,55 +144,48 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/deepmap/oapi-codegen v1.6.0 h1:w/d1ntwh91XI0b/8ja7+u5SvA4IFfM0UNNLmiDR1gg0= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao= +github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 h1:qwcF+vdFrvPSEUDSX5RVoRccG8a5DhOdWdQ4zN62zzo= -github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= +github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 h1:+3HCtB74++ClLy8GgjUQYeC8R4ILzVcIe8+5edAJJnE= +github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= -github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= +github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk= +github.com/ferranbt/fastssz v0.1.2/go.mod h1:X5UPrE2u1UJjxHA8X54u04SBwdAQjG2sFtWs39YxyWs= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= -github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= +github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= +github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 h1:IZqZOB2fydHte3kUgxrzK5E1fW7RQGeDwE8F/ZZnUYc= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= -github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2GihuqhwdILrV+7GJel5lyPV3u1+PgzrWLc0TkE= github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46/go.mod h1:QNpY22eby74jVhqH4WhDLDwxc/vqsern6pW+u2kbkpc= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -220,8 +195,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= -github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -230,20 +203,13 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -275,11 +241,11 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= -github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -294,8 +260,6 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -318,9 +282,7 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= @@ -333,34 +295,26 @@ github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 h1:3JQNjnMRil1yD0IfZKHF9GxxWKDJGj8I0IqOUol//sw= -github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= +github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6wn4Ej8vjuVGxeHdan+bRb2ebyv4= +github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb-client-go/v2 v2.4.0 h1:HGBfZYStlx3Kqvsv1h2pJixbCl/jhnFtxpKFAv9Tu5k= github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c h1:qSHzRbhzK8RdXOsAdfDgO49TtqC1oZ+acxPrkfTxcCs= github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= -github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= -github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= -github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY= @@ -376,30 +330,20 @@ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= -github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= -github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= -github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= -github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= +github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 h1:msKODTL1m0wigztaqILOtla9HeW1ciscYG4xjLtvk5I= +github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8= github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -414,11 +358,9 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= @@ -428,7 +370,6 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -440,15 +381,12 @@ github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= -github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= -github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= @@ -461,23 +399,18 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= -github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= -github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -485,11 +418,9 @@ github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -522,8 +453,15 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e/go.mod h1:MPZvj2Pr0N8/dXyTPS5REeg2sdLG7t8DRzC1rLv925w= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7 h1:cZC+usqsYgHtlBaGulVnZ1hfKAi8iWtujBnRLQE698c= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7/go.mod h1:IToEjHuttnUzwZI5KBSM/LOOW3qLbbrHOEfp3SbECGY= +github.com/protolambda/messagediff v1.4.0/go.mod h1:LboJp0EwIbJsePYpzh5Op/9G1/4mIztMRYzzwR0dR2M= +github.com/protolambda/zrnt v0.30.0 h1:pHEn69ZgaDFGpLGGYG1oD7DvYI7RDirbMBPfbC+8p4g= +github.com/protolambda/zrnt v0.30.0/go.mod h1:qcdX9CXFeVNCQK/q0nswpzhd+31RHMk2Ax/2lMsJ4Jw= +github.com/protolambda/ztyp v0.2.2 h1:rVcL3vBu9W/aV646zF6caLS/dyn9BN8NYiuJzicLNyY= +github.com/protolambda/ztyp v0.2.2/go.mod h1:9bYgKGqg3wJqT9ac1gI2hnVb0STQq7p/1lapqrqY1dU= +github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220714111606-acbb2962fb48 h1:cSo6/vk8YpvkLbk9v3FO97cakNmUoxwi2KMP8hd5WIw= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -532,26 +470,13 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -573,26 +498,13 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= -github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -606,11 +518,9 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME= go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -657,11 +567,9 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -670,7 +578,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -718,7 +625,6 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -728,7 +634,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -777,8 +682,6 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -803,15 +706,11 @@ golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -879,7 +778,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -909,7 +807,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -934,8 +831,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -944,9 +841,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -960,6 +854,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index f91229d015..f3f9d1778a 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -189,7 +190,7 @@ func TestGraphQLBlockSerializationEIP2718(t *testing.T) { Config: params.AllEthashProtocolChanges, GasLimit: 11500000, Difficulty: big.NewInt(1048576), - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xdad sloads 0x00 and 0x01 dad: { @@ -286,7 +287,7 @@ func TestGraphQLConcurrentResolvers(t *testing.T) { Config: params.AllEthashProtocolChanges, GasLimit: 11500000, Difficulty: big.NewInt(1048576), - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr: {Balance: big.NewInt(params.Ether)}, dad: { // LOG0(0, 0), LOG0(0, 0), RETURN(0, 0) @@ -379,7 +380,7 @@ func TestWithdrawals(t *testing.T) { Config: params.AllEthashProtocolChanges, GasLimit: 11500000, Difficulty: common.Big1, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr: {Balance: big.NewInt(params.Ether)}, }, } @@ -452,6 +453,7 @@ func newGQLService(t *testing.T, stack *node.Node, shanghai bool, gspec *core.Ge TrieDirtyCache: 5, TrieTimeout: 60 * time.Minute, SnapshotCache: 5, + StateScheme: rawdb.HashScheme, } var engine consensus.Engine = ethash.NewFaker() if shanghai { diff --git a/interfaces.go b/interfaces.go index c6aee295ee..53e2e3ae16 100644 --- a/interfaces.go +++ b/interfaces.go @@ -152,6 +152,10 @@ type CallMsg struct { Data []byte // input data, usually an ABI-encoded contract method invocation AccessList types.AccessList // EIP-2930 access list. + + // For BlobTxType + BlobGasFeeCap *big.Int + BlobHashes []common.Hash } // A ContractCaller provides contract calls, essentially transactions that are executed by diff --git a/internal/build/download.go b/internal/build/download.go index 903d0308df..206c51dce1 100644 --- a/internal/build/download.go +++ b/internal/build/download.go @@ -40,7 +40,7 @@ func MustLoadChecksums(file string) *ChecksumDB { if err != nil { log.Fatal("can't load checksum file: " + err.Error()) } - return &ChecksumDB{strings.Split(string(content), "\n")} + return &ChecksumDB{strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n")} } // Verify checks whether the given file is valid according to the checksum database. @@ -84,10 +84,12 @@ func (db *ChecksumDB) DownloadFile(url, dstPath string) error { resp, err := http.Get(url) if err != nil { return fmt.Errorf("download error: %v", err) - } else if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download error: status %d", resp.StatusCode) } defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download error: status %d", resp.StatusCode) + } if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { return err } diff --git a/internal/era/accumulator.go b/internal/era/accumulator.go new file mode 100644 index 0000000000..2ece2755e1 --- /dev/null +++ b/internal/era/accumulator.go @@ -0,0 +1,91 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "errors" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + ssz "github.com/ferranbt/fastssz" +) + +// ComputeAccumulator calculates the SSZ hash tree root of the Era1 +// accumulator of header records. +func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, error) { + if len(hashes) != len(tds) { + return common.Hash{}, errors.New("must have equal number hashes as td values") + } + if len(hashes) > MaxEra1Size { + return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEra1Size) + } + hh := ssz.NewHasher() + for i := range hashes { + rec := headerRecord{hashes[i], tds[i]} + root, err := rec.HashTreeRoot() + if err != nil { + return common.Hash{}, err + } + hh.Append(root[:]) + } + hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxEra1Size)) + return hh.HashRoot() +} + +// headerRecord is an individual record for a historical header. +// +// See https://github.com/ethereum/portal-network-specs/blob/master/history-network.md#the-header-accumulator +// for more information. +type headerRecord struct { + Hash common.Hash + TotalDifficulty *big.Int +} + +// GetTree completes the ssz.HashRoot interface, but is unused. +func (h *headerRecord) GetTree() (*ssz.Node, error) { + return nil, nil +} + +// HashTreeRoot ssz hashes the headerRecord object. +func (h *headerRecord) HashTreeRoot() ([32]byte, error) { + return ssz.HashWithDefaultHasher(h) +} + +// HashTreeRootWith ssz hashes the headerRecord object with a hasher. +func (h *headerRecord) HashTreeRootWith(hh ssz.HashWalker) (err error) { + hh.PutBytes(h.Hash[:]) + td := bigToBytes32(h.TotalDifficulty) + hh.PutBytes(td[:]) + hh.Merkleize(0) + return +} + +// bigToBytes32 converts a big.Int into a little-endian 32-byte array. +func bigToBytes32(n *big.Int) (b [32]byte) { + n.FillBytes(b[:]) + reverseOrder(b[:]) + return +} + +// reverseOrder reverses the byte order of a slice. +func reverseOrder(b []byte) []byte { + for i := 0; i < 16; i++ { + b[i], b[32-i-1] = b[32-i-1], b[i] + } + return b +} diff --git a/internal/era/builder.go b/internal/era/builder.go new file mode 100644 index 0000000000..75782a08c2 --- /dev/null +++ b/internal/era/builder.go @@ -0,0 +1,225 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . +package era + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +// Builder is used to create Era1 archives of block data. +// +// Era1 files are themselves e2store files. For more information on this format, +// see https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md. +// +// The overall structure of an Era1 file follows closely the structure of an Era file +// which contains consensus Layer data (and as a byproduct, EL data after the merge). +// +// The structure can be summarized through this definition: +// +// era1 := Version | block-tuple* | other-entries* | Accumulator | BlockIndex +// block-tuple := CompressedHeader | CompressedBody | CompressedReceipts | TotalDifficulty +// +// Each basic element is its own entry: +// +// Version = { type: [0x65, 0x32], data: nil } +// CompressedHeader = { type: [0x03, 0x00], data: snappyFramed(rlp(header)) } +// CompressedBody = { type: [0x04, 0x00], data: snappyFramed(rlp(body)) } +// CompressedReceipts = { type: [0x05, 0x00], data: snappyFramed(rlp(receipts)) } +// TotalDifficulty = { type: [0x06, 0x00], data: uint256(header.total_difficulty) } +// AccumulatorRoot = { type: [0x07, 0x00], data: accumulator-root } +// BlockIndex = { type: [0x32, 0x66], data: block-index } +// +// Accumulator is computed by constructing an SSZ list of header-records of length at most +// 8192 and then calculating the hash_tree_root of that list. +// +// header-record := { block-hash: Bytes32, total-difficulty: Uint256 } +// accumulator := hash_tree_root([]header-record, 8192) +// +// BlockIndex stores relative offsets to each compressed block entry. The +// format is: +// +// block-index := starting-number | index | index | index ... | count +// +// starting-number is the first block number in the archive. Every index is a +// defined relative to beginning of the record. The total number of block +// entries in the file is recorded with count. +// +// Due to the accumulator size limit of 8192, the maximum number of blocks in +// an Era1 batch is also 8192. +type Builder struct { + w *e2store.Writer + startNum *uint64 + startTd *big.Int + indexes []uint64 + hashes []common.Hash + tds []*big.Int + written int + + buf *bytes.Buffer + snappy *snappy.Writer +} + +// NewBuilder returns a new Builder instance. +func NewBuilder(w io.Writer) *Builder { + buf := bytes.NewBuffer(nil) + return &Builder{ + w: e2store.NewWriter(w), + buf: buf, + snappy: snappy.NewBufferedWriter(buf), + } +} + +// Add writes a compressed block entry and compressed receipts entry to the +// underlying e2store file. +func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) error { + eh, err := rlp.EncodeToBytes(block.Header()) + if err != nil { + return err + } + eb, err := rlp.EncodeToBytes(block.Body()) + if err != nil { + return err + } + er, err := rlp.EncodeToBytes(receipts) + if err != nil { + return err + } + return b.AddRLP(eh, eb, er, block.NumberU64(), block.Hash(), td, block.Difficulty()) +} + +// AddRLP writes a compressed block entry and compressed receipts entry to the +// underlying e2store file. +func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error { + // Write Era1 version entry before first block. + if b.startNum == nil { + n, err := b.w.Write(TypeVersion, nil) + if err != nil { + return err + } + startNum := number + b.startNum = &startNum + b.startTd = new(big.Int).Sub(td, difficulty) + b.written += n + } + if len(b.indexes) >= MaxEra1Size { + return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size) + } + + b.indexes = append(b.indexes, uint64(b.written)) + b.hashes = append(b.hashes, hash) + b.tds = append(b.tds, td) + + // Write block data. + if err := b.snappyWrite(TypeCompressedHeader, header); err != nil { + return err + } + if err := b.snappyWrite(TypeCompressedBody, body); err != nil { + return err + } + if err := b.snappyWrite(TypeCompressedReceipts, receipts); err != nil { + return err + } + + // Also write total difficulty, but don't snappy encode. + btd := bigToBytes32(td) + n, err := b.w.Write(TypeTotalDifficulty, btd[:]) + b.written += n + if err != nil { + return err + } + + return nil +} + +// Finalize computes the accumulator and block index values, then writes the +// corresponding e2store entries. +func (b *Builder) Finalize() (common.Hash, error) { + if b.startNum == nil { + return common.Hash{}, errors.New("finalize called on empty builder") + } + // Compute accumulator root and write entry. + root, err := ComputeAccumulator(b.hashes, b.tds) + if err != nil { + return common.Hash{}, fmt.Errorf("error calculating accumulator root: %w", err) + } + n, err := b.w.Write(TypeAccumulator, root[:]) + b.written += n + if err != nil { + return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err) + } + // Get beginning of index entry to calculate block relative offset. + base := int64(b.written) + + // Construct block index. Detailed format described in Builder + // documentation, but it is essentially encoded as: + // "start | index | index | ... | count" + var ( + count = len(b.indexes) + index = make([]byte, 16+count*8) + ) + binary.LittleEndian.PutUint64(index, *b.startNum) + // Each offset is relative from the position it is encoded in the + // index. This means that even if the same block was to be included in + // the index twice (this would be invalid anyways), the relative offset + // would be different. The idea with this is that after reading a + // relative offset, the corresponding block can be quickly read by + // performing a seek relative to the current position. + for i, offset := range b.indexes { + relative := int64(offset) - base + binary.LittleEndian.PutUint64(index[8+i*8:], uint64(relative)) + } + binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count)) + + // Finally, write the block index entry. + if _, err := b.w.Write(TypeBlockIndex, index); err != nil { + return common.Hash{}, fmt.Errorf("unable to write block index: %w", err) + } + + return root, nil +} + +// snappyWrite is a small helper to take care snappy encoding and writing an e2store entry. +func (b *Builder) snappyWrite(typ uint16, in []byte) error { + var ( + buf = b.buf + s = b.snappy + ) + buf.Reset() + s.Reset(buf) + if _, err := b.snappy.Write(in); err != nil { + return fmt.Errorf("error snappy encoding: %w", err) + } + if err := s.Flush(); err != nil { + return fmt.Errorf("error flushing snappy encoding: %w", err) + } + n, err := b.w.Write(typ, b.buf.Bytes()) + b.written += n + if err != nil { + return fmt.Errorf("error writing e2store entry: %w", err) + } + return nil +} diff --git a/internal/era/e2store/e2store.go b/internal/era/e2store/e2store.go new file mode 100644 index 0000000000..8e4d5dd24a --- /dev/null +++ b/internal/era/e2store/e2store.go @@ -0,0 +1,221 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package e2store + +import ( + "encoding/binary" + "errors" + "fmt" + "io" +) + +const ( + headerSize = 8 + valueSizeLimit = 1024 * 1024 * 50 +) + +// Entry is a variable-length-data record in an e2store. +type Entry struct { + Type uint16 + Value []byte +} + +// Writer writes entries using e2store encoding. +// For more information on this format, see: +// https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md +type Writer struct { + w io.Writer +} + +// NewWriter returns a new Writer that writes to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{w} +} + +// Write writes a single e2store entry to w. +// An entry is encoded in a type-length-value format. The first 8 bytes of the +// record store the type (2 bytes), the length (4 bytes), and some reserved +// data (2 bytes). The remaining bytes store b. +func (w *Writer) Write(typ uint16, b []byte) (int, error) { + buf := make([]byte, headerSize) + binary.LittleEndian.PutUint16(buf, typ) + binary.LittleEndian.PutUint32(buf[2:], uint32(len(b))) + + // Write header. + if n, err := w.w.Write(buf); err != nil { + return n, err + } + // Write value, return combined write size. + n, err := w.w.Write(b) + return n + headerSize, err +} + +// A Reader reads entries from an e2store-encoded file. +// For more information on this format, see +// https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md +type Reader struct { + r io.ReaderAt + offset int64 +} + +// NewReader returns a new Reader that reads from r. +func NewReader(r io.ReaderAt) *Reader { + return &Reader{r, 0} +} + +// Read reads one Entry from r. +func (r *Reader) Read() (*Entry, error) { + var e Entry + n, err := r.ReadAt(&e, r.offset) + if err != nil { + return nil, err + } + r.offset += int64(n) + return &e, nil +} + +// ReadAt reads one Entry from r at the specified offset. +func (r *Reader) ReadAt(entry *Entry, off int64) (int, error) { + typ, length, err := r.ReadMetadataAt(off) + if err != nil { + return 0, err + } + entry.Type = typ + + // Check length bounds. + if length > valueSizeLimit { + return headerSize, fmt.Errorf("item larger than item size limit %d: have %d", valueSizeLimit, length) + } + if length == 0 { + return headerSize, nil + } + + // Read value. + val := make([]byte, length) + if n, err := r.r.ReadAt(val, off+headerSize); err != nil { + n += headerSize + // An entry with a non-zero length should not return EOF when + // reading the value. + if err == io.EOF { + return n, io.ErrUnexpectedEOF + } + return n, err + } + entry.Value = val + return int(headerSize + length), nil +} + +// ReaderAt returns an io.Reader delivering value data for the entry at +// the specified offset. If the entry type does not match the expected type, an +// error is returned. +func (r *Reader) ReaderAt(expectedType uint16, off int64) (io.Reader, int, error) { + // problem = need to return length+headerSize not just value length via section reader + typ, length, err := r.ReadMetadataAt(off) + if err != nil { + return nil, headerSize, err + } + if typ != expectedType { + return nil, headerSize, fmt.Errorf("wrong type, want %d have %d", expectedType, typ) + } + if length > valueSizeLimit { + return nil, headerSize, fmt.Errorf("item larger than item size limit %d: have %d", valueSizeLimit, length) + } + return io.NewSectionReader(r.r, off+headerSize, int64(length)), headerSize + int(length), nil +} + +// LengthAt reads the header at off and returns the total length of the entry, +// including header. +func (r *Reader) LengthAt(off int64) (int64, error) { + _, length, err := r.ReadMetadataAt(off) + if err != nil { + return 0, err + } + return int64(length) + headerSize, nil +} + +// ReadMetadataAt reads the header metadata at the given offset. +func (r *Reader) ReadMetadataAt(off int64) (typ uint16, length uint32, err error) { + b := make([]byte, headerSize) + if n, err := r.r.ReadAt(b, off); err != nil { + if err == io.EOF && n > 0 { + return 0, 0, io.ErrUnexpectedEOF + } + return 0, 0, err + } + typ = binary.LittleEndian.Uint16(b) + length = binary.LittleEndian.Uint32(b[2:]) + + // Check reserved bytes of header. + if b[6] != 0 || b[7] != 0 { + return 0, 0, errors.New("reserved bytes are non-zero") + } + + return typ, length, nil +} + +// Find returns the first entry with the matching type. +func (r *Reader) Find(want uint16) (*Entry, error) { + var ( + off int64 + typ uint16 + length uint32 + err error + ) + for { + typ, length, err = r.ReadMetadataAt(off) + if err == io.EOF { + return nil, io.EOF + } else if err != nil { + return nil, err + } + if typ == want { + var e Entry + if _, err := r.ReadAt(&e, off); err != nil { + return nil, err + } + return &e, nil + } + off += int64(headerSize + length) + } +} + +// FindAll returns all entries with the matching type. +func (r *Reader) FindAll(want uint16) ([]*Entry, error) { + var ( + off int64 + typ uint16 + length uint32 + entries []*Entry + err error + ) + for { + typ, length, err = r.ReadMetadataAt(off) + if err == io.EOF { + return entries, nil + } else if err != nil { + return entries, err + } + if typ == want { + e := new(Entry) + if _, err := r.ReadAt(e, off); err != nil { + return entries, err + } + entries = append(entries, e) + } + off += int64(headerSize + length) + } +} diff --git a/internal/era/e2store/e2store_test.go b/internal/era/e2store/e2store_test.go new file mode 100644 index 0000000000..b0803493c7 --- /dev/null +++ b/internal/era/e2store/e2store_test.go @@ -0,0 +1,150 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package e2store + +import ( + "bytes" + "errors" + "io" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestEncode(t *testing.T) { + for _, test := range []struct { + entries []Entry + want string + name string + }{ + { + name: "emptyEntry", + entries: []Entry{{0xffff, nil}}, + want: "ffff000000000000", + }, + { + name: "beef", + entries: []Entry{{42, common.Hex2Bytes("beef")}}, + want: "2a00020000000000beef", + }, + { + name: "twoEntries", + entries: []Entry{ + {42, common.Hex2Bytes("beef")}, + {9, common.Hex2Bytes("abcdabcd")}, + }, + want: "2a00020000000000beef0900040000000000abcdabcd", + }, + } { + tt := test + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + var ( + b = bytes.NewBuffer(nil) + w = NewWriter(b) + ) + for _, e := range tt.entries { + if _, err := w.Write(e.Type, e.Value); err != nil { + t.Fatalf("encoding error: %v", err) + } + } + if want, have := common.FromHex(tt.want), b.Bytes(); !bytes.Equal(want, have) { + t.Fatalf("encoding mismatch (want %x, have %x", want, have) + } + r := NewReader(bytes.NewReader(b.Bytes())) + for _, want := range tt.entries { + have, err := r.Read() + if err != nil { + t.Fatalf("decoding error: %v", err) + } + if have.Type != want.Type { + t.Fatalf("decoded entry does type mismatch (want %v, got %v)", want.Type, have.Type) + } + if !bytes.Equal(have.Value, want.Value) { + t.Fatalf("decoded entry does not match (want %#x, got %#x)", want.Value, have.Value) + } + } + }) + } +} + +func TestDecode(t *testing.T) { + for i, tt := range []struct { + have string + err error + }{ + { // basic valid decoding + have: "ffff000000000000", + }, + { // basic invalid decoding + have: "ffff000000000001", + err: errors.New("reserved bytes are non-zero"), + }, + { // no more entries to read, returns EOF + have: "", + err: io.EOF, + }, + { // malformed type + have: "bad", + err: io.ErrUnexpectedEOF, + }, + { // malformed length + have: "badbeef", + err: io.ErrUnexpectedEOF, + }, + { // specified length longer than actual value + have: "beef010000000000", + err: io.ErrUnexpectedEOF, + }, + } { + r := NewReader(bytes.NewReader(common.FromHex(tt.have))) + if tt.err != nil { + _, err := r.Read() + if err == nil && tt.err != nil { + t.Fatalf("test %d, expected error, got none", i) + } + if err != nil && tt.err == nil { + t.Fatalf("test %d, expected no error, got %v", i, err) + } + if err != nil && tt.err != nil && err.Error() != tt.err.Error() { + t.Fatalf("expected error %v, got %v", tt.err, err) + } + continue + } + } +} + +func FuzzCodec(f *testing.F) { + f.Fuzz(func(t *testing.T, input []byte) { + r := NewReader(bytes.NewReader(input)) + entry, err := r.Read() + if err != nil { + return + } + var ( + b = bytes.NewBuffer(nil) + w = NewWriter(b) + ) + w.Write(entry.Type, entry.Value) + output := b.Bytes() + // Only care about the input that was actually consumed + input = input[:r.offset] + if !bytes.Equal(input, output) { + t.Fatalf("decode-encode mismatch, input %#x output %#x", input, output) + } + }) +} diff --git a/internal/era/era.go b/internal/era/era.go new file mode 100644 index 0000000000..2099c2d575 --- /dev/null +++ b/internal/era/era.go @@ -0,0 +1,284 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "math/big" + "os" + "path" + "strconv" + "strings" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +var ( + TypeVersion uint16 = 0x3265 + TypeCompressedHeader uint16 = 0x03 + TypeCompressedBody uint16 = 0x04 + TypeCompressedReceipts uint16 = 0x05 + TypeTotalDifficulty uint16 = 0x06 + TypeAccumulator uint16 = 0x07 + TypeBlockIndex uint16 = 0x3266 + + MaxEra1Size = 8192 +) + +// Filename returns a recognizable Era1-formatted file name for the specified +// epoch and network. +func Filename(network string, epoch int, root common.Hash) string { + return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10]) +} + +// ReadDir reads all the era1 files in a directory for a given network. +// Format: --.era1 +func ReadDir(dir, network string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("error reading directory %s: %w", dir, err) + } + var ( + next = uint64(0) + eras []string + ) + for _, entry := range entries { + if path.Ext(entry.Name()) != ".era1" { + continue + } + parts := strings.Split(entry.Name(), "-") + if len(parts) != 3 || parts[0] != network { + // invalid era1 filename, skip + continue + } + if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil { + return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name()) + } else if epoch != next { + return nil, fmt.Errorf("missing epoch %d", next) + } + next += 1 + eras = append(eras, entry.Name()) + } + return eras, nil +} + +type ReadAtSeekCloser interface { + io.ReaderAt + io.Seeker + io.Closer +} + +// Era reads and Era1 file. +type Era struct { + f ReadAtSeekCloser // backing era1 file + s *e2store.Reader // e2store reader over f + m metadata // start, count, length info + mu *sync.Mutex // lock for buf + buf [8]byte // buffer reading entry offsets +} + +// From returns an Era backed by f. +func From(f ReadAtSeekCloser) (*Era, error) { + m, err := readMetadata(f) + if err != nil { + return nil, err + } + return &Era{ + f: f, + s: e2store.NewReader(f), + m: m, + mu: new(sync.Mutex), + }, nil +} + +// Open returns an Era backed by the given filename. +func Open(filename string) (*Era, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + return From(f) +} + +func (e *Era) Close() error { + return e.f.Close() +} + +func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, errors.New("out-of-bounds") + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + r, n, err := newSnappyReader(e.s, TypeCompressedHeader, off) + if err != nil { + return nil, err + } + var header types.Header + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + r, _, err = newSnappyReader(e.s, TypeCompressedBody, off) + if err != nil { + return nil, err + } + var body types.Body + if err := rlp.Decode(r, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil +} + +// Accumulator reads the accumulator entry in the Era1 file. +func (e *Era) Accumulator() (common.Hash, error) { + entry, err := e.s.Find(TypeAccumulator) + if err != nil { + return common.Hash{}, err + } + return common.BytesToHash(entry.Value), nil +} + +// InitialTD returns initial total difficulty before the difficulty of the +// first block of the Era1 is applied. +func (e *Era) InitialTD() (*big.Int, error) { + var ( + r io.Reader + header types.Header + rawTd []byte + n int64 + off int64 + err error + ) + + // Read first header. + if off, err = e.readOffset(e.m.start); err != nil { + return nil, err + } + if r, n, err = newSnappyReader(e.s, TypeCompressedHeader, off); err != nil { + return nil, err + } + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + + // Skip over next two records. + for i := 0; i < 2; i++ { + length, err := e.s.LengthAt(off) + if err != nil { + return nil, err + } + off += length + } + + // Read total difficulty after first block. + if r, _, err = e.s.ReaderAt(TypeTotalDifficulty, off); err != nil { + return nil, err + } + rawTd, err = io.ReadAll(r) + if err != nil { + return nil, err + } + td := new(big.Int).SetBytes(reverseOrder(rawTd)) + return td.Sub(td, header.Difficulty), nil +} + +// Start returns the listed start block. +func (e *Era) Start() uint64 { + return e.m.start +} + +// Count returns the total number of blocks in the Era1. +func (e *Era) Count() uint64 { + return e.m.count +} + +// readOffset reads a specific block's offset from the block index. The value n +// is the absolute block number desired. +func (e *Era) readOffset(n uint64) (int64, error) { + var ( + blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header + firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num + indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes + offOffset = firstIndex + indexOffset // offset of block offset + ) + e.mu.Lock() + defer e.mu.Unlock() + clearBuffer(e.buf[:]) + if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil { + return 0, err + } + // Since the block offset is relative from the start of the block index record + // we need to add the record offset to it's offset to get the block's absolute + // offset. + return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil +} + +// newReader returns a snappy.Reader for the e2store entry value at off. +func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) { + r, n, err := e.ReaderAt(expectedType, off) + if err != nil { + return nil, 0, err + } + return snappy.NewReader(r), int64(n), err +} + +// clearBuffer zeroes out the buffer. +func clearBuffer(buf []byte) { + for i := 0; i < len(buf); i++ { + buf[i] = 0 + } +} + +// metadata wraps the metadata in the block index. +type metadata struct { + start uint64 + count uint64 + length int64 +} + +// readMetadata reads the metadata stored in an Era1 file's block index. +func readMetadata(f ReadAtSeekCloser) (m metadata, err error) { + // Determine length of reader. + if m.length, err = f.Seek(0, io.SeekEnd); err != nil { + return + } + b := make([]byte, 16) + // Read count. It's the last 8 bytes of the file. + if _, err = f.ReadAt(b[:8], m.length-8); err != nil { + return + } + m.count = binary.LittleEndian.Uint64(b) + // Read start. It's at the offset -sizeof(m.count) - + // count*sizeof(indexEntry) - sizeof(m.start) + if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil { + return + } + m.start = binary.LittleEndian.Uint64(b[8:]) + return +} diff --git a/internal/era/era_test.go b/internal/era/era_test.go new file mode 100644 index 0000000000..ee5d9e82a0 --- /dev/null +++ b/internal/era/era_test.go @@ -0,0 +1,142 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "bytes" + "io" + "math/big" + "os" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +type testchain struct { + headers [][]byte + bodies [][]byte + receipts [][]byte + tds []*big.Int +} + +func TestEra1Builder(t *testing.T) { + // Get temp directory. + f, err := os.CreateTemp("", "era1-test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer f.Close() + + var ( + builder = NewBuilder(f) + chain = testchain{} + ) + for i := 0; i < 128; i++ { + chain.headers = append(chain.headers, []byte{byte('h'), byte(i)}) + chain.bodies = append(chain.bodies, []byte{byte('b'), byte(i)}) + chain.receipts = append(chain.receipts, []byte{byte('r'), byte(i)}) + chain.tds = append(chain.tds, big.NewInt(int64(i))) + } + + // Write blocks to Era1. + for i := 0; i < len(chain.headers); i++ { + var ( + header = chain.headers[i] + body = chain.bodies[i] + receipts = chain.receipts[i] + hash = common.Hash{byte(i)} + td = chain.tds[i] + ) + if err = builder.AddRLP(header, body, receipts, uint64(i), hash, td, big.NewInt(1)); err != nil { + t.Fatalf("error adding entry: %v", err) + } + } + + // Finalize Era1. + if _, err := builder.Finalize(); err != nil { + t.Fatalf("error finalizing era1: %v", err) + } + + // Verify Era1 contents. + e, err := Open(f.Name()) + if err != nil { + t.Fatalf("failed to open era: %v", err) + } + it, err := NewRawIterator(e) + if err != nil { + t.Fatalf("failed to make iterator: %s", err) + } + for i := uint64(0); i < uint64(len(chain.headers)); i++ { + if !it.Next() { + t.Fatalf("expected more entries") + } + if it.Error() != nil { + t.Fatalf("unexpected error %v", it.Error()) + } + // Check headers. + header, err := io.ReadAll(it.Header) + if err != nil { + t.Fatalf("error reading header: %v", err) + } + if !bytes.Equal(header, chain.headers[i]) { + t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], header) + } + // Check bodies. + body, err := io.ReadAll(it.Body) + if err != nil { + t.Fatalf("error reading body: %v", err) + } + if !bytes.Equal(body, chain.bodies[i]) { + t.Fatalf("mismatched body: want %s, got %s", chain.bodies[i], body) + } + // Check receipts. + receipts, err := io.ReadAll(it.Receipts) + if err != nil { + t.Fatalf("error reading receipts: %v", err) + } + if !bytes.Equal(receipts, chain.receipts[i]) { + t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], receipts) + } + + // Check total difficulty. + rawTd, err := io.ReadAll(it.TotalDifficulty) + if err != nil { + t.Fatalf("error reading td: %v", err) + } + td := new(big.Int).SetBytes(reverseOrder(rawTd)) + if td.Cmp(chain.tds[i]) != 0 { + t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td) + } + } +} + +func TestEraFilename(t *testing.T) { + for i, tt := range []struct { + network string + epoch int + root common.Hash + expected string + }{ + {"mainnet", 1, common.Hash{1}, "mainnet-00001-01000000.era1"}, + {"goerli", 99999, common.HexToHash("0xdeadbeef00000000000000000000000000000000000000000000000000000000"), "goerli-99999-deadbeef.era1"}, + } { + got := Filename(tt.network, tt.epoch, tt.root) + if tt.expected != got { + t.Errorf("test %d: invalid filename: want %s, got %s", i, tt.expected, got) + } + } +} diff --git a/internal/era/iterator.go b/internal/era/iterator.go new file mode 100644 index 0000000000..d90e9586a4 --- /dev/null +++ b/internal/era/iterator.go @@ -0,0 +1,198 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "errors" + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// Iterator wraps RawIterator and returns decoded Era1 entries. +type Iterator struct { + inner *RawIterator +} + +// NewIterator returns a new Iterator instance. Next must be immediately +// called on new iterators to load the first item. +func NewIterator(e *Era) (*Iterator, error) { + inner, err := NewRawIterator(e) + if err != nil { + return nil, err + } + return &Iterator{inner}, nil +} + +// Next moves the iterator to the next block entry. It returns false when all +// items have been read or an error has halted its progress. Block, Receipts, +// and BlockAndReceipts should no longer be called after false is returned. +func (it *Iterator) Next() bool { + return it.inner.Next() +} + +// Number returns the current number block the iterator will return. +func (it *Iterator) Number() uint64 { + return it.inner.next - 1 +} + +// Error returns the error status of the iterator. It should be called before +// reading from any of the iterator's values. +func (it *Iterator) Error() error { + return it.inner.Error() +} + +// Block returns the block for the iterator's current position. +func (it *Iterator) Block() (*types.Block, error) { + if it.inner.Header == nil || it.inner.Body == nil { + return nil, errors.New("header and body must be non-nil") + } + var ( + header types.Header + body types.Body + ) + if err := rlp.Decode(it.inner.Header, &header); err != nil { + return nil, err + } + if err := rlp.Decode(it.inner.Body, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil +} + +// Receipts returns the receipts for the iterator's current position. +func (it *Iterator) Receipts() (types.Receipts, error) { + if it.inner.Receipts == nil { + return nil, fmt.Errorf("receipts must be non-nil") + } + var receipts types.Receipts + err := rlp.Decode(it.inner.Receipts, &receipts) + return receipts, err +} + +// BlockAndReceipts returns the block and receipts for the iterator's current +// position. +func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) { + b, err := it.Block() + if err != nil { + return nil, nil, err + } + r, err := it.Receipts() + if err != nil { + return nil, nil, err + } + return b, r, nil +} + +// TotalDifficulty returns the total difficulty for the iterator's current +// position. +func (it *Iterator) TotalDifficulty() (*big.Int, error) { + td, err := io.ReadAll(it.inner.TotalDifficulty) + if err != nil { + return nil, err + } + return new(big.Int).SetBytes(reverseOrder(td)), nil +} + +// RawIterator reads an RLP-encode Era1 entries. +type RawIterator struct { + e *Era // backing Era1 + next uint64 // next block to read + err error // last error + + Header io.Reader + Body io.Reader + Receipts io.Reader + TotalDifficulty io.Reader +} + +// NewRawIterator returns a new RawIterator instance. Next must be immediately +// called on new iterators to load the first item. +func NewRawIterator(e *Era) (*RawIterator, error) { + return &RawIterator{ + e: e, + next: e.m.start, + }, nil +} + +// Next moves the iterator to the next block entry. It returns false when all +// items have been read or an error has halted its progress. Header, Body, +// Receipts, TotalDifficulty will be set to nil in the case returning false or +// finding an error and should therefore no longer be read from. +func (it *RawIterator) Next() bool { + // Clear old errors. + it.err = nil + if it.e.m.start+it.e.m.count <= it.next { + it.clear() + return false + } + off, err := it.e.readOffset(it.next) + if err != nil { + // Error here means block index is corrupted, so don't + // continue. + it.clear() + it.err = err + return false + } + var n int64 + if it.Header, n, it.err = newSnappyReader(it.e.s, TypeCompressedHeader, off); it.err != nil { + it.clear() + return true + } + off += n + if it.Body, n, it.err = newSnappyReader(it.e.s, TypeCompressedBody, off); it.err != nil { + it.clear() + return true + } + off += n + if it.Receipts, n, it.err = newSnappyReader(it.e.s, TypeCompressedReceipts, off); it.err != nil { + it.clear() + return true + } + off += n + if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(TypeTotalDifficulty, off); it.err != nil { + it.clear() + return true + } + it.next += 1 + return true +} + +// Number returns the current number block the iterator will return. +func (it *RawIterator) Number() uint64 { + return it.next - 1 +} + +// Error returns the error status of the iterator. It should be called before +// reading from any of the iterator's values. +func (it *RawIterator) Error() error { + if it.err == io.EOF { + return nil + } + return it.err +} + +// clear sets all the outputs to nil. +func (it *RawIterator) clear() { + it.Header = nil + it.Body = nil + it.Receipts = nil + it.TotalDifficulty = nil +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 20e3446cfd..5acab3fd22 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip1559" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -288,7 +289,7 @@ type PersonalAccountAPI struct { b Backend } -// NewPersonalAccountAPI create a new PersonalAccountAPI. +// NewPersonalAccountAPI creates a new PersonalAccountAPI. func NewPersonalAccountAPI(b Backend, nonceLock *AddrLocker) *PersonalAccountAPI { return &PersonalAccountAPI{ am: b.AccountManager(), @@ -453,7 +454,7 @@ func (s *PersonalAccountAPI) signTransaction(ctx context.Context, args *Transact return nil, err } // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Assemble the transaction and sign with the wallet @@ -530,7 +531,7 @@ func (s *PersonalAccountAPI) SignTransaction(ctx context.Context, args Transacti // // The key used to calculate the signature is decrypted with the given password. // -// https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_sign +// https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-personal#personal-sign func (s *PersonalAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr common.Address, passwd string) (hexutil.Bytes, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: addr} @@ -558,7 +559,7 @@ func (s *PersonalAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr // Note, the signature must conform to the secp256k1 curve R, S and V values, where // the V value must be 27 or 28 for legacy reasons. // -// https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover +// https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-personal#personal-ecrecover func (s *PersonalAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Bytes) (common.Address, error) { if len(sig) != crypto.SignatureLength { return common.Address{}, fmt.Errorf("signature must be %d bytes long", crypto.SignatureLength) @@ -655,7 +656,7 @@ func (s *BlockChainAPI) GetBalance(ctx context.Context, address common.Address, return (*hexutil.Big)(b), state.Error() } -// Result structs for GetProof +// AccountResult structs for GetProof type AccountResult struct { Address common.Address `json:"address"` AccountProof []string `json:"accountProof"` @@ -978,7 +979,7 @@ func (diff *StateOverride) Apply(statedb *state.StateDB) error { // Override account balance. if account.Balance != nil { u256Balance, _ := uint256.FromBig((*big.Int)(*account.Balance)) - statedb.SetBalance(addr, u256Balance, state.BalanceChangeUnspecified) + statedb.SetBalance(addr, u256Balance, tracing.BalanceChangeUnspecified) } if account.State != nil && account.StateDiff != nil { return fmt.Errorf("account %s has both 'state' and 'stateDiff'", addr.Hex()) @@ -1093,14 +1094,14 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S defer cancel() // Get a new instance of the EVM. - msg, err := args.ToMessage(globalGasCap, header.BaseFee) - if err != nil { - return nil, err - } blockCtx := core.NewEVMBlockContext(header, NewChainContext(ctx, b), nil) if blockOverrides != nil { blockOverrides.Apply(&blockCtx) } + msg, err := args.ToMessage(globalGasCap, blockCtx.BaseFee) + if err != nil { + return nil, err + } evm := b.GetEVM(ctx, msg, state, header, &vm.Config{NoBaseFee: true}, &blockCtx) // Wait for the context to be done and cancel the evm. Even if the @@ -1485,14 +1486,9 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if db == nil || err != nil { return nil, 0, nil, err } - // If the gas amount is not set, default to RPC gas cap. - if args.Gas == nil { - tmp := hexutil.Uint64(b.RPCGasCap()) - args.Gas = &tmp - } // Ensure any missing fields are filled, extract the recipient and input data - if err := args.setDefaults(ctx, b); err != nil { + if err := args.setDefaults(ctx, b, true); err != nil { return nil, 0, nil, err } var to common.Address @@ -1526,7 +1522,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH // Apply the transaction with the access list tracer tracer := logger.NewAccessListTracer(accessList, args.from(), to, precompiles) - config := vm.Config{Tracer: tracer, NoBaseFee: true} + config := vm.Config{Tracer: tracer.Hooks(), NoBaseFee: true} vmenv := b.GetEVM(ctx, msg, statedb, header, &config, nil) res, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)) if err != nil { @@ -1795,7 +1791,7 @@ func (s *TransactionAPI) SendTransaction(ctx context.Context, args TransactionAr } // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return common.Hash{}, err } // Assemble the transaction and sign with the wallet @@ -1812,13 +1808,14 @@ func (s *TransactionAPI) SendTransaction(ctx context.Context, args TransactionAr // on a given unsigned transaction, and returns it to the caller for further // processing (signing + broadcast). func (s *TransactionAPI) FillTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { + args.blobSidecarAllowed = true + // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Assemble the transaction and obtain rlp tx := args.toTransaction() - // TODO(s1na): fill in blob proofs, commitments data, err := tx.MarshalBinary() if err != nil { return nil, err @@ -1883,7 +1880,7 @@ func (s *TransactionAPI) SignTransaction(ctx context.Context, args TransactionAr if args.Nonce == nil { return nil, errors.New("nonce not specified") } - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Before actually sign the transaction, ensure the transaction fee is reasonable. @@ -1932,7 +1929,7 @@ func (s *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, g if sendArgs.Nonce == nil { return common.Hash{}, errors.New("missing transaction nonce in transaction spec") } - if err := sendArgs.setDefaults(ctx, s.b); err != nil { + if err := sendArgs.setDefaults(ctx, s.b, false); err != nil { return common.Hash{}, err } matchTx := sendArgs.toTransaction() diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 623aa1fe42..5636309589 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "crypto/ecdsa" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -45,6 +46,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/blocktest" @@ -442,7 +444,7 @@ func newTestBackend(t *testing.T, n int, gspec *core.Genesis, engine consensus.E } ) accman, acc := newTestAccountManager(t) - gspec.Alloc[acc.Address] = core.GenesisAccount{Balance: big.NewInt(params.Ether)} + gspec.Alloc[acc.Address] = types.Account{Balance: big.NewInt(params.Ether)} // Generate blocks for testing db, blocks, _ := core.GenerateChainWithGenesis(gspec, engine, n, generator) txlookupLimit := uint64(0) @@ -545,7 +547,7 @@ func (b testBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOr } panic("only implemented for number") } -func (b testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { panic("implement me") } +func (b testBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { panic("implement me") } func (b testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { header, err := b.HeaderByHash(ctx, hash) if header == nil || err != nil { @@ -613,9 +615,6 @@ func (b testBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) func (b testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { panic("implement me") } -func (b testBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - panic("implement me") -} func (b testBackend) BloomStatus() (uint64, uint64) { panic("implement me") } func (b testBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { panic("implement me") @@ -628,7 +627,7 @@ func TestEstimateGas(t *testing.T) { accounts = newAccounts(2) genesis = &core.Genesis{ Config: params.MergedTestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, }, @@ -785,7 +784,7 @@ func TestCall(t *testing.T) { accounts = newAccounts(3) genesis = &core.Genesis{ Config: params.MergedTestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, accounts[2].addr: {Balance: big.NewInt(params.Ether)}, @@ -982,7 +981,7 @@ func TestSignTransaction(t *testing.T) { to = crypto.PubkeyToAddress(key.PublicKey) genesis = &core.Genesis{ Config: params.MergedTestChainConfig, - Alloc: core.GenesisAlloc{}, + Alloc: types.GenesisAlloc{}, } ) b := newTestBackend(t, 1, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { @@ -1020,7 +1019,7 @@ func TestSignBlobTransaction(t *testing.T) { to = crypto.PubkeyToAddress(key.PublicKey) genesis = &core.Genesis{ Config: params.MergedTestChainConfig, - Alloc: core.GenesisAlloc{}, + Alloc: types.GenesisAlloc{}, } ) b := newTestBackend(t, 1, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { @@ -1054,7 +1053,7 @@ func TestSendBlobTransaction(t *testing.T) { to = crypto.PubkeyToAddress(key.PublicKey) genesis = &core.Genesis{ Config: params.MergedTestChainConfig, - Alloc: core.GenesisAlloc{}, + Alloc: types.GenesisAlloc{}, } ) b := newTestBackend(t, 1, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { @@ -1079,12 +1078,206 @@ func TestSendBlobTransaction(t *testing.T) { } } +func TestFillBlobTransaction(t *testing.T) { + t.Parallel() + // Initialize test accounts + var ( + key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + to = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.MergedTestChainConfig, + Alloc: types.GenesisAlloc{}, + } + emptyBlob = new(kzg4844.Blob) + emptyBlobs = []kzg4844.Blob{*emptyBlob} + emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) + emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) + emptyBlobHash common.Hash = kzg4844.CalcBlobHashV1(sha256.New(), &emptyBlobCommit) + ) + b := newTestBackend(t, 1, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { + b.SetPoS() + }) + api := NewTransactionAPI(b, nil) + type result struct { + Hashes []common.Hash + Sidecar *types.BlobTxSidecar + } + suite := []struct { + name string + args TransactionArgs + err string + want *result + }{ + { + name: "TestInvalidParamsCombination1", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}}, + Proofs: []kzg4844.Proof{{}}, + }, + err: `blob proofs provided while commitments were not`, + }, + { + name: "TestInvalidParamsCombination2", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}}, + Commitments: []kzg4844.Commitment{{}}, + }, + err: `blob commitments provided while proofs were not`, + }, + { + name: "TestInvalidParamsCount1", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}}, + Commitments: []kzg4844.Commitment{{}, {}}, + Proofs: []kzg4844.Proof{{}, {}}, + }, + err: `number of blobs and commitments mismatch (have=2, want=1)`, + }, + { + name: "TestInvalidParamsCount2", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}, {}}, + Commitments: []kzg4844.Commitment{{}, {}}, + Proofs: []kzg4844.Proof{{}}, + }, + err: `number of blobs and proofs mismatch (have=1, want=2)`, + }, + { + name: "TestInvalidProofVerification", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}, {}}, + Commitments: []kzg4844.Commitment{{}, {}}, + Proofs: []kzg4844.Proof{{}, {}}, + }, + err: `failed to verify blob proof: short buffer`, + }, + { + name: "TestGenerateBlobHashes", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: emptyBlobs, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + want: &result{ + Hashes: []common.Hash{emptyBlobHash}, + Sidecar: &types.BlobTxSidecar{ + Blobs: emptyBlobs, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + }, + }, + { + name: "TestValidBlobHashes", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + BlobHashes: []common.Hash{emptyBlobHash}, + Blobs: emptyBlobs, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + want: &result{ + Hashes: []common.Hash{emptyBlobHash}, + Sidecar: &types.BlobTxSidecar{ + Blobs: emptyBlobs, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + }, + }, + { + name: "TestInvalidBlobHashes", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + BlobHashes: []common.Hash{{0x01, 0x22}}, + Blobs: emptyBlobs, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + err: fmt.Sprintf("blob hash verification failed (have=%s, want=%s)", common.Hash{0x01, 0x22}, emptyBlobHash), + }, + { + name: "TestGenerateBlobProofs", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: emptyBlobs, + }, + want: &result{ + Hashes: []common.Hash{emptyBlobHash}, + Sidecar: &types.BlobTxSidecar{ + Blobs: emptyBlobs, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + }, + }, + } + for _, tc := range suite { + t.Run(tc.name, func(t *testing.T) { + res, err := api.FillTransaction(context.Background(), tc.args) + if len(tc.err) > 0 { + if err == nil { + t.Fatalf("missing error. want: %s", tc.err) + } else if err.Error() != tc.err { + t.Fatalf("error mismatch. want: %s, have: %s", tc.err, err.Error()) + } + return + } + if err != nil && len(tc.err) == 0 { + t.Fatalf("expected no error. have: %s", err) + } + if res == nil { + t.Fatal("result missing") + } + want, err := json.Marshal(tc.want) + if err != nil { + t.Fatalf("failed to encode expected: %v", err) + } + have, err := json.Marshal(result{Hashes: res.Tx.BlobHashes(), Sidecar: res.Tx.BlobTxSidecar()}) + if err != nil { + t.Fatalf("failed to encode computed sidecar: %v", err) + } + if !bytes.Equal(have, want) { + t.Errorf("blob sidecar mismatch. Have: %s, want: %s", have, want) + } + }) + } +} + func argsFromTransaction(tx *types.Transaction, from common.Address) TransactionArgs { var ( - gas = tx.Gas() - nonce = tx.Nonce() - input = tx.Data() + gas = tx.Gas() + nonce = tx.Nonce() + input = tx.Data() + accessList *types.AccessList ) + if acl := tx.AccessList(); acl != nil { + accessList = &acl + } return TransactionArgs{ From: &from, To: tx.To(), @@ -1095,10 +1288,9 @@ func argsFromTransaction(tx *types.Transaction, from common.Address) Transaction Nonce: (*hexutil.Uint64)(&nonce), Input: (*hexutil.Bytes)(&input), ChainID: (*hexutil.Big)(tx.ChainId()), - // TODO: impl accessList conversion - //AccessList: tx.AccessList(), - BlobFeeCap: (*hexutil.Big)(tx.BlobGasFeeCap()), - BlobHashes: tx.BlobHashes(), + AccessList: accessList, + BlobFeeCap: (*hexutil.Big)(tx.BlobGasFeeCap()), + BlobHashes: tx.BlobHashes(), } } @@ -1347,7 +1539,7 @@ func TestRPCGetBlockOrHeader(t *testing.T) { acc2Addr = crypto.PubkeyToAddress(acc2Key.PublicKey) genesis = &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ acc1Addr: {Balance: big.NewInt(params.Ether)}, acc2Addr: {Balance: big.NewInt(params.Ether)}, }, @@ -1602,7 +1794,7 @@ func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Ha Config: &config, ExcessBlobGas: new(uint64), BlobGasUsed: new(uint64), - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ acc1Addr: {Balance: big.NewInt(params.Ether)}, acc2Addr: {Balance: big.NewInt(params.Ether)}, // // SPDX-License-Identifier: GPL-3.0 @@ -1627,6 +1819,7 @@ func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Ha tx *types.Transaction err error ) + b.SetPoS() switch i { case 0: // transfer 1000wei @@ -1675,7 +1868,6 @@ func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Ha b.AddTx(tx) txHashes[i] = tx.Hash() } - b.SetPoS() }) return backend, txHashes } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 5f408ba20b..fd2f5699ea 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -65,7 +65,7 @@ type Backend interface { BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) - PendingBlockAndReceipts() (*types.Block, types.Receipts) + Pending() (*types.Block, types.Receipts, *state.StateDB) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) GetTd(ctx context.Context, hash common.Hash) *big.Int GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM @@ -94,7 +94,6 @@ type Backend interface { GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription BloomStatus() (uint64, uint64) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) } diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 75dbe38a59..2751d5b5aa 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -19,6 +19,7 @@ package ethapi import ( "bytes" "context" + "crypto/sha256" "errors" "fmt" "math/big" @@ -29,11 +30,17 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/holiman/uint256" ) +var ( + maxBlobsPerTransaction = params.MaxBlobGasPerBlock / params.BlobTxBlobGasPerBlob +) + // TransactionArgs represents the arguments to construct a new transaction // or a message call. type TransactionArgs struct { @@ -56,9 +63,17 @@ type TransactionArgs struct { AccessList *types.AccessList `json:"accessList,omitempty"` ChainID *hexutil.Big `json:"chainId,omitempty"` - // Introduced by EIP-4844. + // For BlobTxType BlobFeeCap *hexutil.Big `json:"maxFeePerBlobGas"` BlobHashes []common.Hash `json:"blobVersionedHashes,omitempty"` + + // For BlobTxType transactions with blob sidecar + Blobs []kzg4844.Blob `json:"blobs"` + Commitments []kzg4844.Commitment `json:"commitments"` + Proofs []kzg4844.Proof `json:"proofs"` + + // This configures whether blobs are allowed to be passed. + blobSidecarAllowed bool } // from retrieves the transaction sender address. @@ -81,10 +96,14 @@ func (args *TransactionArgs) data() []byte { } // setDefaults fills in default values for unspecified tx fields. -func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { +func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, skipGasEstimation bool) error { + if err := args.setBlobTxSidecar(ctx, b); err != nil { + return err + } if err := args.setFeeDefaults(ctx, b); err != nil { return err } + if args.Value == nil { args.Value = new(hexutil.Big) } @@ -98,38 +117,58 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { if args.Data != nil && args.Input != nil && !bytes.Equal(*args.Data, *args.Input) { return errors.New(`both "data" and "input" are set and not equal. Please use "input" to pass transaction call data`) } - if args.BlobHashes != nil && args.To == nil { - return errors.New(`blob transactions cannot have the form of a create transaction`) - } + + // BlobTx fields if args.BlobHashes != nil && len(args.BlobHashes) == 0 { return errors.New(`need at least 1 blob for a blob transaction`) } - if args.To == nil && len(args.data()) == 0 { - return errors.New(`contract creation without any data provided`) + if args.BlobHashes != nil && len(args.BlobHashes) > maxBlobsPerTransaction { + return fmt.Errorf(`too many blobs in transaction (have=%d, max=%d)`, len(args.BlobHashes), maxBlobsPerTransaction) } - // Estimate the gas usage if necessary. + + // create check + if args.To == nil { + if args.BlobHashes != nil { + return errors.New(`missing "to" in blob transaction`) + } + if len(args.data()) == 0 { + return errors.New(`contract creation without any data provided`) + } + } + if args.Gas == nil { - // These fields are immutable during the estimation, safe to - // pass the pointer directly. - data := args.data() - callArgs := TransactionArgs{ - From: args.From, - To: args.To, - GasPrice: args.GasPrice, - MaxFeePerGas: args.MaxFeePerGas, - MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, - Value: args.Value, - Data: (*hexutil.Bytes)(&data), - AccessList: args.AccessList, + if skipGasEstimation { // Skip gas usage estimation if a precise gas limit is not critical, e.g., in non-transaction calls. + gas := hexutil.Uint64(b.RPCGasCap()) + if gas == 0 { + gas = hexutil.Uint64(math.MaxUint64 / 2) + } + args.Gas = &gas + } else { // Estimate the gas usage otherwise. + // These fields are immutable during the estimation, safe to + // pass the pointer directly. + data := args.data() + callArgs := TransactionArgs{ + From: args.From, + To: args.To, + GasPrice: args.GasPrice, + MaxFeePerGas: args.MaxFeePerGas, + MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, + Value: args.Value, + Data: (*hexutil.Bytes)(&data), + AccessList: args.AccessList, + BlobFeeCap: args.BlobFeeCap, + BlobHashes: args.BlobHashes, + } + latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, b.RPCGasCap()) + if err != nil { + return err + } + args.Gas = &estimated + log.Trace("Estimate gas usage automatically", "gas", args.Gas) } - latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) - estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, b.RPCGasCap()) - if err != nil { - return err - } - args.Gas = &estimated - log.Trace("Estimate gas usage automatically", "gas", args.Gas) } + // If chain id is provided, ensure it matches the local chain id. Otherwise, set the local // chain id as the default. want := b.ChainConfig().ChainID @@ -145,6 +184,14 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { // setFeeDefaults fills in default fee values for unspecified tx fields. func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) error { + head := b.CurrentHeader() + // Sanity check the EIP-4844 fee parameters. + if args.BlobFeeCap != nil && args.BlobFeeCap.ToInt().Sign() == 0 { + return errors.New("maxFeePerBlobGas, if specified, must be non-zero") + } + if err := args.setCancunFeeDefaults(ctx, head, b); err != nil { + return err + } // If both gasPrice and at least one of the EIP-1559 fee parameters are specified, error. if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") @@ -154,7 +201,6 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro // other tx values. See https://github.com/ethereum/go-ethereum/pull/23274 // for more information. eip1559ParamsSet := args.MaxFeePerGas != nil && args.MaxPriorityFeePerGas != nil - // Sanity check the EIP-1559 fee parameters if present. if args.GasPrice == nil && eip1559ParamsSet { if args.MaxFeePerGas.ToInt().Sign() == 0 { @@ -165,12 +211,8 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro } return nil // No need to set anything, user already set MaxFeePerGas and MaxPriorityFeePerGas } - // Sanity check the EIP-4844 fee parameters. - if args.BlobFeeCap != nil && args.BlobFeeCap.ToInt().Sign() == 0 { - return errors.New("maxFeePerBlobGas must be non-zero") - } + // Sanity check the non-EIP-1559 fee parameters. - head := b.CurrentHeader() isLondon := b.ChainConfig().IsLondon(head.Number) if args.GasPrice != nil && !eip1559ParamsSet { // Zero gas-price is not allowed after London fork @@ -181,21 +223,14 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro } // Now attempt to fill in default value depending on whether London is active or not. - if b.ChainConfig().IsCancun(head.Number, head.Time) { - if err := args.setCancunFeeDefaults(ctx, head, b); err != nil { - return err - } - } else if isLondon { - if args.BlobFeeCap != nil { - return errors.New("maxFeePerBlobGas is not valid before Cancun is active") - } + if isLondon { // London is active, set maxPriorityFeePerGas and maxFeePerGas. if err := args.setLondonFeeDefaults(ctx, head, b); err != nil { return err } } else { - if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil || args.BlobFeeCap != nil { - return errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active") + if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { + return errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active") } // London not active, set gas price. price, err := b.SuggestGasTipCap(ctx) @@ -211,15 +246,19 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro func (args *TransactionArgs) setCancunFeeDefaults(ctx context.Context, head *types.Header, b Backend) error { // Set maxFeePerBlobGas if it is missing. if args.BlobHashes != nil && args.BlobFeeCap == nil { + var excessBlobGas uint64 + if head.ExcessBlobGas != nil { + excessBlobGas = *head.ExcessBlobGas + } // ExcessBlobGas must be set for a Cancun block. - blobBaseFee := eip4844.CalcBlobFee(*head.ExcessBlobGas) + blobBaseFee := eip4844.CalcBlobFee(excessBlobGas) // Set the max fee to be 2 times larger than the previous block's blob base fee. // The additional slack allows the tx to not become invalidated if the base // fee is rising. val := new(big.Int).Mul(blobBaseFee, big.NewInt(2)) args.BlobFeeCap = (*hexutil.Big)(val) } - return args.setLondonFeeDefaults(ctx, head, b) + return nil } // setLondonFeeDefaults fills in reasonable default fee values for unspecified fields. @@ -250,6 +289,81 @@ func (args *TransactionArgs) setLondonFeeDefaults(ctx context.Context, head *typ return nil } +// setBlobTxSidecar adds the blob tx +func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, b Backend) error { + // No blobs, we're done. + if args.Blobs == nil { + return nil + } + + // Passing blobs is not allowed in all contexts, only in specific methods. + if !args.blobSidecarAllowed { + return errors.New(`"blobs" is not supported for this RPC method`) + } + + n := len(args.Blobs) + // Assume user provides either only blobs (w/o hashes), or + // blobs together with commitments and proofs. + if args.Commitments == nil && args.Proofs != nil { + return errors.New(`blob proofs provided while commitments were not`) + } else if args.Commitments != nil && args.Proofs == nil { + return errors.New(`blob commitments provided while proofs were not`) + } + + // len(blobs) == len(commitments) == len(proofs) == len(hashes) + if args.Commitments != nil && len(args.Commitments) != n { + return fmt.Errorf("number of blobs and commitments mismatch (have=%d, want=%d)", len(args.Commitments), n) + } + if args.Proofs != nil && len(args.Proofs) != n { + return fmt.Errorf("number of blobs and proofs mismatch (have=%d, want=%d)", len(args.Proofs), n) + } + if args.BlobHashes != nil && len(args.BlobHashes) != n { + return fmt.Errorf("number of blobs and hashes mismatch (have=%d, want=%d)", len(args.BlobHashes), n) + } + + if args.Commitments == nil { + // Generate commitment and proof. + commitments := make([]kzg4844.Commitment, n) + proofs := make([]kzg4844.Proof, n) + for i, b := range args.Blobs { + c, err := kzg4844.BlobToCommitment(&b) + if err != nil { + return fmt.Errorf("blobs[%d]: error computing commitment: %v", i, err) + } + commitments[i] = c + p, err := kzg4844.ComputeBlobProof(&b, c) + if err != nil { + return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) + } + proofs[i] = p + } + args.Commitments = commitments + args.Proofs = proofs + } else { + for i, b := range args.Blobs { + if err := kzg4844.VerifyBlobProof(&b, args.Commitments[i], args.Proofs[i]); err != nil { + return fmt.Errorf("failed to verify blob proof: %v", err) + } + } + } + + hashes := make([]common.Hash, n) + hasher := sha256.New() + for i, c := range args.Commitments { + hashes[i] = kzg4844.CalcBlobHashV1(hasher, &c) + } + if args.BlobHashes != nil { + for i, h := range hashes { + if h != args.BlobHashes[i] { + return fmt.Errorf("blob hash verification failed (have=%s, want=%s)", args.BlobHashes[i], h) + } + } + } else { + args.BlobHashes = hashes + } + return nil +} + // ToMessage converts the transaction arguments to the Message type used by the // core evm. This method is used in calls and traces that do not require a real // live transaction. @@ -363,6 +477,14 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { BlobHashes: args.BlobHashes, BlobFeeCap: uint256.MustFromBig((*big.Int)(args.BlobFeeCap)), } + if args.Blobs != nil { + data.(*types.BlobTx).Sidecar = &types.BlobTxSidecar{ + Blobs: args.Blobs, + Commitments: args.Commitments, + Proofs: args.Proofs, + } + } + case args.MaxFeePerGas != nil: al := types.AccessList{} if args.AccessList != nil { @@ -379,6 +501,7 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { Data: args.data(), AccessList: al, } + case args.AccessList != nil: data = &types.AccessListTx{ To: args.To, @@ -390,6 +513,7 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { Data: args.data(), AccessList: *args.AccessList, } + default: data = &types.LegacyTx{ To: args.To, @@ -403,12 +527,6 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { return types.NewTx(data) } -// ToTransaction converts the arguments to a transaction. -// This assumes that setDefaults has been called. -func (args *TransactionArgs) ToTransaction() *types.Transaction { - return args.toTransaction() -} - // IsEIP4844 returns an indicator if the args contains EIP4844 fields. func (args *TransactionArgs) IsEIP4844() bool { return args.BlobHashes != nil || args.BlobFeeCap != nil diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index f0fdb6d8ee..24ecb1dee4 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -153,14 +153,14 @@ func TestSetFeeDefaults(t *testing.T) { "legacy", &TransactionArgs{MaxFeePerGas: maxFee}, nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), + errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), }, { "dynamic fee tx pre-London, priorityFee set", "legacy", &TransactionArgs{MaxPriorityFeePerGas: fortytwo}, nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), + errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), }, { "dynamic fee tx, maxFee < priorityFee", @@ -207,20 +207,6 @@ func TestSetFeeDefaults(t *testing.T) { errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), }, // EIP-4844 - { - "set maxFeePerBlobGas pre cancun", - "london", - &TransactionArgs{BlobFeeCap: fortytwo}, - nil, - errors.New("maxFeePerBlobGas is not valid before Cancun is active"), - }, - { - "set maxFeePerBlobGas pre london", - "legacy", - &TransactionArgs{BlobFeeCap: fortytwo}, - nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), - }, { "set gas price and maxFee for blob transaction", "cancun", @@ -235,6 +221,13 @@ func TestSetFeeDefaults(t *testing.T) { &TransactionArgs{BlobHashes: []common.Hash{}, BlobFeeCap: (*hexutil.Big)(big.NewInt(4)), MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, nil, }, + { + "fill maxFeePerBlobGas when dynamic fees are set", + "cancun", + &TransactionArgs{BlobHashes: []common.Hash{}, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + &TransactionArgs{BlobHashes: []common.Hash{}, BlobFeeCap: (*hexutil.Big)(big.NewInt(4)), MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, } ctx := context.Background() @@ -244,11 +237,16 @@ func TestSetFeeDefaults(t *testing.T) { } got := test.in err := got.setFeeDefaults(ctx, b) - if err != nil && err.Error() == test.err.Error() { - // Test threw expected error. + if err != nil { + if test.err == nil { + t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } else if err.Error() != test.err.Error() { + t.Fatalf("test %d (%s): unexpected error: (got: %s, want: %s)", i, test.name, err, test.err) + } + // Matching error. continue - } else if err != nil { - t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } else if test.err != nil { + t.Fatalf("test %d (%s): expected error: %s", i, test.name, test.err) } if !reflect.DeepEqual(got, test.want) { t.Fatalf("test %d (%s): did not fill defaults as expected: (got: %v, want: %v)", i, test.name, got, test.want) @@ -360,7 +358,7 @@ func (b *backendMock) StateAndHeaderByNumber(ctx context.Context, number rpc.Blo func (b *backendMock) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { return nil, nil, nil } -func (b *backendMock) PendingBlockAndReceipts() (*types.Block, types.Receipts) { return nil, nil } +func (b *backendMock) Pending() (*types.Block, types.Receipts, *state.StateDB) { return nil, nil, nil } func (b *backendMock) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { return nil, nil } @@ -398,9 +396,6 @@ func (b *backendMock) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscr func (b *backendMock) BloomStatus() (uint64, uint64) { return 0, 0 } func (b *backendMock) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {} func (b *backendMock) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return nil } -func (b *backendMock) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - return nil -} func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { return nil } diff --git a/internal/flags/categories.go b/internal/flags/categories.go index 3ff0767921..d426add55b 100644 --- a/internal/flags/categories.go +++ b/internal/flags/categories.go @@ -20,7 +20,7 @@ import "github.com/urfave/cli/v2" const ( EthCategory = "ETHEREUM" - LightCategory = "LIGHT CLIENT" + BeaconCategory = "BEACON CHAIN" DevCategory = "DEVELOPER CHAIN" StateCategory = "STATE HISTORY MANAGEMENT" TxPoolCategory = "TRANSACTION POOL (EVM)" diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 69e9743556..bf62c53adf 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -256,7 +256,8 @@ type BigFlag struct { Hidden bool HasBeenSet bool - Value *big.Int + Value *big.Int + defaultValue *big.Int Aliases []string EnvVars []string @@ -269,6 +270,10 @@ func (f *BigFlag) IsSet() bool { return f.HasBeenSet } func (f *BigFlag) String() string { return cli.FlagStringer(f) } func (f *BigFlag) Apply(set *flag.FlagSet) error { + // Set default value so that environment wont be able to overwrite it + if f.Value != nil { + f.defaultValue = new(big.Int).Set(f.Value) + } for _, envVar := range f.EnvVars { envVar = strings.TrimSpace(envVar) if value, found := syscall.Getenv(envVar); found { @@ -283,7 +288,6 @@ func (f *BigFlag) Apply(set *flag.FlagSet) error { f.Value = new(big.Int) set.Var((*bigValue)(f.Value), f.Name, f.Usage) }) - return nil } @@ -310,7 +314,7 @@ func (f *BigFlag) GetDefaultText() string { if f.DefaultText != "" { return f.DefaultText } - return f.GetValue() + return f.defaultValue.String() } // bigValue turns *big.Int into a flag.Value diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index 6ccf09b1cc..4196cb8db0 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -2031,7 +2031,7 @@ var fromAscii = function(str) { * * @method transformToFullName * @param {Object} json-abi - * @return {String} full fnction/event name + * @return {String} full function/event name */ var transformToFullName = function (json) { if (json.name.indexOf('(') !== -1) { @@ -2361,7 +2361,7 @@ var isFunction = function (object) { }; /** - * Returns true if object is Objet, otherwise false + * Returns true if object is Object, otherwise false * * @method isObject * @param {Object} @@ -2757,7 +2757,7 @@ var Batch = function (web3) { * Should be called to add create new request to batch request * * @method add - * @param {Object} jsonrpc requet object + * @param {Object} jsonrpc request object */ Batch.prototype.add = function (request) { this.requests.push(request); @@ -3734,7 +3734,7 @@ var inputCallFormatter = function (options){ options.to = inputAddressFormatter(options.to); } - ['maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { + ['maxFeePerBlobGas', 'maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { return options[key] !== undefined; }).forEach(function(key){ options[key] = utils.fromDecimal(options[key]); @@ -3759,7 +3759,7 @@ var inputTransactionFormatter = function (options){ options.to = inputAddressFormatter(options.to); } - ['maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { + ['maxFeePerBlobGas', 'maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { return options[key] !== undefined; }).forEach(function(key){ options[key] = utils.fromDecimal(options[key]); @@ -3789,6 +3789,9 @@ var outputTransactionFormatter = function (tx){ if(tx.maxPriorityFeePerGas !== undefined) { tx.maxPriorityFeePerGas = utils.toBigNumber(tx.maxPriorityFeePerGas); } + if(tx.maxFeePerBlobGas !== undefined) { + tx.maxFeePerBlobGas = utils.toBigNumber(tx.maxFeePerBlobGas); + } tx.value = utils.toBigNumber(tx.value); return tx; }; @@ -3810,6 +3813,12 @@ var outputTransactionReceiptFormatter = function (receipt){ if(receipt.effectiveGasPrice !== undefined) { receipt.effectiveGasPrice = utils.toBigNumber(receipt.effectiveGasPrice); } + if(receipt.blobGasPrice !== undefined) { + receipt.blobGasPrice = utils.toBigNumber(receipt.blobGasPrice); + } + if(receipt.blobGasUsed !== undefined) { + receipt.blobGasUsed = utils.toBigNumber(receipt.blobGasUsed); + } if(utils.isArray(receipt.logs)) { receipt.logs = receipt.logs.map(function(log){ return outputLogFormatter(log); @@ -3831,11 +3840,17 @@ var outputBlockFormatter = function(block) { if (block.baseFeePerGas !== undefined) { block.baseFeePerGas = utils.toBigNumber(block.baseFeePerGas); } + if (block.blobGasUsed !== undefined) { + block.blobGasUsed = utils.toBigNumber(block.blobGasUsed); + } + if (block.excessBlobGas !== undefined) { + block.excessBlobGas = utils.toBigNumber(block.excessBlobGas); + } block.gasLimit = utils.toDecimal(block.gasLimit); block.gasUsed = utils.toDecimal(block.gasUsed); block.size = utils.toDecimal(block.size); block.timestamp = utils.toDecimal(block.timestamp); - if(block.number !== null) + if (block.number !== null) block.number = utils.toDecimal(block.number); block.difficulty = utils.toBigNumber(block.difficulty); @@ -4559,7 +4574,7 @@ Iban.createIndirect = function (options) { }; /** - * Thos method should be used to check if given string is valid iban object + * This method should be used to check if given string is valid iban object * * @method isValid * @param {String} iban string @@ -6708,7 +6723,7 @@ var exchangeAbi = require('../contracts/SmartExchange.json'); * @method transfer * @param {String} from * @param {String} to iban - * @param {Value} value to be tranfered + * @param {Value} value to be transferred * @param {Function} callback, callback */ var transfer = function (eth, from, to, value, callback) { @@ -6738,7 +6753,7 @@ var transfer = function (eth, from, to, value, callback) { * @method transferToAddress * @param {String} from * @param {String} to - * @param {Value} value to be tranfered + * @param {Value} value to be transferred * @param {Function} callback, callback */ var transferToAddress = function (eth, from, to, value, callback) { @@ -7092,7 +7107,7 @@ module.exports = transfer; /** * Initializes a newly created cipher. * - * @param {number} xformMode Either the encryption or decryption transormation mode constant. + * @param {number} xformMode Either the encryption or decryption transformation mode constant. * @param {WordArray} key The key. * @param {Object} cfg (Optional) The configuration options to use for this operation. * @@ -9446,7 +9461,7 @@ module.exports = transfer; var M_offset_14 = M[offset + 14]; var M_offset_15 = M[offset + 15]; - // Working varialbes + // Working variables var a = H[0]; var b = H[1]; var c = H[2]; diff --git a/internal/testlog/testlog.go b/internal/testlog/testlog.go index 037b7ee9c1..3cdbea6e05 100644 --- a/internal/testlog/testlog.go +++ b/internal/testlog/testlog.go @@ -98,6 +98,10 @@ func LoggerWithHandler(t *testing.T, handler slog.Handler) log.Logger { } } +func (l *logger) Handler() slog.Handler { + return l.l.Handler() +} + func (l *logger) Write(level slog.Level, msg string, ctx ...interface{}) {} func (l *logger) Enabled(ctx context.Context, level slog.Level) bool { diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index b86b5909d2..1da7d737dd 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -649,20 +649,6 @@ const MinerJs = ` web3._extend({ property: 'miner', methods: [ - new web3._extend.Method({ - name: 'start', - call: 'miner_start', - }), - new web3._extend.Method({ - name: 'stop', - call: 'miner_stop' - }), - new web3._extend.Method({ - name: 'setEtherbase', - call: 'miner_setEtherbase', - params: 1, - inputFormatter: [web3._extend.formatters.inputAddressFormatter] - }), new web3._extend.Method({ name: 'setExtra', call: 'miner_setExtra', @@ -680,15 +666,6 @@ web3._extend({ params: 1, inputFormatter: [web3._extend.utils.fromDecimal] }), - new web3._extend.Method({ - name: 'setRecommitInterval', - call: 'miner_setRecommitInterval', - params: 1, - }), - new web3._extend.Method({ - name: 'getHashrate', - call: 'miner_getHashrate' - }), ], properties: [] }); diff --git a/log/handler_glog.go b/log/handler_glog.go index fb1e03c5b5..f51bae2a4a 100644 --- a/log/handler_glog.go +++ b/log/handler_glog.go @@ -192,7 +192,7 @@ func (h *GlogHandler) Handle(_ context.Context, r slog.Record) error { frame, _ := fs.Next() for _, rule := range h.patterns { - if rule.pattern.MatchString(fmt.Sprintf("%+s", frame.File)) { + if rule.pattern.MatchString(fmt.Sprintf("+%s", frame.File)) { h.siteCache[r.PC], lvl, ok = rule.level, rule.level, true } } diff --git a/log/logger.go b/log/logger.go index 75e3643044..c28bbde568 100644 --- a/log/logger.go +++ b/log/logger.go @@ -137,6 +137,9 @@ type Logger interface { // Enabled reports whether l emits log records at the given context and level. Enabled(ctx context.Context, level slog.Level) bool + + // Handler returns the underlying handler of the inner logger. + Handler() slog.Handler } type logger struct { @@ -150,6 +153,10 @@ func NewLogger(h slog.Handler) Logger { } } +func (l *logger) Handler() slog.Handler { + return l.inner.Handler() +} + // write logs a message at the specified level: func (l *logger) Write(level slog.Level, msg string, attrs ...any) { if !l.inner.Enabled(context.Background(), level) { diff --git a/log/logger_test.go b/log/logger_test.go index a633f5ad7a..ff981fd018 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -2,6 +2,7 @@ package log import ( "bytes" + "errors" "fmt" "io" "math/big" @@ -77,7 +78,7 @@ func benchmarkLogger(b *testing.B, l Logger) { tt = time.Now() bigint = big.NewInt(100) nilbig *big.Int - err = fmt.Errorf("Oh nooes it's crap") + err = errors.New("Oh nooes it's crap") ) b.ReportAllocs() b.ResetTimer() @@ -106,7 +107,7 @@ func TestLoggerOutput(t *testing.T) { tt = time.Time{} bigint = big.NewInt(100) nilbig *big.Int - err = fmt.Errorf("Oh nooes it's crap") + err = errors.New("Oh nooes it's crap") smallUint = uint256.NewInt(500_000) bigUint = &uint256.Int{0xff, 0xff, 0xff, 0xff} ) diff --git a/metrics/config.go b/metrics/config.go index 2eb09fb48a..72f94dd194 100644 --- a/metrics/config.go +++ b/metrics/config.go @@ -19,7 +19,7 @@ package metrics // Config contains the configuration for the metric collection. type Config struct { Enabled bool `toml:",omitempty"` - EnabledExpensive bool `toml:",omitempty"` + EnabledExpensive bool `toml:"-"` HTTP string `toml:",omitempty"` Port int `toml:",omitempty"` EnableInfluxDB bool `toml:",omitempty"` diff --git a/metrics/counter.go b/metrics/counter.go index cb81599c21..dbe8e16a90 100644 --- a/metrics/counter.go +++ b/metrics/counter.go @@ -8,7 +8,7 @@ type CounterSnapshot interface { Count() int64 } -// Counters hold an int64 value that can be incremented and decremented. +// Counter hold an int64 value that can be incremented and decremented. type Counter interface { Clear() Dec(int64) diff --git a/metrics/gauge.go b/metrics/gauge.go index 68f8f11abc..5933df3107 100644 --- a/metrics/gauge.go +++ b/metrics/gauge.go @@ -2,12 +2,12 @@ package metrics import "sync/atomic" -// gaugeSnapshot contains a readonly int64. +// GaugeSnapshot contains a readonly int64. type GaugeSnapshot interface { Value() int64 } -// Gauges hold an int64 value that can be set arbitrarily. +// Gauge holds an int64 value that can be set arbitrarily. type Gauge interface { Snapshot() GaugeSnapshot Update(int64) @@ -74,7 +74,7 @@ func (g *StandardGauge) Update(v int64) { g.value.Store(v) } -// Update updates the gauge's value if v is larger then the current valie. +// Update updates the gauge's value if v is larger then the current value. func (g *StandardGauge) UpdateIfGt(v int64) { for { exist := g.value.Load() diff --git a/metrics/gauge_float64.go b/metrics/gauge_float64.go index 967f2bc60e..c1c3c6b6e6 100644 --- a/metrics/gauge_float64.go +++ b/metrics/gauge_float64.go @@ -48,7 +48,7 @@ type gaugeFloat64Snapshot float64 // Value returns the value at the time the snapshot was taken. func (g gaugeFloat64Snapshot) Value() float64 { return float64(g) } -// NilGauge is a no-op Gauge. +// NilGaugeFloat64 is a no-op Gauge. type NilGaugeFloat64 struct{} func (NilGaugeFloat64) Snapshot() GaugeFloat64Snapshot { return NilGaugeFloat64{} } diff --git a/metrics/gauge_info.go b/metrics/gauge_info.go index c44b2d85f3..0010edc324 100644 --- a/metrics/gauge_info.go +++ b/metrics/gauge_info.go @@ -9,7 +9,7 @@ type GaugeInfoSnapshot interface { Value() GaugeInfoValue } -// GaugeInfos hold a GaugeInfoValue value that can be set arbitrarily. +// GaugeInfo holds a GaugeInfoValue value that can be set arbitrarily. type GaugeInfo interface { Update(GaugeInfoValue) Snapshot() GaugeInfoSnapshot diff --git a/metrics/healthcheck.go b/metrics/healthcheck.go index f1ae31e34a..adcd15ab58 100644 --- a/metrics/healthcheck.go +++ b/metrics/healthcheck.go @@ -1,6 +1,6 @@ package metrics -// Healthchecks hold an error value describing an arbitrary up/down status. +// Healthcheck holds an error value describing an arbitrary up/down status. type Healthcheck interface { Check() Error() error diff --git a/metrics/histogram.go b/metrics/histogram.go index 44de588bc1..10259a2463 100644 --- a/metrics/histogram.go +++ b/metrics/histogram.go @@ -4,7 +4,7 @@ type HistogramSnapshot interface { SampleSnapshot } -// Histograms calculate distribution statistics from a series of int64 values. +// Histogram calculates distribution statistics from a series of int64 values. type Histogram interface { Clear() Update(int64) diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index bbc4fc024b..5c8501fd9d 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -98,20 +98,23 @@ func readMeter(namespace, name string, i interface{}) (string, map[string]interf } return measurement, fields case metrics.ResettingTimer: - t := metric.Snapshot() - if t.Count() == 0 { + ms := metric.Snapshot() + if ms.Count() == 0 { break } - ps := t.Percentiles([]float64{0.50, 0.95, 0.99}) - measurement := fmt.Sprintf("%s%s.span", namespace, name) + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + measurement := fmt.Sprintf("%s%s.timer", namespace, name) fields := map[string]interface{}{ - "count": t.Count(), - "max": t.Max(), - "mean": t.Mean(), - "min": t.Min(), - "p50": int(ps[0]), - "p95": int(ps[1]), - "p99": int(ps[2]), + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], } return measurement, fields } diff --git a/metrics/influxdb/influxdbv2.go b/metrics/influxdb/influxdbv2.go index 0be5137d5e..114d57ae07 100644 --- a/metrics/influxdb/influxdbv2.go +++ b/metrics/influxdb/influxdbv2.go @@ -25,7 +25,7 @@ type v2Reporter struct { write api.WriteAPI } -// InfluxDBWithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags +// InfluxDBV2WithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags func InfluxDBV2WithTags(r metrics.Registry, d time.Duration, endpoint string, token string, bucket string, organization string, namespace string, tags map[string]string) { rep := &v2Reporter{ reg: r, diff --git a/metrics/influxdb/testdata/influxdbv1.want b/metrics/influxdb/testdata/influxdbv1.want index 9443faedc5..ded9434c73 100644 --- a/metrics/influxdb/testdata/influxdbv1.want +++ b/metrics/influxdb/testdata/influxdbv1.want @@ -7,5 +7,5 @@ goth.test/gauge_float64.gauge value=34567.89 978307200000000000 goth.test/gauge_info.gauge value="{\"arch\":\"amd64\",\"commit\":\"7caa2d8163ae3132c1c2d6978c76610caee2d949\",\"os\":\"linux\",\"protocol_versions\":\"64 65 66\",\"version\":\"1.10.18-unstable\"}" 978307200000000000 goth.test/histogram.histogram count=3i,max=3i,mean=2,min=1i,p25=1,p50=2,p75=3,p95=3,p99=3,p999=3,p9999=3,stddev=0.816496580927726,variance=0.6666666666666666 978307200000000000 goth.test/meter.meter count=0i,m1=0,m15=0,m5=0,mean=0 978307200000000000 -goth.test/resetting_timer.span count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000i,p95=120000000i,p99=120000000i 978307200000000000 +goth.test/resetting_timer.timer count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000,p75=40500000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000 978307200000000000 goth.test/timer.timer count=6i,m1=0,m15=0,m5=0,max=120000000i,mean=38333333.333333336,meanrate=0,min=20000000i,p50=22500000,p75=48000000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000,stddev=36545253.529775314,variance=1335555555555555.2 978307200000000000 diff --git a/metrics/influxdb/testdata/influxdbv2.want b/metrics/influxdb/testdata/influxdbv2.want index 9443faedc5..ded9434c73 100644 --- a/metrics/influxdb/testdata/influxdbv2.want +++ b/metrics/influxdb/testdata/influxdbv2.want @@ -7,5 +7,5 @@ goth.test/gauge_float64.gauge value=34567.89 978307200000000000 goth.test/gauge_info.gauge value="{\"arch\":\"amd64\",\"commit\":\"7caa2d8163ae3132c1c2d6978c76610caee2d949\",\"os\":\"linux\",\"protocol_versions\":\"64 65 66\",\"version\":\"1.10.18-unstable\"}" 978307200000000000 goth.test/histogram.histogram count=3i,max=3i,mean=2,min=1i,p25=1,p50=2,p75=3,p95=3,p99=3,p999=3,p9999=3,stddev=0.816496580927726,variance=0.6666666666666666 978307200000000000 goth.test/meter.meter count=0i,m1=0,m15=0,m5=0,mean=0 978307200000000000 -goth.test/resetting_timer.span count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000i,p95=120000000i,p99=120000000i 978307200000000000 +goth.test/resetting_timer.timer count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000,p75=40500000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000 978307200000000000 goth.test/timer.timer count=6i,m1=0,m15=0,m5=0,max=120000000i,mean=38333333.333333336,meanrate=0,min=20000000i,p50=22500000,p75=48000000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000,stddev=36545253.529775314,variance=1335555555555555.2 978307200000000000 diff --git a/metrics/metrics.go b/metrics/metrics.go index 9ca8f115c0..9e0ac23dd5 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -24,23 +24,12 @@ import ( // for less cluttered pprof profiles. var Enabled = false -// EnabledExpensive is a soft-flag meant for external packages to check if costly -// metrics gathering is allowed or not. The goal is to separate standard metrics -// for health monitoring and debug metrics that might impact runtime performance. -var EnabledExpensive = false - // enablerFlags is the CLI flag names to use to enable metrics collections. var enablerFlags = []string{"metrics"} // enablerEnvVars is the env var names to use to enable metrics collections. var enablerEnvVars = []string{"GETH_METRICS"} -// expensiveEnablerFlags is the CLI flag names to use to enable metrics collections. -var expensiveEnablerFlags = []string{"metrics.expensive"} - -// expensiveEnablerEnvVars is the env var names to use to enable metrics collections. -var expensiveEnablerEnvVars = []string{"GETH_METRICS_EXPENSIVE"} - // Init enables or disables the metrics system. Since we need this to run before // any other code gets to create meters and timers, we'll actually do an ugly hack // and peek into the command line args for the metrics flag. @@ -53,14 +42,6 @@ func init() { } } } - for _, enabler := range expensiveEnablerEnvVars { - if val, found := syscall.Getenv(enabler); found && !EnabledExpensive { - if enable, _ := strconv.ParseBool(val); enable { // ignore error, flag parser will choke on it later - log.Info("Enabling expensive metrics collection") - EnabledExpensive = true - } - } - } for _, arg := range os.Args { flag := strings.TrimLeft(arg, "-") @@ -70,12 +51,6 @@ func init() { Enabled = true } } - for _, enabler := range expensiveEnablerFlags { - if !EnabledExpensive && flag == enabler { - log.Info("Enabling expensive metrics collection") - EnabledExpensive = true - } - } } } diff --git a/metrics/prometheus/collector.go b/metrics/prometheus/collector.go index 25b258d56a..353336763b 100644 --- a/metrics/prometheus/collector.go +++ b/metrics/prometheus/collector.go @@ -125,12 +125,13 @@ func (c *collector) addResettingTimer(name string, m metrics.ResettingTimerSnaps if m.Count() <= 0 { return } - ps := m.Percentiles([]float64{0.50, 0.95, 0.99}) + pv := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999} + ps := m.Percentiles(pv) c.writeSummaryCounter(name, m.Count()) c.buff.WriteString(fmt.Sprintf(typeSummaryTpl, mutateKey(name))) - c.writeSummaryPercentile(name, "0.50", ps[0]) - c.writeSummaryPercentile(name, "0.95", ps[1]) - c.writeSummaryPercentile(name, "0.99", ps[2]) + for i := range pv { + c.writeSummaryPercentile(name, strconv.FormatFloat(pv[i], 'f', -1, 64), ps[i]) + } c.buff.WriteRune('\n') } diff --git a/metrics/prometheus/testdata/prometheus.want b/metrics/prometheus/testdata/prometheus.want index 861c5f5cf0..a999d83801 100644 --- a/metrics/prometheus/testdata/prometheus.want +++ b/metrics/prometheus/testdata/prometheus.want @@ -53,9 +53,12 @@ test_meter 0 test_resetting_timer_count 6 # TYPE test_resetting_timer summary -test_resetting_timer {quantile="0.50"} 1.25e+07 +test_resetting_timer {quantile="0.5"} 1.25e+07 +test_resetting_timer {quantile="0.75"} 4.05e+07 test_resetting_timer {quantile="0.95"} 1.2e+08 test_resetting_timer {quantile="0.99"} 1.2e+08 +test_resetting_timer {quantile="0.999"} 1.2e+08 +test_resetting_timer {quantile="0.9999"} 1.2e+08 # TYPE test_timer_count counter test_timer_count 6 diff --git a/metrics/sample.go b/metrics/sample.go index 5398dd42d5..bb81e105cf 100644 --- a/metrics/sample.go +++ b/metrics/sample.go @@ -148,7 +148,7 @@ func (NilSample) Clear() {} func (NilSample) Snapshot() SampleSnapshot { return (*emptySnapshot)(nil) } func (NilSample) Update(v int64) {} -// SamplePercentiles returns an arbitrary percentile of the slice of int64. +// SamplePercentile returns an arbitrary percentile of the slice of int64. func SamplePercentile(values []int64, p float64) float64 { return CalculatePercentiles(values, []float64{p})[0] } diff --git a/miner/miner.go b/miner/miner.go index b7273948f5..430efcb2fc 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -30,9 +30,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) @@ -45,202 +42,124 @@ type Backend interface { // Config is the configuration parameters of mining. type Config struct { - Etherbase common.Address `toml:",omitempty"` // Public address for block mining rewards - ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner - GasFloor uint64 // Target gas floor for mined blocks. - GasCeil uint64 // Target gas ceiling for mined blocks. - GasPrice *big.Int // Minimum gas price for mining a transaction - Recommit time.Duration // The time interval for miner to re-create mining work. - - NewPayloadTimeout time.Duration // The maximum time allowance for creating a new payload + Etherbase common.Address `toml:"-"` // Deprecated + PendingFeeRecipient common.Address `toml:"-"` // Address for pending block rewards. + ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner + GasCeil uint64 // Target gas ceiling for mined blocks. + GasPrice *big.Int // Minimum gas price for mining a transaction + Recommit time.Duration // The time interval for miner to re-create mining work. } // DefaultConfig contains default settings for miner. var DefaultConfig = Config{ - GasCeil: 30000000, + GasCeil: 30_000_000, GasPrice: big.NewInt(params.GWei), // The default recommit time is chosen as two seconds since // consensus-layer usually will wait a half slot of time(6s) // for payload generation. It should be enough for Geth to // run 3 rounds. - Recommit: 2 * time.Second, - NewPayloadTimeout: 2 * time.Second, + Recommit: 2 * time.Second, } -// Miner creates blocks and searches for proof-of-work values. +// Miner is the main object which takes care of submitting new work to consensus +// engine and gathering the sealing result. type Miner struct { - mux *event.TypeMux - eth Backend - engine consensus.Engine - exitCh chan struct{} - startCh chan struct{} - stopCh chan struct{} - worker *worker - - wg sync.WaitGroup + confMu sync.RWMutex // The lock used to protect the config fields: GasCeil, GasTip and Extradata + config *Config + chainConfig *params.ChainConfig + engine consensus.Engine + txpool *txpool.TxPool + chain *core.BlockChain + pending *pending + pendingMu sync.Mutex // Lock protects the pending block } -func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(header *types.Header) bool) *Miner { - miner := &Miner{ - mux: mux, - eth: eth, - engine: engine, - exitCh: make(chan struct{}), - startCh: make(chan struct{}), - stopCh: make(chan struct{}), - worker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true), - } - miner.wg.Add(1) - go miner.update() - return miner -} - -// update keeps track of the downloader events. Please be aware that this is a one shot type of update loop. -// It's entered once and as soon as `Done` or `Failed` has been broadcasted the events are unregistered and -// the loop is exited. This to prevent a major security vuln where external parties can DOS you with blocks -// and halt your mining operation for as long as the DOS continues. -func (miner *Miner) update() { - defer miner.wg.Done() - - events := miner.mux.Subscribe(downloader.StartEvent{}, downloader.DoneEvent{}, downloader.FailedEvent{}) - defer func() { - if !events.Closed() { - events.Unsubscribe() - } - }() - - shouldStart := false - canStart := true - dlEventCh := events.Chan() - for { - select { - case ev := <-dlEventCh: - if ev == nil { - // Unsubscription done, stop listening - dlEventCh = nil - continue - } - switch ev.Data.(type) { - case downloader.StartEvent: - wasMining := miner.Mining() - miner.worker.stop() - canStart = false - if wasMining { - // Resume mining after sync was finished - shouldStart = true - log.Info("Mining aborted due to sync") - } - miner.worker.syncing.Store(true) - - case downloader.FailedEvent: - canStart = true - if shouldStart { - miner.worker.start() - } - miner.worker.syncing.Store(false) - - case downloader.DoneEvent: - canStart = true - if shouldStart { - miner.worker.start() - } - miner.worker.syncing.Store(false) - - // Stop reacting to downloader events - events.Unsubscribe() - } - case <-miner.startCh: - if canStart { - miner.worker.start() - } - shouldStart = true - case <-miner.stopCh: - shouldStart = false - miner.worker.stop() - case <-miner.exitCh: - miner.worker.close() - return - } +// New creates a new miner with provided config. +func New(eth Backend, config Config, engine consensus.Engine) *Miner { + return &Miner{ + config: &config, + chainConfig: eth.BlockChain().Config(), + engine: engine, + txpool: eth.TxPool(), + chain: eth.BlockChain(), + pending: &pending{}, } } -func (miner *Miner) Start() { - miner.startCh <- struct{}{} -} - -func (miner *Miner) Stop() { - miner.stopCh <- struct{}{} -} - -func (miner *Miner) Close() { - close(miner.exitCh) - miner.wg.Wait() -} - -func (miner *Miner) Mining() bool { - return miner.worker.isRunning() -} - -func (miner *Miner) Hashrate() uint64 { - if pow, ok := miner.engine.(consensus.PoW); ok { - return uint64(pow.Hashrate()) +// Pending returns the currently pending block and associated receipts, logs +// and statedb. The returned values can be nil in case the pending block is +// not initialized. +func (miner *Miner) Pending() (*types.Block, types.Receipts, *state.StateDB) { + pending := miner.getPending() + if pending == nil { + return nil, nil, nil } - return 0 + return pending.block, pending.receipts, pending.stateDB.Copy() } +// SetExtra sets the content used to initialize the block extra field. func (miner *Miner) SetExtra(extra []byte) error { if uint64(len(extra)) > params.MaximumExtraDataSize { return fmt.Errorf("extra exceeds max length. %d > %v", len(extra), params.MaximumExtraDataSize) } - miner.worker.setExtra(extra) + miner.confMu.Lock() + miner.config.ExtraData = extra + miner.confMu.Unlock() return nil } -// SetRecommitInterval sets the interval for sealing work resubmitting. -func (miner *Miner) SetRecommitInterval(interval time.Duration) { - miner.worker.setRecommitInterval(interval) -} - -// Pending returns the currently pending block and associated state. The returned -// values can be nil in case the pending block is not initialized -func (miner *Miner) Pending() (*types.Block, *state.StateDB) { - return miner.worker.pending() -} - -// PendingBlock returns the currently pending block. The returned block can be -// nil in case the pending block is not initialized. -// -// Note, to access both the pending block and the pending state -// simultaneously, please use Pending(), as the pending state can -// change between multiple method calls -func (miner *Miner) PendingBlock() *types.Block { - return miner.worker.pendingBlock() -} - -// PendingBlockAndReceipts returns the currently pending block and corresponding receipts. -// The returned values can be nil in case the pending block is not initialized. -func (miner *Miner) PendingBlockAndReceipts() (*types.Block, types.Receipts) { - return miner.worker.pendingBlockAndReceipts() -} - -func (miner *Miner) SetEtherbase(addr common.Address) { - miner.worker.setEtherbase(addr) -} - // SetGasCeil sets the gaslimit to strive for when mining blocks post 1559. // For pre-1559 blocks, it sets the ceiling. func (miner *Miner) SetGasCeil(ceil uint64) { - miner.worker.setGasCeil(ceil) + miner.confMu.Lock() + miner.config.GasCeil = ceil + miner.confMu.Unlock() } -// SubscribePendingLogs starts delivering logs from pending transactions -// to the given channel. -func (miner *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription { - return miner.worker.pendingLogsFeed.Subscribe(ch) +// SetGasTip sets the minimum gas tip for inclusion. +func (miner *Miner) SetGasTip(tip *big.Int) error { + miner.confMu.Lock() + miner.config.GasPrice = tip + miner.confMu.Unlock() + return nil } // BuildPayload builds the payload according to the provided parameters. func (miner *Miner) BuildPayload(args *BuildPayloadArgs) (*Payload, error) { - return miner.worker.buildPayload(args) + return miner.buildPayload(args) +} + +// getPending retrieves the pending block based on the current head block. +// The result might be nil if pending generation is failed. +func (miner *Miner) getPending() *newPayloadResult { + header := miner.chain.CurrentHeader() + miner.pendingMu.Lock() + defer miner.pendingMu.Unlock() + if cached := miner.pending.resolve(header.Hash()); cached != nil { + return cached + } + + var ( + timestamp = uint64(time.Now().Unix()) + withdrawal types.Withdrawals + ) + if miner.chainConfig.IsShanghai(new(big.Int).Add(header.Number, big.NewInt(1)), timestamp) { + withdrawal = []*types.Withdrawal{} + } + ret := miner.generateWork(&generateParams{ + timestamp: timestamp, + forceTime: false, + parentHash: header.Hash(), + coinbase: miner.config.PendingFeeRecipient, + random: common.Hash{}, + withdrawals: withdrawal, + beaconRoot: nil, + noTxs: false, + }) + if ret.err != nil { + return nil + } + miner.pending.update(header.Hash(), ret) + return ret } diff --git a/miner/miner_test.go b/miner/miner_test.go index 411d6026ce..7c39564240 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -18,10 +18,9 @@ package miner import ( - "errors" "math/big" + "sync" "testing" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/clique" @@ -33,10 +32,10 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) type mockBackend struct { @@ -59,10 +58,6 @@ func (m *mockBackend) TxPool() *txpool.TxPool { return m.txPool } -func (m *mockBackend) StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error) { - return nil, errors.New("not supported") -} - type testBlockChain struct { root common.Hash config *params.ChainConfig @@ -98,171 +93,18 @@ func (bc *testBlockChain) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) return bc.chainHeadFeed.Subscribe(ch) } -func TestMiner(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - // Subsequent downloader events after a successful DoneEvent should not cause the - // miner to start or stop. This prevents a security vulnerability - // that would allow entities to present fake high blocks that would - // stop mining operations by causing a downloader sync - // until it was discovered they were invalid, whereon mining would resume. - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, true) - - mux.Post(downloader.FailedEvent{}) - waitForMiningState(t, miner, true) -} - -// TestMinerDownloaderFirstFails tests that mining is only -// permitted to run indefinitely once the downloader sees a DoneEvent (success). -// An initial FailedEvent should allow mining to stop on a subsequent -// downloader StartEvent. -func TestMinerDownloaderFirstFails(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.FailedEvent{}) - waitForMiningState(t, miner, true) - - // Since the downloader hasn't yet emitted a successful DoneEvent, - // we expect the miner to stop on next StartEvent. - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - - // Downloader finally succeeds. - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - // Downloader starts again. - // Since it has achieved a DoneEvent once, we expect miner - // state to be unchanged. - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, true) - - mux.Post(downloader.FailedEvent{}) - waitForMiningState(t, miner, true) -} - -func TestMinerStartStopAfterDownloaderEvents(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - - // Downloader finally succeeds. - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - miner.Stop() - waitForMiningState(t, miner, false) - - miner.Start() - waitForMiningState(t, miner, true) - - miner.Stop() - waitForMiningState(t, miner, false) -} - -func TestStartWhileDownload(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - waitForMiningState(t, miner, false) - miner.Start() - waitForMiningState(t, miner, true) - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - // Starting the miner after the downloader should not work - miner.Start() - waitForMiningState(t, miner, false) -} - -func TestStartStopMiner(t *testing.T) { - t.Parallel() - miner, _, cleanup := createMiner(t) - defer cleanup(false) - waitForMiningState(t, miner, false) - miner.Start() - waitForMiningState(t, miner, true) - miner.Stop() - waitForMiningState(t, miner, false) -} - -func TestCloseMiner(t *testing.T) { - t.Parallel() - miner, _, cleanup := createMiner(t) - defer cleanup(true) - waitForMiningState(t, miner, false) - miner.Start() - waitForMiningState(t, miner, true) - // Terminate the miner and wait for the update loop to run - miner.Close() - waitForMiningState(t, miner, false) -} - -// TestMinerSetEtherbase checks that etherbase becomes set even if mining isn't -// possible at the moment -func TestMinerSetEtherbase(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - // Now user tries to configure proper mining address - miner.Start() - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - coinbase := common.HexToAddress("0xdeedbeef") - miner.SetEtherbase(coinbase) - if addr := miner.worker.etherbase(); addr != coinbase { - t.Fatalf("Unexpected etherbase want %x got %x", coinbase, addr) - } -} - -// waitForMiningState waits until either -// * the desired mining state was reached -// * a timeout was reached which fails the test -func waitForMiningState(t *testing.T, m *Miner, mining bool) { - t.Helper() - - var state bool - for i := 0; i < 100; i++ { - time.Sleep(10 * time.Millisecond) - if state = m.Mining(); state == mining { - return +func TestBuildPendingBlocks(t *testing.T) { + miner := createMiner(t) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + block, _, _ := miner.Pending() + if block == nil { + t.Error("Pending failed") } - } - t.Fatalf("Mining() == %t, want %t", state, mining) + }() + wg.Wait() } func minerTestGenesisBlock(period uint64, gasLimit uint64, faucet common.Address) *core.Genesis { @@ -279,7 +121,7 @@ func minerTestGenesisBlock(period uint64, gasLimit uint64, faucet common.Address GasLimit: gasLimit, BaseFee: big.NewInt(params.InitialBaseFee), Difficulty: big.NewInt(1), - Alloc: map[common.Address]core.GenesisAccount{ + Alloc: map[common.Address]types.Account{ common.BytesToAddress([]byte{1}): {Balance: big.NewInt(1)}, // ECRecover common.BytesToAddress([]byte{2}): {Balance: big.NewInt(1)}, // SHA256 common.BytesToAddress([]byte{3}): {Balance: big.NewInt(1)}, // RIPEMD @@ -293,14 +135,15 @@ func minerTestGenesisBlock(period uint64, gasLimit uint64, faucet common.Address }, } } -func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { + +func createMiner(t *testing.T) *Miner { // Create Ethash config config := Config{ - Etherbase: common.HexToAddress("123456789"), + PendingFeeRecipient: common.HexToAddress("123456789"), } // Create chainConfig chainDB := rawdb.NewMemoryDatabase() - triedb := trie.NewDatabase(chainDB, nil) + triedb := triedb.NewDatabase(chainDB, nil) genesis := minerTestGenesisBlock(15, 11_500_000, common.HexToAddress("12345")) chainConfig, _, err := core.SetupGenesisBlock(chainDB, triedb, genesis) if err != nil { @@ -317,20 +160,10 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { blockchain := &testBlockChain{bc.Genesis().Root(), chainConfig, statedb, 10000000, new(event.Feed)} pool := legacypool.New(testTxPoolConfig, blockchain) - txpool, _ := txpool.New(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain, []txpool.SubPool{pool}) + txpool, _ := txpool.New(testTxPoolConfig.PriceLimit, blockchain, []txpool.SubPool{pool}) - backend := NewMockBackend(bc, txpool) - // Create event Mux - mux := new(event.TypeMux) // Create Miner - miner := New(backend, &config, chainConfig, mux, engine, nil) - cleanup := func(skipMiner bool) { - bc.Stop() - engine.Close() - txpool.Close() - if !skipMiner { - miner.Close() - } - } - return miner, mux, cleanup + backend := NewMockBackend(bc, txpool) + miner := New(backend, config, engine) + return miner } diff --git a/miner/ordering.go b/miner/ordering.go index 4c3055f0d3..bcf7af46e8 100644 --- a/miner/ordering.go +++ b/miner/ordering.go @@ -21,28 +21,31 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/holiman/uint256" ) // txWithMinerFee wraps a transaction with its gas price or effective miner gasTipCap type txWithMinerFee struct { tx *txpool.LazyTransaction from common.Address - fees *big.Int + fees *uint256.Int } // newTxWithMinerFee creates a wrapped transaction, calculating the effective // miner gasTipCap if a base fee is provided. // Returns error in case of a negative effective miner gasTipCap. -func newTxWithMinerFee(tx *txpool.LazyTransaction, from common.Address, baseFee *big.Int) (*txWithMinerFee, error) { - tip := new(big.Int).Set(tx.GasTipCap) +func newTxWithMinerFee(tx *txpool.LazyTransaction, from common.Address, baseFee *uint256.Int) (*txWithMinerFee, error) { + tip := new(uint256.Int).Set(tx.GasTipCap) if baseFee != nil { if tx.GasFeeCap.Cmp(baseFee) < 0 { return nil, types.ErrGasFeeCapTooLow } - tip = math.BigMin(tx.GasTipCap, new(big.Int).Sub(tx.GasFeeCap, baseFee)) + tip = new(uint256.Int).Sub(tx.GasFeeCap, baseFee) + if tip.Gt(tx.GasTipCap) { + tip = tx.GasTipCap + } } return &txWithMinerFee{ tx: tx, @@ -87,7 +90,7 @@ type transactionsByPriceAndNonce struct { txs map[common.Address][]*txpool.LazyTransaction // Per account nonce-sorted list of transactions heads txByPriceAndTime // Next transaction for each unique account (price heap) signer types.Signer // Signer for the set of transactions - baseFee *big.Int // Current base fee + baseFee *uint256.Int // Current base fee } // newTransactionsByPriceAndNonce creates a transaction set that can retrieve @@ -96,10 +99,15 @@ type transactionsByPriceAndNonce struct { // Note, the input map is reowned so the caller should not interact any more with // if after providing it to the constructor. func newTransactionsByPriceAndNonce(signer types.Signer, txs map[common.Address][]*txpool.LazyTransaction, baseFee *big.Int) *transactionsByPriceAndNonce { + // Convert the basefee from header format to uint256 format + var baseFeeUint *uint256.Int + if baseFee != nil { + baseFeeUint = uint256.MustFromBig(baseFee) + } // Initialize a price and received time based heap with the head transactions heads := make(txByPriceAndTime, 0, len(txs)) for from, accTxs := range txs { - wrapped, err := newTxWithMinerFee(accTxs[0], from, baseFee) + wrapped, err := newTxWithMinerFee(accTxs[0], from, baseFeeUint) if err != nil { delete(txs, from) continue @@ -114,16 +122,16 @@ func newTransactionsByPriceAndNonce(signer types.Signer, txs map[common.Address] txs: txs, heads: heads, signer: signer, - baseFee: baseFee, + baseFee: baseFeeUint, } } // Peek returns the next transaction by price. -func (t *transactionsByPriceAndNonce) Peek() *txpool.LazyTransaction { +func (t *transactionsByPriceAndNonce) Peek() (*txpool.LazyTransaction, *uint256.Int) { if len(t.heads) == 0 { - return nil + return nil, nil } - return t.heads[0].tx + return t.heads[0].tx, t.heads[0].fees } // Shift replaces the current best head with the next one from the same account. @@ -145,3 +153,14 @@ func (t *transactionsByPriceAndNonce) Shift() { func (t *transactionsByPriceAndNonce) Pop() { heap.Pop(&t.heads) } + +// Empty returns if the price heap is empty. It can be used to check it simpler +// than calling peek and checking for nil return. +func (t *transactionsByPriceAndNonce) Empty() bool { + return len(t.heads) == 0 +} + +// Clear removes the entire content of the heap. +func (t *transactionsByPriceAndNonce) Clear() { + t.heads, t.txs = nil, nil +} diff --git a/miner/ordering_test.go b/miner/ordering_test.go index e5868d7a06..3587a835c8 100644 --- a/miner/ordering_test.go +++ b/miner/ordering_test.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/holiman/uint256" ) func TestTransactionPriceNonceSortLegacy(t *testing.T) { @@ -92,8 +93,8 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) @@ -104,7 +105,7 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { txset := newTransactionsByPriceAndNonce(signer, groups, baseFee) txs := types.Transactions{} - for tx := txset.Peek(); tx != nil; tx = txset.Peek() { + for tx, _ := txset.Peek(); tx != nil; tx, _ = txset.Peek() { txs = append(txs, tx.Tx) txset.Shift() } @@ -160,8 +161,8 @@ func TestTransactionTimeSort(t *testing.T) { Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) @@ -170,7 +171,7 @@ func TestTransactionTimeSort(t *testing.T) { txset := newTransactionsByPriceAndNonce(signer, groups, nil) txs := types.Transactions{} - for tx := txset.Peek(); tx != nil; tx = txset.Peek() { + for tx, _ := txset.Peek(); tx != nil; tx, _ = txset.Peek() { txs = append(txs, tx.Tx) txset.Shift() } diff --git a/miner/payload_building.go b/miner/payload_building.go index 719736c479..d027cd1e1f 100644 --- a/miner/payload_building.go +++ b/miner/payload_building.go @@ -46,7 +46,6 @@ type BuildPayloadArgs struct { // Id computes an 8-byte identifier by hashing the components of the payload arguments. func (args *BuildPayloadArgs) Id() engine.PayloadID { - // Hash hasher := sha256.New() hasher.Write(args.Parent[:]) binary.Write(hasher, binary.BigEndian, args.Timestamp) @@ -177,7 +176,7 @@ func (payload *Payload) ResolveFull() *engine.ExecutionPayloadEnvelope { } // buildPayload builds the payload according to the provided parameters. -func (w *worker) buildPayload(args *BuildPayloadArgs) (*Payload, error) { +func (miner *Miner) buildPayload(args *BuildPayloadArgs) (*Payload, error) { // Build the initial version with no transaction included. It should be fast // enough to run. The empty payload can at least make sure there is something // to deliver for not missing slot. @@ -191,7 +190,7 @@ func (w *worker) buildPayload(args *BuildPayloadArgs) (*Payload, error) { beaconRoot: args.BeaconRoot, noTxs: true, } - empty := w.getSealingBlock(emptyParams) + empty := miner.generateWork(emptyParams) if empty.err != nil { return nil, empty.err } @@ -227,11 +226,13 @@ func (w *worker) buildPayload(args *BuildPayloadArgs) (*Payload, error) { select { case <-timer.C: start := time.Now() - r := w.getSealingBlock(fullParams) + r := miner.generateWork(fullParams) if r.err == nil { payload.update(r, time.Since(start)) + } else { + log.Info("Error while generating work", "id", payload.id, "err", r.err) } - timer.Reset(w.recommit) + timer.Reset(miner.config.Recommit) case <-payload.stop: log.Info("Stopping work on payload", "id", payload.id, "reason", "delivery") return diff --git a/miner/payload_building_test.go b/miner/payload_building_test.go index 708072b5ec..1728b9e5bd 100644 --- a/miner/payload_building_test.go +++ b/miner/payload_building_test.go @@ -17,26 +17,141 @@ package miner import ( + "math/big" "reflect" "testing" "time" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) +var ( + // Test chain configurations + testTxPoolConfig legacypool.Config + ethashChainConfig *params.ChainConfig + cliqueChainConfig *params.ChainConfig + + // Test accounts + testBankKey, _ = crypto.GenerateKey() + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) + testBankFunds = big.NewInt(1000000000000000000) + + testUserKey, _ = crypto.GenerateKey() + testUserAddress = crypto.PubkeyToAddress(testUserKey.PublicKey) + + // Test transactions + pendingTxs []*types.Transaction + newTxs []*types.Transaction + + testConfig = Config{ + PendingFeeRecipient: testBankAddress, + Recommit: time.Second, + GasCeil: params.GenesisGasLimit, + } +) + +func init() { + testTxPoolConfig = legacypool.DefaultConfig + testTxPoolConfig.Journal = "" + ethashChainConfig = new(params.ChainConfig) + *ethashChainConfig = *params.TestChainConfig + cliqueChainConfig = new(params.ChainConfig) + *cliqueChainConfig = *params.TestChainConfig + cliqueChainConfig.Clique = ¶ms.CliqueConfig{ + Period: 10, + Epoch: 30000, + } + + signer := types.LatestSigner(params.TestChainConfig) + tx1 := types.MustSignNewTx(testBankKey, signer, &types.AccessListTx{ + ChainID: params.TestChainConfig.ChainID, + Nonce: 0, + To: &testUserAddress, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: big.NewInt(params.InitialBaseFee), + }) + pendingTxs = append(pendingTxs, tx1) + + tx2 := types.MustSignNewTx(testBankKey, signer, &types.LegacyTx{ + Nonce: 1, + To: &testUserAddress, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: big.NewInt(params.InitialBaseFee), + }) + newTxs = append(newTxs, tx2) +} + +// testWorkerBackend implements worker.Backend interfaces and wraps all information needed during the testing. +type testWorkerBackend struct { + db ethdb.Database + txPool *txpool.TxPool + chain *core.BlockChain + genesis *core.Genesis +} + +func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, n int) *testWorkerBackend { + var gspec = &core.Genesis{ + Config: chainConfig, + Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + } + switch e := engine.(type) { + case *clique.Clique: + gspec.ExtraData = make([]byte, 32+common.AddressLength+crypto.SignatureLength) + copy(gspec.ExtraData[32:32+common.AddressLength], testBankAddress.Bytes()) + e.Authorize(testBankAddress, func(account accounts.Account, s string, data []byte) ([]byte, error) { + return crypto.Sign(crypto.Keccak256(data), testBankKey) + }) + case *ethash.Ethash: + default: + t.Fatalf("unexpected consensus engine type: %T", engine) + } + chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec, nil, engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("core.NewBlockChain failed: %v", err) + } + pool := legacypool.New(testTxPoolConfig, chain) + txpool, _ := txpool.New(testTxPoolConfig.PriceLimit, chain, []txpool.SubPool{pool}) + + return &testWorkerBackend{ + db: db, + chain: chain, + txPool: txpool, + genesis: gspec, + } +} + +func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain } +func (b *testWorkerBackend) TxPool() *txpool.TxPool { return b.txPool } + +func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*Miner, *testWorkerBackend) { + backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) + backend.txPool.Add(pendingTxs, true, false) + w := New(backend, testConfig, engine) + return w, backend +} + func TestBuildPayload(t *testing.T) { - t.Parallel() var ( db = rawdb.NewMemoryDatabase() recipient = common.HexToAddress("0xdeadbeef") ) w, b := newTestWorker(t, params.TestChainConfig, ethash.NewFaker(), db, 0) - defer w.close() timestamp := uint64(time.Now().Unix()) args := &BuildPayloadArgs{ diff --git a/miner/pending.go b/miner/pending.go new file mode 100644 index 0000000000..bb91fe8969 --- /dev/null +++ b/miner/pending.go @@ -0,0 +1,67 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package miner + +import ( + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" +) + +// pendingTTL indicates the period of time a generated pending block should +// exist to serve RPC requests before being discarded if the parent block +// has not changed yet. The value is chosen to align with the recommit interval. +const pendingTTL = 2 * time.Second + +// pending wraps a pending block with additional metadata. +type pending struct { + created time.Time + parentHash common.Hash + result *newPayloadResult + lock sync.Mutex +} + +// resolve retrieves the cached pending result if it's available. Nothing will be +// returned if the parentHash is not matched or the result is already too old. +// +// Note, don't modify the returned payload result. +func (p *pending) resolve(parentHash common.Hash) *newPayloadResult { + p.lock.Lock() + defer p.lock.Unlock() + + if p.result == nil { + return nil + } + if parentHash != p.parentHash { + return nil + } + if time.Since(p.created) > pendingTTL { + return nil + } + return p.result +} + +// update refreshes the cached pending block with newly created one. +func (p *pending) update(parent common.Hash, result *newPayloadResult) { + p.lock.Lock() + defer p.lock.Unlock() + + p.parentHash = parent + p.result = result + p.created = time.Now() +} diff --git a/miner/stress/clique/main.go b/miner/stress/clique/main.go deleted file mode 100644 index 13336cd83c..0000000000 --- a/miner/stress/clique/main.go +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// This file contains a miner stress test based on the Clique consensus engine. -package main - -import ( - "bytes" - "crypto/ecdsa" - "math/big" - "math/rand" - "os" - "os/signal" - "time" - - "github.com/ethereum/go-ethereum/accounts/keystore" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/fdlimit" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/txpool/legacypool" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/miner" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/params" -) - -func main() { - log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) - fdlimit.Raise(2048) - - // Generate a batch of accounts to seal and fund with - faucets := make([]*ecdsa.PrivateKey, 128) - for i := 0; i < len(faucets); i++ { - faucets[i], _ = crypto.GenerateKey() - } - sealers := make([]*ecdsa.PrivateKey, 4) - for i := 0; i < len(sealers); i++ { - sealers[i], _ = crypto.GenerateKey() - } - // Create a Clique network based off of the Sepolia config - genesis := makeGenesis(faucets, sealers) - - // Handle interrupts. - interruptCh := make(chan os.Signal, 5) - signal.Notify(interruptCh, os.Interrupt) - - var ( - stacks []*node.Node - nodes []*eth.Ethereum - enodes []*enode.Node - ) - for _, sealer := range sealers { - // Start the node and wait until it's up - stack, ethBackend, err := makeSealer(genesis) - if err != nil { - panic(err) - } - defer stack.Close() - - for stack.Server().NodeInfo().Ports.Listener == 0 { - time.Sleep(250 * time.Millisecond) - } - // Connect the node to all the previous ones - for _, n := range enodes { - stack.Server().AddPeer(n) - } - // Start tracking the node and its enode - stacks = append(stacks, stack) - nodes = append(nodes, ethBackend) - enodes = append(enodes, stack.Server().Self()) - - // Inject the signer key and start sealing with it - ks := keystore.NewKeyStore(stack.KeyStoreDir(), keystore.LightScryptN, keystore.LightScryptP) - signer, err := ks.ImportECDSA(sealer, "") - if err != nil { - panic(err) - } - if err := ks.Unlock(signer, ""); err != nil { - panic(err) - } - stack.AccountManager().AddBackend(ks) - } - - // Iterate over all the nodes and start signing on them - time.Sleep(3 * time.Second) - for _, node := range nodes { - if err := node.StartMining(); err != nil { - panic(err) - } - } - time.Sleep(3 * time.Second) - - // Start injecting transactions from the faucet like crazy - nonces := make([]uint64, len(faucets)) - for { - // Stop when interrupted. - select { - case <-interruptCh: - for _, node := range stacks { - node.Close() - } - return - default: - } - - // Pick a random signer node - index := rand.Intn(len(faucets)) - backend := nodes[index%len(nodes)] - - // Create a self transaction and inject into the pool - tx, err := types.SignTx(types.NewTransaction(nonces[index], crypto.PubkeyToAddress(faucets[index].PublicKey), new(big.Int), 21000, big.NewInt(100000000000), nil), types.HomesteadSigner{}, faucets[index]) - if err != nil { - panic(err) - } - if err := backend.TxPool().Add([]*types.Transaction{tx}, true, false); err != nil { - panic(err) - } - nonces[index]++ - - // Wait if we're too saturated - if pend, _ := backend.TxPool().Stats(); pend > 2048 { - time.Sleep(100 * time.Millisecond) - } - } -} - -// makeGenesis creates a custom Clique genesis block based on some pre-defined -// signer and faucet accounts. -func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core.Genesis { - // Create a Clique network based off of the Sepolia config - genesis := core.DefaultSepoliaGenesisBlock() - genesis.GasLimit = 25000000 - - genesis.Config.ChainID = big.NewInt(18) - genesis.Config.Clique.Period = 1 - - genesis.Alloc = core.GenesisAlloc{} - for _, faucet := range faucets { - genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = core.GenesisAccount{ - Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), - } - } - // Sort the signers and embed into the extra-data section - signers := make([]common.Address, len(sealers)) - for i, sealer := range sealers { - signers[i] = crypto.PubkeyToAddress(sealer.PublicKey) - } - for i := 0; i < len(signers); i++ { - for j := i + 1; j < len(signers); j++ { - if bytes.Compare(signers[i][:], signers[j][:]) > 0 { - signers[i], signers[j] = signers[j], signers[i] - } - } - } - genesis.ExtraData = make([]byte, 32+len(signers)*common.AddressLength+65) - for i, signer := range signers { - copy(genesis.ExtraData[32+i*common.AddressLength:], signer[:]) - } - // Return the genesis block for initialization - return genesis -} - -func makeSealer(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { - // Define the basic configurations for the Ethereum node - datadir, _ := os.MkdirTemp("", "") - - config := &node.Config{ - Name: "geth", - Version: params.Version, - DataDir: datadir, - P2P: p2p.Config{ - ListenAddr: "0.0.0.0:0", - NoDiscovery: true, - MaxPeers: 25, - }, - } - // Start the node and configure a full Ethereum node on it - stack, err := node.New(config) - if err != nil { - return nil, nil, err - } - // Create and register the backend - ethBackend, err := eth.New(stack, ðconfig.Config{ - Genesis: genesis, - NetworkId: genesis.Config.ChainID.Uint64(), - SyncMode: downloader.FullSync, - DatabaseCache: 256, - DatabaseHandles: 256, - TxPool: legacypool.DefaultConfig, - GPO: ethconfig.Defaults.GPO, - Miner: miner.Config{ - GasCeil: genesis.GasLimit * 11 / 10, - GasPrice: big.NewInt(1), - Recommit: time.Second, - }, - }) - if err != nil { - return nil, nil, err - } - - err = stack.Start() - return stack, ethBackend, err -} diff --git a/miner/worker.go b/miner/worker.go index 38673e6025..8391cd14c6 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -20,12 +20,10 @@ import ( "errors" "fmt" "math/big" - "sync" "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" @@ -33,44 +31,9 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" -) - -const ( - // resultQueueSize is the size of channel listening to sealing result. - resultQueueSize = 10 - - // txChanSize is the size of channel listening to NewTxsEvent. - // The number is referenced from the size of tx pool. - txChanSize = 4096 - - // chainHeadChanSize is the size of channel listening to ChainHeadEvent. - chainHeadChanSize = 10 - - // resubmitAdjustChanSize is the size of resubmitting interval adjustment channel. - resubmitAdjustChanSize = 10 - - // minRecommitInterval is the minimal time interval to recreate the sealing block with - // any newly arrived transactions. - minRecommitInterval = 1 * time.Second - - // maxRecommitInterval is the maximum time interval to recreate the sealing block with - // any newly arrived transactions. - maxRecommitInterval = 15 * time.Second - - // intervalAdjustRatio is the impact a single interval adjustment has on sealing work - // resubmitting interval. - intervalAdjustRatio = 0.1 - - // intervalAdjustBias is applied during the new resubmit interval calculation in favor of - // increasing upper limit or decreasing lower limit so that the limit can be reachable. - intervalAdjustBias = 200 * 1000.0 * 1000.0 - - // staleThreshold is the maximum depth of the acceptable stale block. - staleThreshold = 7 + "github.com/holiman/uint256" ) var ( @@ -95,47 +58,6 @@ type environment struct { blobs int } -// copy creates a deep copy of environment. -func (env *environment) copy() *environment { - cpy := &environment{ - signer: env.signer, - state: env.state.Copy(), - tcount: env.tcount, - coinbase: env.coinbase, - header: types.CopyHeader(env.header), - receipts: copyReceipts(env.receipts), - } - if env.gasPool != nil { - gasPool := *env.gasPool - cpy.gasPool = &gasPool - } - cpy.txs = make([]*types.Transaction, len(env.txs)) - copy(cpy.txs, env.txs) - - cpy.sidecars = make([]*types.BlobTxSidecar, len(env.sidecars)) - copy(cpy.sidecars, env.sidecars) - - return cpy -} - -// discard terminates the background prefetcher go-routine. It should -// always be called for all created environment instances otherwise -// the go-routine leak can happen. -func (env *environment) discard() { - if env.state == nil { - return - } - env.state.StopPrefetcher() -} - -// task contains all information for consensus engine sealing and result submitting. -type task struct { - receipts []*types.Receipt - state *state.StateDB - block *types.Block - createdAt time.Time -} - const ( commitInterruptNone int32 = iota commitInterruptNewHead @@ -143,618 +65,174 @@ const ( commitInterruptTimeout ) -// newWorkReq represents a request for new sealing work submitting with relative interrupt notifier. -type newWorkReq struct { - interrupt *atomic.Int32 - timestamp int64 -} - // newPayloadResult is the result of payload generation. type newPayloadResult struct { err error block *types.Block fees *big.Int // total block fees sidecars []*types.BlobTxSidecar // collected blobs of blob transactions + stateDB *state.StateDB // StateDB after executing the transactions + receipts []*types.Receipt // Receipts collected during construction } -// getWorkReq represents a request for getting a new sealing work with provided parameters. -type getWorkReq struct { - params *generateParams - result chan *newPayloadResult // non-blocking channel +// generateParams wraps various of settings for generating sealing task. +type generateParams struct { + timestamp uint64 // The timestamp for sealing task + forceTime bool // Flag whether the given timestamp is immutable or not + parentHash common.Hash // Parent block hash, empty means the latest chain head + coinbase common.Address // The fee recipient address for including transaction + random common.Hash // The randomness generated by beacon chain, empty before the merge + withdrawals types.Withdrawals // List of withdrawals to include in block (shanghai field) + beaconRoot *common.Hash // The beacon root (cancun field). + noTxs bool // Flag whether an empty block without any transaction is expected } -// intervalAdjust represents a resubmitting interval adjustment. -type intervalAdjust struct { - ratio float64 - inc bool -} - -// worker is the main object which takes care of submitting new work to consensus engine -// and gathering the sealing result. -type worker struct { - config *Config - chainConfig *params.ChainConfig - engine consensus.Engine - eth Backend - chain *core.BlockChain - - // Feeds - pendingLogsFeed event.Feed - - // Subscriptions - mux *event.TypeMux - txsCh chan core.NewTxsEvent - txsSub event.Subscription - chainHeadCh chan core.ChainHeadEvent - chainHeadSub event.Subscription - - // Channels - newWorkCh chan *newWorkReq - getWorkCh chan *getWorkReq - taskCh chan *task - resultCh chan *types.Block - startCh chan struct{} - exitCh chan struct{} - resubmitIntervalCh chan time.Duration - resubmitAdjustCh chan *intervalAdjust - - wg sync.WaitGroup - - current *environment // An environment for current running cycle. - - mu sync.RWMutex // The lock used to protect the coinbase and extra fields - coinbase common.Address - extra []byte - - pendingMu sync.RWMutex - pendingTasks map[common.Hash]*task - - snapshotMu sync.RWMutex // The lock used to protect the snapshots below - snapshotBlock *types.Block - snapshotReceipts types.Receipts - snapshotState *state.StateDB - - // atomic status counters - running atomic.Bool // The indicator whether the consensus engine is running or not. - newTxs atomic.Int32 // New arrival transaction count since last sealing work submitting. - syncing atomic.Bool // The indicator whether the node is still syncing. - - // newpayloadTimeout is the maximum timeout allowance for creating payload. - // The default value is 2 seconds but node operator can set it to arbitrary - // large value. A large timeout allowance may cause Geth to fail creating - // a non-empty payload within the specified time and eventually miss the slot - // in case there are some computation expensive transactions in txpool. - newpayloadTimeout time.Duration - - // recommit is the time interval to re-create sealing work or to re-build - // payload in proof-of-stake stage. - recommit time.Duration - - // External functions - isLocalBlock func(header *types.Header) bool // Function used to determine whether the specified block is mined by local miner. - - // Test hooks - newTaskHook func(*task) // Method to call upon receiving a new sealing task. - skipSealHook func(*task) bool // Method to decide whether skipping the sealing. - fullTaskHook func() // Method to call before pushing the full sealing task. - resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. -} - -func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(header *types.Header) bool, init bool) *worker { - worker := &worker{ - config: config, - chainConfig: chainConfig, - engine: engine, - eth: eth, - chain: eth.BlockChain(), - mux: mux, - isLocalBlock: isLocalBlock, - coinbase: config.Etherbase, - extra: config.ExtraData, - pendingTasks: make(map[common.Hash]*task), - txsCh: make(chan core.NewTxsEvent, txChanSize), - chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), - newWorkCh: make(chan *newWorkReq), - getWorkCh: make(chan *getWorkReq), - taskCh: make(chan *task), - resultCh: make(chan *types.Block, resultQueueSize), - startCh: make(chan struct{}, 1), - exitCh: make(chan struct{}), - resubmitIntervalCh: make(chan time.Duration), - resubmitAdjustCh: make(chan *intervalAdjust, resubmitAdjustChanSize), - } - // Subscribe for transaction insertion events (whether from network or resurrects) - worker.txsSub = eth.TxPool().SubscribeTransactions(worker.txsCh, true) - // Subscribe events for blockchain - worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) - - // Sanitize recommit interval if the user-specified one is too short. - recommit := worker.config.Recommit - if recommit < minRecommitInterval { - log.Warn("Sanitizing miner recommit interval", "provided", recommit, "updated", minRecommitInterval) - recommit = minRecommitInterval - } - worker.recommit = recommit - - // Sanitize the timeout config for creating payload. - newpayloadTimeout := worker.config.NewPayloadTimeout - if newpayloadTimeout == 0 { - log.Warn("Sanitizing new payload timeout to default", "provided", newpayloadTimeout, "updated", DefaultConfig.NewPayloadTimeout) - newpayloadTimeout = DefaultConfig.NewPayloadTimeout - } - if newpayloadTimeout < time.Millisecond*100 { - log.Warn("Low payload timeout may cause high amount of non-full blocks", "provided", newpayloadTimeout, "default", DefaultConfig.NewPayloadTimeout) - } - worker.newpayloadTimeout = newpayloadTimeout - - worker.wg.Add(4) - go worker.mainLoop() - go worker.newWorkLoop(recommit) - go worker.resultLoop() - go worker.taskLoop() - - // Submit first work to initialize pending state. - if init { - worker.startCh <- struct{}{} - } - return worker -} - -// setEtherbase sets the etherbase used to initialize the block coinbase field. -func (w *worker) setEtherbase(addr common.Address) { - w.mu.Lock() - defer w.mu.Unlock() - w.coinbase = addr -} - -// etherbase retrieves the configured etherbase address. -func (w *worker) etherbase() common.Address { - w.mu.RLock() - defer w.mu.RUnlock() - return w.coinbase -} - -func (w *worker) setGasCeil(ceil uint64) { - w.mu.Lock() - defer w.mu.Unlock() - w.config.GasCeil = ceil -} - -// setExtra sets the content used to initialize the block extra field. -func (w *worker) setExtra(extra []byte) { - w.mu.Lock() - defer w.mu.Unlock() - w.extra = extra -} - -// setRecommitInterval updates the interval for miner sealing work recommitting. -func (w *worker) setRecommitInterval(interval time.Duration) { - select { - case w.resubmitIntervalCh <- interval: - case <-w.exitCh: - } -} - -// pending returns the pending state and corresponding block. The returned -// values can be nil in case the pending block is not initialized. -func (w *worker) pending() (*types.Block, *state.StateDB) { - w.snapshotMu.RLock() - defer w.snapshotMu.RUnlock() - if w.snapshotState == nil { - return nil, nil - } - return w.snapshotBlock, w.snapshotState.Copy() -} - -// pendingBlock returns pending block. The returned block can be nil in case the -// pending block is not initialized. -func (w *worker) pendingBlock() *types.Block { - w.snapshotMu.RLock() - defer w.snapshotMu.RUnlock() - return w.snapshotBlock -} - -// pendingBlockAndReceipts returns pending block and corresponding receipts. -// The returned values can be nil in case the pending block is not initialized. -func (w *worker) pendingBlockAndReceipts() (*types.Block, types.Receipts) { - w.snapshotMu.RLock() - defer w.snapshotMu.RUnlock() - return w.snapshotBlock, w.snapshotReceipts -} - -// start sets the running status as 1 and triggers new work submitting. -func (w *worker) start() { - w.running.Store(true) - w.startCh <- struct{}{} -} - -// stop sets the running status as 0. -func (w *worker) stop() { - w.running.Store(false) -} - -// isRunning returns an indicator whether worker is running or not. -func (w *worker) isRunning() bool { - return w.running.Load() -} - -// close terminates all background threads maintained by the worker. -// Note the worker does not support being closed multiple times. -func (w *worker) close() { - w.running.Store(false) - close(w.exitCh) - w.wg.Wait() -} - -// recalcRecommit recalculates the resubmitting interval upon feedback. -func recalcRecommit(minRecommit, prev time.Duration, target float64, inc bool) time.Duration { - var ( - prevF = float64(prev.Nanoseconds()) - next float64 - ) - if inc { - next = prevF*(1-intervalAdjustRatio) + intervalAdjustRatio*(target+intervalAdjustBias) - max := float64(maxRecommitInterval.Nanoseconds()) - if next > max { - next = max - } - } else { - next = prevF*(1-intervalAdjustRatio) + intervalAdjustRatio*(target-intervalAdjustBias) - min := float64(minRecommit.Nanoseconds()) - if next < min { - next = min - } - } - return time.Duration(int64(next)) -} - -// newWorkLoop is a standalone goroutine to submit new sealing work upon received events. -func (w *worker) newWorkLoop(recommit time.Duration) { - defer w.wg.Done() - var ( - interrupt *atomic.Int32 - minRecommit = recommit // minimal resubmit interval specified by user. - timestamp int64 // timestamp for each round of sealing. - ) - - timer := time.NewTimer(0) - defer timer.Stop() - <-timer.C // discard the initial tick - - // commit aborts in-flight transaction execution with given signal and resubmits a new one. - commit := func(s int32) { - if interrupt != nil { - interrupt.Store(s) - } - interrupt = new(atomic.Int32) - select { - case w.newWorkCh <- &newWorkReq{interrupt: interrupt, timestamp: timestamp}: - case <-w.exitCh: - return - } - timer.Reset(recommit) - w.newTxs.Store(0) - } - // clearPending cleans the stale pending tasks. - clearPending := func(number uint64) { - w.pendingMu.Lock() - for h, t := range w.pendingTasks { - if t.block.NumberU64()+staleThreshold <= number { - delete(w.pendingTasks, h) - } - } - w.pendingMu.Unlock() - } - - for { - select { - case <-w.startCh: - clearPending(w.chain.CurrentBlock().Number.Uint64()) - timestamp = time.Now().Unix() - commit(commitInterruptNewHead) - - case head := <-w.chainHeadCh: - clearPending(head.Block.NumberU64()) - timestamp = time.Now().Unix() - commit(commitInterruptNewHead) - - case <-timer.C: - // If sealing is running resubmit a new work cycle periodically to pull in - // higher priced transactions. Disable this overhead for pending blocks. - if w.isRunning() && (w.chainConfig.Clique == nil || w.chainConfig.Clique.Period > 0) { - // Short circuit if no new transaction arrives. - if w.newTxs.Load() == 0 { - timer.Reset(recommit) - continue - } - commit(commitInterruptResubmit) - } - - case interval := <-w.resubmitIntervalCh: - // Adjust resubmit interval explicitly by user. - if interval < minRecommitInterval { - log.Warn("Sanitizing miner recommit interval", "provided", interval, "updated", minRecommitInterval) - interval = minRecommitInterval - } - log.Info("Miner recommit interval update", "from", minRecommit, "to", interval) - minRecommit, recommit = interval, interval - - if w.resubmitHook != nil { - w.resubmitHook(minRecommit, recommit) - } - - case adjust := <-w.resubmitAdjustCh: - // Adjust resubmit interval by feedback. - if adjust.inc { - before := recommit - target := float64(recommit.Nanoseconds()) / adjust.ratio - recommit = recalcRecommit(minRecommit, recommit, target, true) - log.Trace("Increase miner recommit interval", "from", before, "to", recommit) - } else { - before := recommit - recommit = recalcRecommit(minRecommit, recommit, float64(minRecommit.Nanoseconds()), false) - log.Trace("Decrease miner recommit interval", "from", before, "to", recommit) - } - - if w.resubmitHook != nil { - w.resubmitHook(minRecommit, recommit) - } - - case <-w.exitCh: - return - } - } -} - -// mainLoop is responsible for generating and submitting sealing work based on -// the received event. It can support two modes: automatically generate task and -// submit it or return task according to given parameters for various proposes. -func (w *worker) mainLoop() { - defer w.wg.Done() - defer w.txsSub.Unsubscribe() - defer w.chainHeadSub.Unsubscribe() - defer func() { - if w.current != nil { - w.current.discard() - } - }() - - for { - select { - case req := <-w.newWorkCh: - w.commitWork(req.interrupt, req.timestamp) - - case req := <-w.getWorkCh: - req.result <- w.generateWork(req.params) - - case ev := <-w.txsCh: - // Apply transactions to the pending state if we're not sealing - // - // Note all transactions received may not be continuous with transactions - // already included in the current sealing block. These transactions will - // be automatically eliminated. - if !w.isRunning() && w.current != nil { - // If block is already full, abort - if gp := w.current.gasPool; gp != nil && gp.Gas() < params.TxGas { - continue - } - txs := make(map[common.Address][]*txpool.LazyTransaction, len(ev.Txs)) - for _, tx := range ev.Txs { - acc, _ := types.Sender(w.current.signer, tx) - txs[acc] = append(txs[acc], &txpool.LazyTransaction{ - Pool: w.eth.TxPool(), // We don't know where this came from, yolo resolve from everywhere - Hash: tx.Hash(), - Tx: nil, // Do *not* set this! We need to resolve it later to pull blobs in - Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), - Gas: tx.Gas(), - BlobGas: tx.BlobGas(), - }) - } - txset := newTransactionsByPriceAndNonce(w.current.signer, txs, w.current.header.BaseFee) - tcount := w.current.tcount - w.commitTransactions(w.current, txset, nil) - - // Only update the snapshot if any new transactions were added - // to the pending block - if tcount != w.current.tcount { - w.updateSnapshot(w.current) - } - } else { - // Special case, if the consensus engine is 0 period clique(dev mode), - // submit sealing work here since all empty submission will be rejected - // by clique. Of course the advance sealing(empty submission) is disabled. - if w.chainConfig.Clique != nil && w.chainConfig.Clique.Period == 0 { - w.commitWork(nil, time.Now().Unix()) - } - } - w.newTxs.Add(int32(len(ev.Txs))) - - // System stopped - case <-w.exitCh: - return - case <-w.txsSub.Err(): - return - case <-w.chainHeadSub.Err(): - return - } - } -} - -// taskLoop is a standalone goroutine to fetch sealing task from the generator and -// push them to consensus engine. -func (w *worker) taskLoop() { - defer w.wg.Done() - var ( - stopCh chan struct{} - prev common.Hash - ) - - // interrupt aborts the in-flight sealing task. - interrupt := func() { - if stopCh != nil { - close(stopCh) - stopCh = nil - } - } - for { - select { - case task := <-w.taskCh: - if w.newTaskHook != nil { - w.newTaskHook(task) - } - // Reject duplicate sealing work due to resubmitting. - sealHash := w.engine.SealHash(task.block.Header()) - if sealHash == prev { - continue - } - // Interrupt previous sealing operation - interrupt() - stopCh, prev = make(chan struct{}), sealHash - - if w.skipSealHook != nil && w.skipSealHook(task) { - continue - } - w.pendingMu.Lock() - w.pendingTasks[sealHash] = task - w.pendingMu.Unlock() - - if err := w.engine.Seal(w.chain, task.block, w.resultCh, stopCh); err != nil { - log.Warn("Block sealing failed", "err", err) - w.pendingMu.Lock() - delete(w.pendingTasks, sealHash) - w.pendingMu.Unlock() - } - case <-w.exitCh: - interrupt() - return - } - } -} - -// resultLoop is a standalone goroutine to handle sealing result submitting -// and flush relative data to the database. -func (w *worker) resultLoop() { - defer w.wg.Done() - for { - select { - case block := <-w.resultCh: - // Short circuit when receiving empty result. - if block == nil { - continue - } - // Short circuit when receiving duplicate result caused by resubmitting. - if w.chain.HasBlock(block.Hash(), block.NumberU64()) { - continue - } - var ( - sealhash = w.engine.SealHash(block.Header()) - hash = block.Hash() - ) - w.pendingMu.RLock() - task, exist := w.pendingTasks[sealhash] - w.pendingMu.RUnlock() - if !exist { - log.Error("Block found but no relative pending task", "number", block.Number(), "sealhash", sealhash, "hash", hash) - continue - } - // Different block could share same sealhash, deep copy here to prevent write-write conflict. - var ( - receipts = make([]*types.Receipt, len(task.receipts)) - logs []*types.Log - ) - for i, taskReceipt := range task.receipts { - receipt := new(types.Receipt) - receipts[i] = receipt - *receipt = *taskReceipt - - // add block location fields - receipt.BlockHash = hash - receipt.BlockNumber = block.Number() - receipt.TransactionIndex = uint(i) - - // Update the block hash in all logs since it is now available and not when the - // receipt/log of individual transactions were created. - receipt.Logs = make([]*types.Log, len(taskReceipt.Logs)) - for i, taskLog := range taskReceipt.Logs { - log := new(types.Log) - receipt.Logs[i] = log - *log = *taskLog - log.BlockHash = hash - } - logs = append(logs, receipt.Logs...) - } - // Commit block and state to database. - _, err := w.chain.WriteBlockAndSetHead(block, receipts, logs, task.state, true) - if err != nil { - log.Error("Failed writing block to chain", "err", err) - continue - } - log.Info("Successfully sealed new block", "number", block.Number(), "sealhash", sealhash, "hash", hash, - "elapsed", common.PrettyDuration(time.Since(task.createdAt))) - - // Broadcast the block and announce chain insertion event - w.mux.Post(core.NewMinedBlockEvent{Block: block}) - - case <-w.exitCh: - return - } - } -} - -// makeEnv creates a new environment for the sealing block. -func (w *worker) makeEnv(parent *types.Header, header *types.Header, coinbase common.Address) (*environment, error) { - // Retrieve the parent state to execute on top and start a prefetcher for - // the miner to speed block sealing up a bit. - state, err := w.chain.StateAt(parent.Root) +// generateWork generates a sealing block based on the given parameters. +func (miner *Miner) generateWork(params *generateParams) *newPayloadResult { + work, err := miner.prepareWork(params) if err != nil { + return &newPayloadResult{err: err} + } + if !params.noTxs { + interrupt := new(atomic.Int32) + timer := time.AfterFunc(miner.config.Recommit, func() { + interrupt.Store(commitInterruptTimeout) + }) + defer timer.Stop() + + err := miner.fillTransactions(interrupt, work) + if errors.Is(err, errBlockInterruptedByTimeout) { + log.Warn("Block building is interrupted", "allowance", common.PrettyDuration(miner.config.Recommit)) + } + } + block, err := miner.engine.FinalizeAndAssemble(miner.chain, work.header, work.state, work.txs, nil, work.receipts, params.withdrawals) + if err != nil { + return &newPayloadResult{err: err} + } + return &newPayloadResult{ + block: block, + fees: totalFees(block, work.receipts), + sidecars: work.sidecars, + stateDB: work.state, + receipts: work.receipts, + } +} + +// prepareWork constructs the sealing task according to the given parameters, +// either based on the last chain head or specified parent. In this function +// the pending transactions are not filled yet, only the empty task returned. +func (miner *Miner) prepareWork(genParams *generateParams) (*environment, error) { + miner.confMu.RLock() + defer miner.confMu.RUnlock() + + // Find the parent block for sealing task + parent := miner.chain.CurrentBlock() + if genParams.parentHash != (common.Hash{}) { + block := miner.chain.GetBlockByHash(genParams.parentHash) + if block == nil { + return nil, fmt.Errorf("missing parent") + } + parent = block.Header() + } + // Sanity check the timestamp correctness, recap the timestamp + // to parent+1 if the mutation is allowed. + timestamp := genParams.timestamp + if parent.Time >= timestamp { + if genParams.forceTime { + return nil, fmt.Errorf("invalid timestamp, parent %d given %d", parent.Time, timestamp) + } + timestamp = parent.Time + 1 + } + // Construct the sealing block header. + header := &types.Header{ + ParentHash: parent.Hash(), + Number: new(big.Int).Add(parent.Number, common.Big1), + GasLimit: core.CalcGasLimit(parent.GasLimit, miner.config.GasCeil), + Time: timestamp, + Coinbase: genParams.coinbase, + } + // Set the extra field. + if len(miner.config.ExtraData) != 0 { + header.Extra = miner.config.ExtraData + } + // Set the randomness field from the beacon chain if it's available. + if genParams.random != (common.Hash{}) { + header.MixDigest = genParams.random + } + // Set baseFee and GasLimit if we are on an EIP-1559 chain + if miner.chainConfig.IsLondon(header.Number) { + header.BaseFee = eip1559.CalcBaseFee(miner.chainConfig, parent) + if !miner.chainConfig.IsLondon(parent.Number) { + parentGasLimit := parent.GasLimit * miner.chainConfig.ElasticityMultiplier() + header.GasLimit = core.CalcGasLimit(parentGasLimit, miner.config.GasCeil) + } + } + // Apply EIP-4844, EIP-4788. + if miner.chainConfig.IsCancun(header.Number, header.Time) { + var excessBlobGas uint64 + if miner.chainConfig.IsCancun(parent.Number, parent.Time) { + excessBlobGas = eip4844.CalcExcessBlobGas(*parent.ExcessBlobGas, *parent.BlobGasUsed) + } else { + // For the first post-fork block, both parent.data_gas_used and parent.excess_data_gas are evaluated as 0 + excessBlobGas = eip4844.CalcExcessBlobGas(0, 0) + } + header.BlobGasUsed = new(uint64) + header.ExcessBlobGas = &excessBlobGas + header.ParentBeaconRoot = genParams.beaconRoot + } + // Run the consensus preparation with the default or customized consensus engine. + if err := miner.engine.Prepare(miner.chain, header); err != nil { + log.Error("Failed to prepare header for sealing", "err", err) return nil, err } - state.StartPrefetcher("miner") - - // Note the passed coinbase may be different with header.Coinbase. - env := &environment{ - signer: types.MakeSigner(w.chainConfig, header.Number, header.Time), - state: state, - coinbase: coinbase, - header: header, + // Could potentially happen if starting to mine in an odd state. + // Note genParams.coinbase can be different with header.Coinbase + // since clique algorithm can modify the coinbase field in header. + env, err := miner.makeEnv(parent, header, genParams.coinbase) + if err != nil { + log.Error("Failed to create sealing context", "err", err) + return nil, err + } + if header.ParentBeaconRoot != nil { + context := core.NewEVMBlockContext(header, miner.chain, nil) + vmenv := vm.NewEVM(context, vm.TxContext{}, env.state, miner.chainConfig, vm.Config{}) + core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, vmenv, env.state) } - // Keep track of transactions which return errors so they can be removed - env.tcount = 0 return env, nil } -// updateSnapshot updates pending snapshot block, receipts and state. -func (w *worker) updateSnapshot(env *environment) { - w.snapshotMu.Lock() - defer w.snapshotMu.Unlock() - - w.snapshotBlock = types.NewBlock( - env.header, - env.txs, - nil, - env.receipts, - trie.NewStackTrie(nil), - ) - w.snapshotReceipts = copyReceipts(env.receipts) - w.snapshotState = env.state.Copy() -} - -func (w *worker) commitTransaction(env *environment, tx *types.Transaction) ([]*types.Log, error) { - if tx.Type() == types.BlobTxType { - return w.commitBlobTransaction(env, tx) - } - receipt, err := w.applyTransaction(env, tx) +// makeEnv creates a new environment for the sealing block. +func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase common.Address) (*environment, error) { + // Retrieve the parent state to execute on top and start a prefetcher for + // the miner to speed block sealing up a bit. + state, err := miner.chain.StateAt(parent.Root) if err != nil { return nil, err } - env.txs = append(env.txs, tx) - env.receipts = append(env.receipts, receipt) - return receipt.Logs, nil + // Note the passed coinbase may be different with header.Coinbase. + return &environment{ + signer: types.MakeSigner(miner.chainConfig, header.Number, header.Time), + state: state, + coinbase: coinbase, + header: header, + }, nil } -func (w *worker) commitBlobTransaction(env *environment, tx *types.Transaction) ([]*types.Log, error) { +func (miner *Miner) commitTransaction(env *environment, tx *types.Transaction) error { + if tx.Type() == types.BlobTxType { + return miner.commitBlobTransaction(env, tx) + } + receipt, err := miner.applyTransaction(env, tx) + if err != nil { + return err + } + env.txs = append(env.txs, tx) + env.receipts = append(env.receipts, receipt) + env.tcount++ + return nil +} + +func (miner *Miner) commitBlobTransaction(env *environment, tx *types.Transaction) error { sc := tx.BlobTxSidecar() if sc == nil { panic("blob transaction without blobs in miner") @@ -764,27 +242,28 @@ func (w *worker) commitBlobTransaction(env *environment, tx *types.Transaction) // and not during execution. This means core.ApplyTransaction will not return an error if the // tx has too many blobs. So we have to explicitly check it here. if (env.blobs+len(sc.Blobs))*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { - return nil, errors.New("max data blobs reached") + return errors.New("max data blobs reached") } - receipt, err := w.applyTransaction(env, tx) + receipt, err := miner.applyTransaction(env, tx) if err != nil { - return nil, err + return err } env.txs = append(env.txs, tx.WithoutBlobTxSidecar()) env.receipts = append(env.receipts, receipt) env.sidecars = append(env.sidecars, sc) env.blobs += len(sc.Blobs) *env.header.BlobGasUsed += receipt.BlobGasUsed - return receipt.Logs, nil + env.tcount++ + return nil } // applyTransaction runs the transaction. If execution fails, state and gas pool are reverted. -func (w *worker) applyTransaction(env *environment, tx *types.Transaction) (*types.Receipt, error) { +func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (*types.Receipt, error) { var ( snap = env.state.Snapshot() gp = env.gasPool.Gas() ) - receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &env.coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, vm.Config{}) + receipt, err := core.ApplyTransaction(miner.chainConfig, miner.chain, &env.coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, vm.Config{}) if err != nil { env.state.RevertToSnapshot(snap) env.gasPool.SetGas(gp) @@ -792,13 +271,11 @@ func (w *worker) applyTransaction(env *environment, tx *types.Transaction) (*typ return receipt, err } -func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { +func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { gasLimit := env.header.GasLimit if env.gasPool == nil { env.gasPool = new(core.GasPool).AddGas(gasLimit) } - var coalescedLogs []*types.Log - for { // Check interruption signal and abort building if it's fired. if interrupt != nil { @@ -811,8 +288,33 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", params.TxGas) break } + // If we don't have enough blob space for any further blob transactions, + // skip that list altogether + if !blobTxs.Empty() && env.blobs*params.BlobTxBlobGasPerBlob >= params.MaxBlobGasPerBlock { + log.Trace("Not enough blob space for further blob transactions") + blobTxs.Clear() + // Fall though to pick up any plain txs + } // Retrieve the next transaction and abort if all done. - ltx := txs.Peek() + var ( + ltx *txpool.LazyTransaction + txs *transactionsByPriceAndNonce + ) + pltx, ptip := plainTxs.Peek() + bltx, btip := blobTxs.Peek() + + switch { + case pltx == nil: + txs, ltx = blobTxs, bltx + case bltx == nil: + txs, ltx = plainTxs, pltx + default: + if ptip.Lt(btip) { + txs, ltx = blobTxs, bltx + } else { + txs, ltx = plainTxs, pltx + } + } if ltx == nil { break } @@ -840,15 +342,15 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn // Check whether the tx is replay protected. If we're not in the EIP155 hf // phase, start ignoring the sender until we do. - if tx.Protected() && !w.chainConfig.IsEIP155(env.header.Number) { - log.Trace("Ignoring replay protected transaction", "hash", ltx.Hash, "eip155", w.chainConfig.EIP155Block) + if tx.Protected() && !miner.chainConfig.IsEIP155(env.header.Number) { + log.Trace("Ignoring replay protected transaction", "hash", ltx.Hash, "eip155", miner.chainConfig.EIP155Block) txs.Pop() continue } // Start executing the transaction env.state.SetTxContext(tx.Hash(), env.tcount) - logs, err := w.commitTransaction(env, tx) + err := miner.commitTransaction(env, tx) switch { case errors.Is(err, core.ErrNonceTooLow): // New head notification data race between the transaction pool and miner, shift @@ -857,8 +359,6 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn case errors.Is(err, nil): // Everything ok, collect the logs and shift in the next transaction from the same account - coalescedLogs = append(coalescedLogs, logs...) - env.tcount++ txs.Shift() default: @@ -868,326 +368,67 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn txs.Pop() } } - if !w.isRunning() && len(coalescedLogs) > 0 { - // We don't push the pendingLogsEvent while we are sealing. The reason is that - // when we are sealing, the worker will regenerate a sealing block every 3 seconds. - // In order to avoid pushing the repeated pendingLog, we disable the pending log pushing. - - // make a copy, the state caches the logs and these logs get "upgraded" from pending to mined - // logs by filling in the block hash when the block was mined by the local miner. This can - // cause a race condition if a log was "upgraded" before the PendingLogsEvent is processed. - cpy := make([]*types.Log, len(coalescedLogs)) - for i, l := range coalescedLogs { - cpy[i] = new(types.Log) - *cpy[i] = *l - } - w.pendingLogsFeed.Send(cpy) - } return nil } -// generateParams wraps various of settings for generating sealing task. -type generateParams struct { - timestamp uint64 // The timstamp for sealing task - forceTime bool // Flag whether the given timestamp is immutable or not - parentHash common.Hash // Parent block hash, empty means the latest chain head - coinbase common.Address // The fee recipient address for including transaction - random common.Hash // The randomness generated by beacon chain, empty before the merge - withdrawals types.Withdrawals // List of withdrawals to include in block. - beaconRoot *common.Hash // The beacon root (cancun field). - noTxs bool // Flag whether an empty block without any transaction is expected -} - -// prepareWork constructs the sealing task according to the given parameters, -// either based on the last chain head or specified parent. In this function -// the pending transactions are not filled yet, only the empty task returned. -func (w *worker) prepareWork(genParams *generateParams) (*environment, error) { - w.mu.RLock() - defer w.mu.RUnlock() - - // Find the parent block for sealing task - parent := w.chain.CurrentBlock() - if genParams.parentHash != (common.Hash{}) { - block := w.chain.GetBlockByHash(genParams.parentHash) - if block == nil { - return nil, fmt.Errorf("missing parent") - } - parent = block.Header() - } - // Sanity check the timestamp correctness, recap the timestamp - // to parent+1 if the mutation is allowed. - timestamp := genParams.timestamp - if parent.Time >= timestamp { - if genParams.forceTime { - return nil, fmt.Errorf("invalid timestamp, parent %d given %d", parent.Time, timestamp) - } - timestamp = parent.Time + 1 - } - // Construct the sealing block header. - header := &types.Header{ - ParentHash: parent.Hash(), - Number: new(big.Int).Add(parent.Number, common.Big1), - GasLimit: core.CalcGasLimit(parent.GasLimit, w.config.GasCeil), - Time: timestamp, - Coinbase: genParams.coinbase, - } - // Set the extra field. - if len(w.extra) != 0 { - header.Extra = w.extra - } - // Set the randomness field from the beacon chain if it's available. - if genParams.random != (common.Hash{}) { - header.MixDigest = genParams.random - } - // Set baseFee and GasLimit if we are on an EIP-1559 chain - if w.chainConfig.IsLondon(header.Number) { - header.BaseFee = eip1559.CalcBaseFee(w.chainConfig, parent) - if !w.chainConfig.IsLondon(parent.Number) { - parentGasLimit := parent.GasLimit * w.chainConfig.ElasticityMultiplier() - header.GasLimit = core.CalcGasLimit(parentGasLimit, w.config.GasCeil) - } - } - // Apply EIP-4844, EIP-4788. - if w.chainConfig.IsCancun(header.Number, header.Time) { - var excessBlobGas uint64 - if w.chainConfig.IsCancun(parent.Number, parent.Time) { - excessBlobGas = eip4844.CalcExcessBlobGas(*parent.ExcessBlobGas, *parent.BlobGasUsed) - } else { - // For the first post-fork block, both parent.data_gas_used and parent.excess_data_gas are evaluated as 0 - excessBlobGas = eip4844.CalcExcessBlobGas(0, 0) - } - header.BlobGasUsed = new(uint64) - header.ExcessBlobGas = &excessBlobGas - header.ParentBeaconRoot = genParams.beaconRoot - } - // Run the consensus preparation with the default or customized consensus engine. - if err := w.engine.Prepare(w.chain, header); err != nil { - log.Error("Failed to prepare header for sealing", "err", err) - return nil, err - } - // Could potentially happen if starting to mine in an odd state. - // Note genParams.coinbase can be different with header.Coinbase - // since clique algorithm can modify the coinbase field in header. - env, err := w.makeEnv(parent, header, genParams.coinbase) - if err != nil { - log.Error("Failed to create sealing context", "err", err) - return nil, err - } - if header.ParentBeaconRoot != nil { - context := core.NewEVMBlockContext(header, w.chain, nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, env.state, w.chainConfig, vm.Config{}) - core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, vmenv, env.state, nil) - } - return env, nil -} - // fillTransactions retrieves the pending transactions from the txpool and fills them // into the given sealing block. The transaction selection and ordering strategy can // be customized with the plugin in the future. -func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) error { - pending := w.eth.TxPool().Pending(true) +func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) error { + miner.confMu.RLock() + tip := miner.config.GasPrice + miner.confMu.RUnlock() + + // Retrieve the pending transactions pre-filtered by the 1559/4844 dynamic fees + filter := txpool.PendingFilter{ + MinTip: uint256.MustFromBig(tip), + } + if env.header.BaseFee != nil { + filter.BaseFee = uint256.MustFromBig(env.header.BaseFee) + } + if env.header.ExcessBlobGas != nil { + filter.BlobFee = uint256.MustFromBig(eip4844.CalcBlobFee(*env.header.ExcessBlobGas)) + } + filter.OnlyPlainTxs, filter.OnlyBlobTxs = true, false + pendingPlainTxs := miner.txpool.Pending(filter) + + filter.OnlyPlainTxs, filter.OnlyBlobTxs = false, true + pendingBlobTxs := miner.txpool.Pending(filter) // Split the pending transactions into locals and remotes. - localTxs, remoteTxs := make(map[common.Address][]*txpool.LazyTransaction), pending - for _, account := range w.eth.TxPool().Locals() { - if txs := remoteTxs[account]; len(txs) > 0 { - delete(remoteTxs, account) - localTxs[account] = txs + localPlainTxs, remotePlainTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingPlainTxs + localBlobTxs, remoteBlobTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingBlobTxs + + for _, account := range miner.txpool.Locals() { + if txs := remotePlainTxs[account]; len(txs) > 0 { + delete(remotePlainTxs, account) + localPlainTxs[account] = txs + } + if txs := remoteBlobTxs[account]; len(txs) > 0 { + delete(remoteBlobTxs, account) + localBlobTxs[account] = txs } } - // Fill the block with all available pending transactions. - if len(localTxs) > 0 { - txs := newTransactionsByPriceAndNonce(env.signer, localTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt); err != nil { + if len(localPlainTxs) > 0 || len(localBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, localPlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, localBlobTxs, env.header.BaseFee) + + if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } - if len(remoteTxs) > 0 { - txs := newTransactionsByPriceAndNonce(env.signer, remoteTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt); err != nil { + if len(remotePlainTxs) > 0 || len(remoteBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, remotePlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, remoteBlobTxs, env.header.BaseFee) + + if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } return nil } -// generateWork generates a sealing block based on the given parameters. -func (w *worker) generateWork(params *generateParams) *newPayloadResult { - work, err := w.prepareWork(params) - if err != nil { - return &newPayloadResult{err: err} - } - defer work.discard() - - if !params.noTxs { - interrupt := new(atomic.Int32) - timer := time.AfterFunc(w.newpayloadTimeout, func() { - interrupt.Store(commitInterruptTimeout) - }) - defer timer.Stop() - - err := w.fillTransactions(interrupt, work) - if errors.Is(err, errBlockInterruptedByTimeout) { - log.Warn("Block building is interrupted", "allowance", common.PrettyDuration(w.newpayloadTimeout)) - } - } - block, err := w.engine.FinalizeAndAssemble(w.chain, work.header, work.state, work.txs, nil, work.receipts, params.withdrawals) - if err != nil { - return &newPayloadResult{err: err} - } - return &newPayloadResult{ - block: block, - fees: totalFees(block, work.receipts), - sidecars: work.sidecars, - } -} - -// commitWork generates several new sealing tasks based on the parent block -// and submit them to the sealer. -func (w *worker) commitWork(interrupt *atomic.Int32, timestamp int64) { - // Abort committing if node is still syncing - if w.syncing.Load() { - return - } - start := time.Now() - - // Set the coinbase if the worker is running or it's required - var coinbase common.Address - if w.isRunning() { - coinbase = w.etherbase() - if coinbase == (common.Address{}) { - log.Error("Refusing to mine without etherbase") - return - } - } - work, err := w.prepareWork(&generateParams{ - timestamp: uint64(timestamp), - coinbase: coinbase, - }) - if err != nil { - return - } - // Fill pending transactions from the txpool into the block. - err = w.fillTransactions(interrupt, work) - switch { - case err == nil: - // The entire block is filled, decrease resubmit interval in case - // of current interval is larger than the user-specified one. - w.adjustResubmitInterval(&intervalAdjust{inc: false}) - - case errors.Is(err, errBlockInterruptedByRecommit): - // Notify resubmit loop to increase resubmitting interval if the - // interruption is due to frequent commits. - gaslimit := work.header.GasLimit - ratio := float64(gaslimit-work.gasPool.Gas()) / float64(gaslimit) - if ratio < 0.1 { - ratio = 0.1 - } - w.adjustResubmitInterval(&intervalAdjust{ - ratio: ratio, - inc: true, - }) - - case errors.Is(err, errBlockInterruptedByNewHead): - // If the block building is interrupted by newhead event, discard it - // totally. Committing the interrupted block introduces unnecessary - // delay, and possibly causes miner to mine on the previous head, - // which could result in higher uncle rate. - work.discard() - return - } - // Submit the generated block for consensus sealing. - w.commit(work.copy(), w.fullTaskHook, true, start) - - // Swap out the old work with the new one, terminating any leftover - // prefetcher processes in the mean time and starting a new one. - if w.current != nil { - w.current.discard() - } - w.current = work -} - -// commit runs any post-transaction state modifications, assembles the final block -// and commits new work if consensus engine is running. -// Note the assumption is held that the mutation is allowed to the passed env, do -// the deep copy first. -func (w *worker) commit(env *environment, interval func(), update bool, start time.Time) error { - if w.isRunning() { - if interval != nil { - interval() - } - // Create a local environment copy, avoid the data race with snapshot state. - // https://github.com/ethereum/go-ethereum/issues/24299 - env := env.copy() - // Withdrawals are set to nil here, because this is only called in PoW. - block, err := w.engine.FinalizeAndAssemble(w.chain, env.header, env.state, env.txs, nil, env.receipts, nil) - if err != nil { - return err - } - // If we're post merge, just ignore - if !w.isTTDReached(block.Header()) { - select { - case w.taskCh <- &task{receipts: env.receipts, state: env.state, block: block, createdAt: time.Now()}: - fees := totalFees(block, env.receipts) - feesInEther := new(big.Float).Quo(new(big.Float).SetInt(fees), big.NewFloat(params.Ether)) - log.Info("Commit new sealing work", "number", block.Number(), "sealhash", w.engine.SealHash(block.Header()), - "txs", env.tcount, "gas", block.GasUsed(), "fees", feesInEther, - "elapsed", common.PrettyDuration(time.Since(start))) - - case <-w.exitCh: - log.Info("Worker has exited") - } - } - } - if update { - w.updateSnapshot(env) - } - return nil -} - -// getSealingBlock generates the sealing block based on the given parameters. -// The generation result will be passed back via the given channel no matter -// the generation itself succeeds or not. -func (w *worker) getSealingBlock(params *generateParams) *newPayloadResult { - req := &getWorkReq{ - params: params, - result: make(chan *newPayloadResult, 1), - } - select { - case w.getWorkCh <- req: - return <-req.result - case <-w.exitCh: - return &newPayloadResult{err: errors.New("miner closed")} - } -} - -// isTTDReached returns the indicator if the given block has reached the total -// terminal difficulty for The Merge transition. -func (w *worker) isTTDReached(header *types.Header) bool { - td, ttd := w.chain.GetTd(header.ParentHash, header.Number.Uint64()-1), w.chain.Config().TerminalTotalDifficulty - return td != nil && ttd != nil && td.Cmp(ttd) >= 0 -} - -// adjustResubmitInterval adjusts the resubmit interval. -func (w *worker) adjustResubmitInterval(message *intervalAdjust) { - select { - case w.resubmitAdjustCh <- message: - default: - log.Warn("the resubmitAdjustCh is full, discard the message") - } -} - -// copyReceipts makes a deep copy of the given receipts. -func copyReceipts(receipts []*types.Receipt) []*types.Receipt { - result := make([]*types.Receipt, len(receipts)) - for i, l := range receipts { - cpy := *l - result[i] = &cpy - } - return result -} - // totalFees computes total consumed miner fees in Wei. Block transactions and receipts have to have the same order. func totalFees(block *types.Block, receipts []*types.Receipt) *big.Int { feesWei := new(big.Int) diff --git a/miner/worker_test.go b/miner/worker_test.go deleted file mode 100644 index 675b8d55b9..0000000000 --- a/miner/worker_test.go +++ /dev/null @@ -1,510 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package miner - -import ( - "math/big" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/consensus/clique" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/txpool" - "github.com/ethereum/go-ethereum/core/txpool/legacypool" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/params" - "github.com/holiman/uint256" -) - -const ( - // testCode is the testing contract binary code which will initialises some - // variables in constructor - testCode = "0x60806040527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0060005534801561003457600080fd5b5060fc806100436000396000f3fe6080604052348015600f57600080fd5b506004361060325760003560e01c80630c4dae8814603757806398a213cf146053575b600080fd5b603d607e565b6040518082815260200191505060405180910390f35b607c60048036036020811015606757600080fd5b81019080803590602001909291905050506084565b005b60005481565b806000819055507fe9e44f9f7da8c559de847a3232b57364adc0354f15a2cd8dc636d54396f9587a6000546040518082815260200191505060405180910390a15056fea265627a7a723058208ae31d9424f2d0bc2a3da1a5dd659db2d71ec322a17db8f87e19e209e3a1ff4a64736f6c634300050a0032" - - // testGas is the gas required for contract deployment. - testGas = 144109 -) - -var ( - // Test chain configurations - testTxPoolConfig legacypool.Config - ethashChainConfig *params.ChainConfig - cliqueChainConfig *params.ChainConfig - - // Test accounts - testBankKey, _ = crypto.GenerateKey() - testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) - testBankFunds = big.NewInt(1000000000000000000) - - testUserKey, _ = crypto.GenerateKey() - testUserAddress = crypto.PubkeyToAddress(testUserKey.PublicKey) - - // Test transactions - pendingTxs []*types.Transaction - newTxs []*types.Transaction - - testConfig = &Config{ - Recommit: time.Second, - GasCeil: params.GenesisGasLimit, - } -) - -func init() { - testTxPoolConfig = legacypool.DefaultConfig - testTxPoolConfig.Journal = "" - ethashChainConfig = new(params.ChainConfig) - *ethashChainConfig = *params.TestChainConfig - cliqueChainConfig = new(params.ChainConfig) - *cliqueChainConfig = *params.TestChainConfig - cliqueChainConfig.Clique = ¶ms.CliqueConfig{ - Period: 10, - Epoch: 30000, - } - - signer := types.LatestSigner(params.TestChainConfig) - tx1 := types.MustSignNewTx(testBankKey, signer, &types.AccessListTx{ - ChainID: params.TestChainConfig.ChainID, - Nonce: 0, - To: &testUserAddress, - Value: big.NewInt(1000), - Gas: params.TxGas, - GasPrice: big.NewInt(params.InitialBaseFee), - }) - pendingTxs = append(pendingTxs, tx1) - - tx2 := types.MustSignNewTx(testBankKey, signer, &types.LegacyTx{ - Nonce: 1, - To: &testUserAddress, - Value: big.NewInt(1000), - Gas: params.TxGas, - GasPrice: big.NewInt(params.InitialBaseFee), - }) - newTxs = append(newTxs, tx2) -} - -// testWorkerBackend implements worker.Backend interfaces and wraps all information needed during the testing. -type testWorkerBackend struct { - db ethdb.Database - txPool *txpool.TxPool - chain *core.BlockChain - genesis *core.Genesis -} - -func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, n int) *testWorkerBackend { - var gspec = &core.Genesis{ - Config: chainConfig, - Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, - } - switch e := engine.(type) { - case *clique.Clique: - gspec.ExtraData = make([]byte, 32+common.AddressLength+crypto.SignatureLength) - copy(gspec.ExtraData[32:32+common.AddressLength], testBankAddress.Bytes()) - e.Authorize(testBankAddress, func(account accounts.Account, s string, data []byte) ([]byte, error) { - return crypto.Sign(crypto.Keccak256(data), testBankKey) - }) - case *ethash.Ethash: - default: - t.Fatalf("unexpected consensus engine type: %T", engine) - } - chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec, nil, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("core.NewBlockChain failed: %v", err) - } - pool := legacypool.New(testTxPoolConfig, chain) - txpool, _ := txpool.New(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), chain, []txpool.SubPool{pool}) - - return &testWorkerBackend{ - db: db, - chain: chain, - txPool: txpool, - genesis: gspec, - } -} - -func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain } -func (b *testWorkerBackend) TxPool() *txpool.TxPool { return b.txPool } - -func (b *testWorkerBackend) newRandomTx(creation bool) *types.Transaction { - var tx *types.Transaction - gasPrice := big.NewInt(10 * params.InitialBaseFee) - if creation { - tx, _ = types.SignTx(types.NewContractCreation(b.txPool.Nonce(testBankAddress), big.NewInt(0), testGas, gasPrice, common.FromHex(testCode)), types.HomesteadSigner{}, testBankKey) - } else { - tx, _ = types.SignTx(types.NewTransaction(b.txPool.Nonce(testBankAddress), testUserAddress, big.NewInt(1000), params.TxGas, gasPrice, nil), types.HomesteadSigner{}, testBankKey) - } - return tx -} - -func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*worker, *testWorkerBackend) { - backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) - backend.txPool.Add(pendingTxs, true, false) - w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false) - w.setEtherbase(testBankAddress) - return w, backend -} - -func TestGenerateAndImportBlock(t *testing.T) { - t.Parallel() - var ( - db = rawdb.NewMemoryDatabase() - config = *params.AllCliqueProtocolChanges - ) - config.Clique = ¶ms.CliqueConfig{Period: 1, Epoch: 30000} - engine := clique.New(config.Clique, db) - - w, b := newTestWorker(t, &config, engine, db, 0) - defer w.close() - - // This test chain imports the mined blocks. - chain, _ := core.NewBlockChain(rawdb.NewMemoryDatabase(), nil, b.genesis, nil, engine, vm.Config{}, nil, nil) - defer chain.Stop() - - // Ignore empty commit here for less noise. - w.skipSealHook = func(task *task) bool { - return len(task.receipts) == 0 - } - - // Wait for mined blocks. - sub := w.mux.Subscribe(core.NewMinedBlockEvent{}) - defer sub.Unsubscribe() - - // Start mining! - w.start() - - for i := 0; i < 5; i++ { - b.txPool.Add([]*types.Transaction{b.newRandomTx(true)}, true, false) - b.txPool.Add([]*types.Transaction{b.newRandomTx(false)}, true, false) - - select { - case ev := <-sub.Chan(): - block := ev.Data.(core.NewMinedBlockEvent).Block - if _, err := chain.InsertChain([]*types.Block{block}); err != nil { - t.Fatalf("failed to insert new mined block %d: %v", block.NumberU64(), err) - } - case <-time.After(3 * time.Second): // Worker needs 1s to include new changes. - t.Fatalf("timeout") - } - } -} - -func TestEmptyWorkEthash(t *testing.T) { - t.Parallel() - testEmptyWork(t, ethashChainConfig, ethash.NewFaker()) -} -func TestEmptyWorkClique(t *testing.T) { - t.Parallel() - testEmptyWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) -} - -func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { - defer engine.Close() - - w, _ := newTestWorker(t, chainConfig, engine, rawdb.NewMemoryDatabase(), 0) - defer w.close() - - taskCh := make(chan struct{}, 2) - checkEqual := func(t *testing.T, task *task) { - // The work should contain 1 tx - receiptLen, balance := 1, uint256.NewInt(1000) - if len(task.receipts) != receiptLen { - t.Fatalf("receipt number mismatch: have %d, want %d", len(task.receipts), receiptLen) - } - if task.state.GetBalance(testUserAddress).Cmp(balance) != 0 { - t.Fatalf("account balance mismatch: have %d, want %d", task.state.GetBalance(testUserAddress), balance) - } - } - w.newTaskHook = func(task *task) { - if task.block.NumberU64() == 1 { - checkEqual(t, task) - taskCh <- struct{}{} - } - } - w.skipSealHook = func(task *task) bool { return true } - w.fullTaskHook = func() { - time.Sleep(100 * time.Millisecond) - } - w.start() // Start mining! - select { - case <-taskCh: - case <-time.NewTimer(3 * time.Second).C: - t.Error("new task timeout") - } -} - -func TestAdjustIntervalEthash(t *testing.T) { - t.Parallel() - testAdjustInterval(t, ethashChainConfig, ethash.NewFaker()) -} - -func TestAdjustIntervalClique(t *testing.T) { - t.Parallel() - testAdjustInterval(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) -} - -func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { - defer engine.Close() - - w, _ := newTestWorker(t, chainConfig, engine, rawdb.NewMemoryDatabase(), 0) - defer w.close() - - w.skipSealHook = func(task *task) bool { - return true - } - w.fullTaskHook = func() { - time.Sleep(100 * time.Millisecond) - } - var ( - progress = make(chan struct{}, 10) - result = make([]float64, 0, 10) - index = 0 - start atomic.Bool - ) - w.resubmitHook = func(minInterval time.Duration, recommitInterval time.Duration) { - // Short circuit if interval checking hasn't started. - if !start.Load() { - return - } - var wantMinInterval, wantRecommitInterval time.Duration - - switch index { - case 0: - wantMinInterval, wantRecommitInterval = 3*time.Second, 3*time.Second - case 1: - origin := float64(3 * time.Second.Nanoseconds()) - estimate := origin*(1-intervalAdjustRatio) + intervalAdjustRatio*(origin/0.8+intervalAdjustBias) - wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(estimate)*time.Nanosecond - case 2: - estimate := result[index-1] - min := float64(3 * time.Second.Nanoseconds()) - estimate = estimate*(1-intervalAdjustRatio) + intervalAdjustRatio*(min-intervalAdjustBias) - wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(estimate)*time.Nanosecond - case 3: - wantMinInterval, wantRecommitInterval = time.Second, time.Second - } - - // Check interval - if minInterval != wantMinInterval { - t.Errorf("resubmit min interval mismatch: have %v, want %v ", minInterval, wantMinInterval) - } - if recommitInterval != wantRecommitInterval { - t.Errorf("resubmit interval mismatch: have %v, want %v", recommitInterval, wantRecommitInterval) - } - result = append(result, float64(recommitInterval.Nanoseconds())) - index += 1 - progress <- struct{}{} - } - w.start() - - time.Sleep(time.Second) // Ensure two tasks have been submitted due to start opt - start.Store(true) - - w.setRecommitInterval(3 * time.Second) - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } - - w.resubmitAdjustCh <- &intervalAdjust{inc: true, ratio: 0.8} - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } - - w.resubmitAdjustCh <- &intervalAdjust{inc: false} - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } - - w.setRecommitInterval(500 * time.Millisecond) - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } -} - -func TestGetSealingWorkEthash(t *testing.T) { - t.Parallel() - testGetSealingWork(t, ethashChainConfig, ethash.NewFaker()) -} - -func TestGetSealingWorkClique(t *testing.T) { - t.Parallel() - testGetSealingWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) -} - -func TestGetSealingWorkPostMerge(t *testing.T) { - t.Parallel() - local := new(params.ChainConfig) - *local = *ethashChainConfig - local.TerminalTotalDifficulty = big.NewInt(0) - testGetSealingWork(t, local, ethash.NewFaker()) -} - -func testGetSealingWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { - defer engine.Close() - - w, b := newTestWorker(t, chainConfig, engine, rawdb.NewMemoryDatabase(), 0) - defer w.close() - - w.setExtra([]byte{0x01, 0x02}) - - w.skipSealHook = func(task *task) bool { - return true - } - w.fullTaskHook = func() { - time.Sleep(100 * time.Millisecond) - } - timestamp := uint64(time.Now().Unix()) - assertBlock := func(block *types.Block, number uint64, coinbase common.Address, random common.Hash) { - if block.Time() != timestamp { - // Sometime the timestamp will be mutated if the timestamp - // is even smaller than parent block's. It's OK. - t.Logf("Invalid timestamp, want %d, get %d", timestamp, block.Time()) - } - _, isClique := engine.(*clique.Clique) - if !isClique { - if len(block.Extra()) != 2 { - t.Error("Unexpected extra field") - } - if block.Coinbase() != coinbase { - t.Errorf("Unexpected coinbase got %x want %x", block.Coinbase(), coinbase) - } - } else { - if block.Coinbase() != (common.Address{}) { - t.Error("Unexpected coinbase") - } - } - if !isClique { - if block.MixDigest() != random { - t.Error("Unexpected mix digest") - } - } - if block.Nonce() != 0 { - t.Error("Unexpected block nonce") - } - if block.NumberU64() != number { - t.Errorf("Mismatched block number, want %d got %d", number, block.NumberU64()) - } - } - var cases = []struct { - parent common.Hash - coinbase common.Address - random common.Hash - expectNumber uint64 - expectErr bool - }{ - { - b.chain.Genesis().Hash(), - common.HexToAddress("0xdeadbeef"), - common.HexToHash("0xcafebabe"), - uint64(1), - false, - }, - { - b.chain.CurrentBlock().Hash(), - common.HexToAddress("0xdeadbeef"), - common.HexToHash("0xcafebabe"), - b.chain.CurrentBlock().Number.Uint64() + 1, - false, - }, - { - b.chain.CurrentBlock().Hash(), - common.Address{}, - common.HexToHash("0xcafebabe"), - b.chain.CurrentBlock().Number.Uint64() + 1, - false, - }, - { - b.chain.CurrentBlock().Hash(), - common.Address{}, - common.Hash{}, - b.chain.CurrentBlock().Number.Uint64() + 1, - false, - }, - { - common.HexToHash("0xdeadbeef"), - common.HexToAddress("0xdeadbeef"), - common.HexToHash("0xcafebabe"), - 0, - true, - }, - } - - // This API should work even when the automatic sealing is not enabled - for _, c := range cases { - r := w.getSealingBlock(&generateParams{ - parentHash: c.parent, - timestamp: timestamp, - coinbase: c.coinbase, - random: c.random, - withdrawals: nil, - beaconRoot: nil, - noTxs: false, - forceTime: true, - }) - if c.expectErr { - if r.err == nil { - t.Error("Expect error but get nil") - } - } else { - if r.err != nil { - t.Errorf("Unexpected error %v", r.err) - } - assertBlock(r.block, c.expectNumber, c.coinbase, c.random) - } - } - - // This API should work even when the automatic sealing is enabled - w.start() - for _, c := range cases { - r := w.getSealingBlock(&generateParams{ - parentHash: c.parent, - timestamp: timestamp, - coinbase: c.coinbase, - random: c.random, - withdrawals: nil, - beaconRoot: nil, - noTxs: false, - forceTime: true, - }) - if c.expectErr { - if r.err == nil { - t.Error("Expect error but get nil") - } - } else { - if r.err != nil { - t.Errorf("Unexpected error %v", r.err) - } - assertBlock(r.block, c.expectNumber, c.coinbase, c.random) - } - } -} diff --git a/node/api.go b/node/api.go index f81f394beb..a71ae6aa29 100644 --- a/node/api.go +++ b/node/api.go @@ -145,8 +145,6 @@ func (api *adminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, error) return case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() diff --git a/node/defaults.go b/node/defaults.go index 42d9d4cde0..307d9e186a 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -41,6 +41,7 @@ const ( // needs of all CLs. engineAPIBatchItemLimit = 2000 engineAPIBatchResponseSizeLimit = 250 * 1000 * 1000 + engineAPIBodyLimit = 128 * 1024 * 1024 ) var ( diff --git a/node/node.go b/node/node.go index 41c9971fe8..c5cb552d27 100644 --- a/node/node.go +++ b/node/node.go @@ -339,15 +339,9 @@ func (n *Node) closeDataDir() { } } -// obtainJWTSecret loads the jwt-secret, either from the provided config, -// or from the default location. If neither of those are present, it generates -// a new secret and stores to the default location. -func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { - fileName := cliParam - if len(fileName) == 0 { - // no path provided, use default - fileName = n.ResolvePath(datadirJWTKey) - } +// ObtainJWTSecret loads the jwt-secret from the provided config. If the file is not +// present, it generates a new secret and stores to the given location. +func ObtainJWTSecret(fileName string) ([]byte, error) { // try reading from file if data, err := os.ReadFile(fileName); err == nil { jwtSecret := common.FromHex(strings.TrimSpace(string(data))) @@ -373,6 +367,18 @@ func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { return jwtSecret, nil } +// obtainJWTSecret loads the jwt-secret, either from the provided config, +// or from the default location. If neither of those are present, it generates +// a new secret and stores to the default location. +func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { + fileName := cliParam + if len(fileName) == 0 { + // no path provided, use default + fileName = n.ResolvePath(datadirJWTKey) + } + return ObtainJWTSecret(fileName) +} + // startRPC is a helper method to configure all the various RPC endpoints during node // startup. It's not meant to be called at any time afterwards as it makes certain // assumptions about the state of the node. @@ -453,14 +459,16 @@ func (n *Node) startRPC() error { jwtSecret: secret, batchItemLimit: engineAPIBatchItemLimit, batchResponseSizeLimit: engineAPIBatchResponseSizeLimit, + httpBodyLimit: engineAPIBodyLimit, } - if err := server.enableRPC(allAPIs, httpConfig{ + err := server.enableRPC(allAPIs, httpConfig{ CorsAllowedOrigins: DefaultAuthCors, Vhosts: n.config.AuthVirtualHosts, Modules: DefaultAuthModules, prefix: DefaultAuthPrefix, rpcEndpointConfig: sharedConfig, - }); err != nil { + }) + if err != nil { return err } servers = append(servers, server) diff --git a/node/node_test.go b/node/node_test.go index 04810a815b..d1d1e5dfe8 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -415,21 +415,6 @@ func TestRegisterHandler_Successful(t *testing.T) { assert.Equal(t, "success", string(buf)) } -// Tests that the given handler will not be successfully mounted since no HTTP server -// is enabled for RPC -func TestRegisterHandler_Unsuccessful(t *testing.T) { - node, err := New(&DefaultConfig) - if err != nil { - t.Fatalf("could not create new node: %v", err) - } - - // create and mount handler - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("success")) - }) - node.RegisterHandler("test", "/test", handler) -} - // Tests whether websocket requests can be handled on the same port as a regular http server. func TestWebsocketHTTPOnSamePort_WebsocketRequest(t *testing.T) { node := startHTTP(t, 0, 0) diff --git a/node/rpcstack.go b/node/rpcstack.go index b33c238051..253db0d564 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -19,6 +19,7 @@ package node import ( "compress/gzip" "context" + "errors" "fmt" "io" "net" @@ -56,6 +57,7 @@ type rpcEndpointConfig struct { jwtSecret []byte // optional JWT secret batchItemLimit int batchResponseSizeLimit int + httpBodyLimit int } type rpcHandler struct { @@ -298,12 +300,15 @@ func (h *httpServer) enableRPC(apis []rpc.API, config httpConfig) error { defer h.mu.Unlock() if h.rpcAllowed() { - return fmt.Errorf("JSON-RPC over HTTP is already enabled") + return errors.New("JSON-RPC over HTTP is already enabled") } // Create RPC server and handler. srv := rpc.NewServer() srv.SetBatchLimits(config.batchItemLimit, config.batchResponseSizeLimit) + if config.httpBodyLimit > 0 { + srv.SetHTTPBodyLimit(config.httpBodyLimit) + } if err := RegisterApis(apis, config.Modules, srv); err != nil { return err } @@ -331,11 +336,14 @@ func (h *httpServer) enableWS(apis []rpc.API, config wsConfig) error { defer h.mu.Unlock() if h.wsAllowed() { - return fmt.Errorf("JSON-RPC over WebSocket is already enabled") + return errors.New("JSON-RPC over WebSocket is already enabled") } // Create RPC server and handler. srv := rpc.NewServer() srv.SetBatchLimits(config.batchItemLimit, config.batchResponseSizeLimit) + if config.httpBodyLimit > 0 { + srv.SetHTTPBodyLimit(config.httpBodyLimit) + } if err := RegisterApis(apis, config.Modules, srv); err != nil { return err } diff --git a/p2p/dial.go b/p2p/dial.go index 5e4ab1d50d..08e1db2877 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -25,6 +25,7 @@ import ( mrand "math/rand" "net" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common/mclock" @@ -248,7 +249,7 @@ loop: } case task := <-d.doneCh: - id := task.dest.ID() + id := task.dest().ID() delete(d.dialing, id) d.updateStaticPool(id) d.doneSinceLastLog++ @@ -410,7 +411,7 @@ func (d *dialScheduler) startStaticDials(n int) (started int) { // updateStaticPool attempts to move the given static dial back into staticPool. func (d *dialScheduler) updateStaticPool(id enode.ID) { task, ok := d.static[id] - if ok && task.staticPoolIndex < 0 && d.checkDial(task.dest) == nil { + if ok && task.staticPoolIndex < 0 && d.checkDial(task.dest()) == nil { d.addToStaticPool(task) } } @@ -437,10 +438,11 @@ func (d *dialScheduler) removeFromStaticPool(idx int) { // startDial runs the given dial task in a separate goroutine. func (d *dialScheduler) startDial(task *dialTask) { - d.log.Trace("Starting p2p dial", "id", task.dest.ID(), "ip", task.dest.IP(), "flag", task.flags) - hkey := string(task.dest.ID().Bytes()) + node := task.dest() + d.log.Trace("Starting p2p dial", "id", node.ID(), "ip", node.IP(), "flag", task.flags) + hkey := string(node.ID().Bytes()) d.history.add(hkey, d.clock.Now().Add(dialHistoryExpiration)) - d.dialing[task.dest.ID()] = task + d.dialing[node.ID()] = task go func() { task.run(d) d.doneCh <- task @@ -451,39 +453,46 @@ func (d *dialScheduler) startDial(task *dialTask) { type dialTask struct { staticPoolIndex int flags connFlag + // These fields are private to the task and should not be // accessed by dialScheduler while the task is running. - dest *enode.Node + destPtr atomic.Pointer[enode.Node] lastResolved mclock.AbsTime resolveDelay time.Duration } func newDialTask(dest *enode.Node, flags connFlag) *dialTask { - return &dialTask{dest: dest, flags: flags, staticPoolIndex: -1} + t := &dialTask{flags: flags, staticPoolIndex: -1} + t.destPtr.Store(dest) + return t } type dialError struct { error } +func (t *dialTask) dest() *enode.Node { + return t.destPtr.Load() +} + func (t *dialTask) run(d *dialScheduler) { if t.needResolve() && !t.resolve(d) { return } - err := t.dial(d, t.dest) + err := t.dial(d, t.dest()) if err != nil { // For static nodes, resolve one more time if dialing fails. if _, ok := err.(*dialError); ok && t.flags&staticDialedConn != 0 { if t.resolve(d) { - t.dial(d, t.dest) + t.dial(d, t.dest()) } } } } func (t *dialTask) needResolve() bool { - return t.flags&staticDialedConn != 0 && t.dest.IP() == nil + return t.flags&staticDialedConn != 0 && t.dest().IP() == nil } // resolve attempts to find the current endpoint for the destination @@ -502,29 +511,31 @@ func (t *dialTask) resolve(d *dialScheduler) bool { if t.lastResolved > 0 && time.Duration(d.clock.Now()-t.lastResolved) < t.resolveDelay { return false } - resolved := d.resolver.Resolve(t.dest) + + node := t.dest() + resolved := d.resolver.Resolve(node) t.lastResolved = d.clock.Now() if resolved == nil { t.resolveDelay *= 2 if t.resolveDelay > maxResolveDelay { t.resolveDelay = maxResolveDelay } - d.log.Debug("Resolving node failed", "id", t.dest.ID(), "newdelay", t.resolveDelay) + d.log.Debug("Resolving node failed", "id", node.ID(), "newdelay", t.resolveDelay) return false } // The node was found. t.resolveDelay = initialResolveDelay - t.dest = resolved - d.log.Debug("Resolved node", "id", t.dest.ID(), "addr", &net.TCPAddr{IP: t.dest.IP(), Port: t.dest.TCP()}) + t.destPtr.Store(resolved) + d.log.Debug("Resolved node", "id", resolved.ID(), "addr", &net.TCPAddr{IP: resolved.IP(), Port: resolved.TCP()}) return true } // dial performs the actual connection attempt. func (t *dialTask) dial(d *dialScheduler, dest *enode.Node) error { dialMeter.Mark(1) - fd, err := d.dialer.Dial(d.ctx, t.dest) + fd, err := d.dialer.Dial(d.ctx, dest) if err != nil { - d.log.Trace("Dial error", "id", t.dest.ID(), "addr", nodeAddr(t.dest), "conn", t.flags, "err", cleanupDialErr(err)) + d.log.Trace("Dial error", "id", dest.ID(), "addr", nodeAddr(dest), "conn", t.flags, "err", cleanupDialErr(err)) dialConnectionError.Mark(1) return &dialError{err} } @@ -532,8 +543,9 @@ func (t *dialTask) dial(d *dialScheduler, dest *enode.Node) error { } func (t *dialTask) String() string { - id := t.dest.ID() - return fmt.Sprintf("%v %x %v:%d", t.flags, id[:8], t.dest.IP(), t.dest.TCP()) + node := t.dest() + id := node.ID() + return fmt.Sprintf("%v %x %v:%d", t.flags, id[:8], node.IP(), node.TCP()) } func cleanupDialErr(err error) error { diff --git a/p2p/discover/metrics.go b/p2p/discover/metrics.go index da8e9cb817..56aae24285 100644 --- a/p2p/discover/metrics.go +++ b/p2p/discover/metrics.go @@ -58,7 +58,7 @@ func newMeteredConn(conn UDPConn) UDPConn { return &meteredUdpConn{UDPConn: conn} } -// Read delegates a network read to the underlying connection, bumping the udp ingress traffic meter along the way. +// ReadFromUDP delegates a network read to the underlying connection, bumping the udp ingress traffic meter along the way. func (c *meteredUdpConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { n, addr, err = c.UDPConn.ReadFromUDP(b) ingressTrafficMeter.Mark(int64(n)) diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 988f16b01d..44b1f5305c 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -364,7 +364,7 @@ func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { return nil, err } if respN.ID() != n.ID() { - return nil, fmt.Errorf("invalid ID in response record") + return nil, errors.New("invalid ID in response record") } if respN.Seq() < n.Seq() { return n, nil // response record is older diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 8b3e33d37c..71f8d8dd08 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -442,7 +442,7 @@ func (t *UDPv5) verifyResponseNode(c *callV5, r *enr.Record, distances []uint, s } } if _, ok := seen[node.ID()]; ok { - return nil, fmt.Errorf("duplicate record") + return nil, errors.New("duplicate record") } seen[node.ID()] = struct{}{} return node, nil diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 5108910620..904a3ddec6 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -367,11 +367,11 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey // key is part of the ID nonce signature. var remotePubkey = new(ecdsa.PublicKey) if err := challenge.Node.Load((*enode.Secp256k1)(remotePubkey)); err != nil { - return nil, nil, fmt.Errorf("can't find secp256k1 key for recipient") + return nil, nil, errors.New("can't find secp256k1 key for recipient") } ephkey, err := c.sc.ephemeralKeyGen() if err != nil { - return nil, nil, fmt.Errorf("can't generate ephemeral key") + return nil, nil, errors.New("can't generate ephemeral key") } ephpubkey := EncodePubkey(&ephkey.PublicKey) auth.pubkey = ephpubkey[:] @@ -395,7 +395,7 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey // Create session keys. sec := deriveKeys(sha256.New, ephkey, remotePubkey, c.localnode.ID(), challenge.Node.ID(), cdata) if sec == nil { - return nil, nil, fmt.Errorf("key derivation failed") + return nil, nil, errors.New("key derivation failed") } return auth, sec, err } diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index 8f1c221b80..4f14d860e1 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -191,7 +191,7 @@ func (c *Client) resolveEntry(ctx context.Context, domain, hash string) (entry, func (c *Client) doResolveEntry(ctx context.Context, domain, hash string) (entry, error) { wantHash, err := b32format.DecodeString(hash) if err != nil { - return nil, fmt.Errorf("invalid base32 hash") + return nil, errors.New("invalid base32 hash") } name := hash + "." + domain txts, err := c.cfg.Resolver.LookupTXT(ctx, hash+"."+domain) diff --git a/p2p/dnsdisc/tree.go b/p2p/dnsdisc/tree.go index 7d9703a345..dfac4fb372 100644 --- a/p2p/dnsdisc/tree.go +++ b/p2p/dnsdisc/tree.go @@ -21,6 +21,7 @@ import ( "crypto/ecdsa" "encoding/base32" "encoding/base64" + "errors" "fmt" "io" "strings" @@ -341,7 +342,7 @@ func parseLinkEntry(e string) (entry, error) { func parseLink(e string) (*linkEntry, error) { if !strings.HasPrefix(e, linkPrefix) { - return nil, fmt.Errorf("wrong/missing scheme 'enrtree' in URL") + return nil, errors.New("wrong/missing scheme 'enrtree' in URL") } e = e[len(linkPrefix):] diff --git a/p2p/enode/idscheme.go b/p2p/enode/idscheme.go index fd5d868b76..6ad7f809a7 100644 --- a/p2p/enode/idscheme.go +++ b/p2p/enode/idscheme.go @@ -18,7 +18,7 @@ package enode import ( "crypto/ecdsa" - "fmt" + "errors" "io" "github.com/ethereum/go-ethereum/common/math" @@ -67,7 +67,7 @@ func (V4ID) Verify(r *enr.Record, sig []byte) error { if err := r.Load(&entry); err != nil { return err } else if len(entry) != 33 { - return fmt.Errorf("invalid public key") + return errors.New("invalid public key") } h := sha3.NewLegacyKeccak256() diff --git a/p2p/enode/nodedb.go b/p2p/enode/nodedb.go index 7e7fb69b29..6d55ce17f1 100644 --- a/p2p/enode/nodedb.go +++ b/p2p/enode/nodedb.go @@ -84,7 +84,7 @@ func OpenDB(path string) (*DB, error) { return newPersistentDB(path) } -// newMemoryNodeDB creates a new in-memory node database without a persistent backend. +// newMemoryDB creates a new in-memory node database without a persistent backend. func newMemoryDB() (*DB, error) { db, err := leveldb.Open(storage.NewMemStorage(), nil) if err != nil { @@ -93,7 +93,7 @@ func newMemoryDB() (*DB, error) { return &DB{lvl: db, quit: make(chan struct{})}, nil } -// newPersistentNodeDB creates/opens a leveldb backed persistent node database, +// newPersistentDB creates/opens a leveldb backed persistent node database, // also flushing its contents in case of a version mismatch. func newPersistentDB(path string) (*DB, error) { opts := &opt.Options{OpenFilesCacheCapacity: 5} diff --git a/p2p/nat/natpmp.go b/p2p/nat/natpmp.go index 97601c99dc..ea2d897829 100644 --- a/p2p/nat/natpmp.go +++ b/p2p/nat/natpmp.go @@ -17,6 +17,7 @@ package nat import ( + "errors" "fmt" "net" "strings" @@ -46,7 +47,7 @@ func (n *pmp) ExternalIP() (net.IP, error) { func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { if lifetime <= 0 { - return 0, fmt.Errorf("lifetime must not be <= 0") + return 0, errors.New("lifetime must not be <= 0") } // Note order of port arguments is switched between our // AddMapping and the client's AddPortMapping. diff --git a/p2p/server.go b/p2p/server.go index 8f42765a8c..5b7afb4565 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -914,13 +914,13 @@ func (srv *Server) checkInboundConn(remoteIP net.IP) error { } // Reject connections that do not match NetRestrict. if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) { - return fmt.Errorf("not in netrestrict list") + return errors.New("not in netrestrict list") } // Reject Internet peers that try too often. now := srv.clock.Now() srv.inboundHistory.expire(now, nil) if !netutil.IsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { - return fmt.Errorf("too many attempts") + return errors.New("too many attempts") } srv.inboundHistory.add(remoteIP.String(), now.Add(inboundThrottleTime)) return nil @@ -937,7 +937,7 @@ func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) c.transport = srv.newTransport(fd, dialDest.Pubkey()) } - err := srv.setupConn(c, flags, dialDest) + err := srv.setupConn(c, dialDest) if err != nil { if !c.is(inboundConn) { markDialError(err) @@ -947,7 +947,7 @@ func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) return err } -func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) error { +func (srv *Server) setupConn(c *conn, dialDest *enode.Node) error { // Prevent leftover pending conns from entering the handshake. srv.lock.Lock() running := srv.running diff --git a/p2p/server_nat.go b/p2p/server_nat.go index 354597cc7a..299d275490 100644 --- a/p2p/server_nat.go +++ b/p2p/server_nat.go @@ -127,7 +127,7 @@ func (srv *Server) portMappingLoop() { } else if !ip.Equal(lastExtIP) { log.Debug("External IP changed", "ip", extip, "interface", srv.NAT) } else { - return + continue } // Here, we either failed to get the external IP, or it has changed. lastExtIP = ip diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 63cc4936c1..17e0f75d5a 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -460,7 +460,7 @@ func startExecNodeStack() (*node.Node, error) { // decode the config confEnv := os.Getenv(envNodeConfig) if confEnv == "" { - return nil, fmt.Errorf("missing " + envNodeConfig) + return nil, errors.New("missing " + envNodeConfig) } var conf execNodeConfig if err := json.Unmarshal([]byte(confEnv), &conf); err != nil { diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index c52917fd0a..349e496b2f 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -172,7 +172,7 @@ type SimNode struct { registerOnce sync.Once } -// Close closes the underlaying node.Node to release +// Close closes the underlying node.Node to release // acquired resources. func (sn *SimNode) Close() error { return sn.node.Close() diff --git a/p2p/simulations/adapters/inproc_test.go b/p2p/simulations/adapters/inproc_test.go index 2a61508fe1..d0539ca867 100644 --- a/p2p/simulations/adapters/inproc_test.go +++ b/p2p/simulations/adapters/inproc_test.go @@ -78,7 +78,7 @@ func TestTCPPipeBidirections(t *testing.T) { } if !bytes.Equal(expected, out) { - t.Fatalf("expected %#v, got %#v", out, expected) + t.Fatalf("expected %#v, got %#v", expected, out) } else { msg := []byte(fmt.Sprintf("pong %02d", i)) if _, err := c2.Write(msg); err != nil { @@ -94,7 +94,7 @@ func TestTCPPipeBidirections(t *testing.T) { t.Fatal(err) } if !bytes.Equal(expected, out) { - t.Fatalf("expected %#v, got %#v", out, expected) + t.Fatalf("expected %#v, got %#v", expected, out) } } } diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index c53a49797b..c04308fe0b 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -282,8 +282,6 @@ func (t *TestAPI) Events(ctx context.Context) (*rpc.Subscription, error) { return case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() diff --git a/p2p/transport.go b/p2p/transport.go index 4f6bb569bf..5fc7686feb 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -19,6 +19,7 @@ package p2p import ( "bytes" "crypto/ecdsa" + "errors" "fmt" "io" "net" @@ -157,7 +158,7 @@ func readProtocolHandshake(rw MsgReader) (*protoHandshake, error) { return nil, err } if msg.Size > baseProtocolMaxMsgSize { - return nil, fmt.Errorf("message too big") + return nil, errors.New("message too big") } if msg.Code == discMsg { // Disconnect before protocol handshake is valid according to the diff --git a/params/config.go b/params/config.go index fb5175119a..439e882189 100644 --- a/params/config.go +++ b/params/config.go @@ -58,6 +58,7 @@ var ( TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 TerminalTotalDifficultyPassed: true, ShanghaiTime: newUint64(1681338455), + CancunTime: newUint64(1710338135), Ethash: new(EthashConfig), } // HoleskyChainConfig contains the chain parameters to run a node on the Holesky test network. @@ -182,6 +183,7 @@ var ( ArrowGlacierBlock: big.NewInt(0), GrayGlacierBlock: big.NewInt(0), ShanghaiTime: newUint64(0), + CancunTime: newUint64(0), TerminalTotalDifficulty: big.NewInt(0), TerminalTotalDifficultyPassed: true, } @@ -359,6 +361,8 @@ type ChainConfig struct { // TerminalTotalDifficultyPassed is a flag specifying that the network already // passed the terminal total difficulty. Its purpose is to disable legacy sync // even without having seen the TTD locally (safer long term). + // + // TODO(karalabe): Drop this field eventually (always assuming PoS mode) TerminalTotalDifficultyPassed bool `json:"terminalTotalDifficultyPassed,omitempty"` // Various consensus engines @@ -466,7 +470,7 @@ func (c *ChainConfig) Description() string { banner += fmt.Sprintf(" - Shanghai: @%-10v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/shanghai.md)\n", *c.ShanghaiTime) } if c.CancunTime != nil { - banner += fmt.Sprintf(" - Cancun: @%-10v\n", *c.CancunTime) + banner += fmt.Sprintf(" - Cancun: @%-10v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/cancun.md)\n", *c.CancunTime) } if c.PragueTime != nil { banner += fmt.Sprintf(" - Prague: @%-10v\n", *c.PragueTime) @@ -642,7 +646,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { lastFork.name, cur.name, cur.block) } else { return fmt.Errorf("unsupported fork ordering: %v not enabled, but %v enabled at timestamp %v", - lastFork.name, cur.name, cur.timestamp) + lastFork.name, cur.name, *cur.timestamp) } // Fork (whether defined by block or timestamp) must follow the fork definition sequence @@ -652,7 +656,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { lastFork.name, lastFork.block, cur.name, cur.block) } else if lastFork.timestamp != nil && *lastFork.timestamp > *cur.timestamp { return fmt.Errorf("unsupported fork ordering: %v enabled at timestamp %v, but %v enabled at timestamp %v", - lastFork.name, lastFork.timestamp, cur.name, cur.timestamp) + lastFork.name, *lastFork.timestamp, cur.name, *cur.timestamp) } // Timestamp based forks can follow block based ones, but not the other way around @@ -909,6 +913,8 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules if chainID == nil { chainID = new(big.Int) } + // disallow setting Merge out of order + isMerge = isMerge && c.IsLondon(num) return Rules{ ChainID: new(big.Int).Set(chainID), IsHomestead: c.IsHomestead(num), @@ -922,9 +928,9 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules IsBerlin: c.IsBerlin(num), IsLondon: c.IsLondon(num), IsMerge: isMerge, - IsShanghai: c.IsShanghai(num, timestamp), - IsCancun: c.IsCancun(num, timestamp), - IsPrague: c.IsPrague(num, timestamp), - IsVerkle: c.IsVerkle(num, timestamp), + IsShanghai: isMerge && c.IsShanghai(num, timestamp), + IsCancun: isMerge && c.IsCancun(num, timestamp), + IsPrague: isMerge && c.IsPrague(num, timestamp), + IsVerkle: isMerge && c.IsVerkle(num, timestamp), } } diff --git a/params/version.go b/params/version.go index a18d6dc914..a49385da7d 100644 --- a/params/version.go +++ b/params/version.go @@ -22,8 +22,8 @@ import ( const ( VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 12 // Patch version component of the current release + VersionMinor = 14 // Minor version component of the current release + VersionPatch = 0 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/rlp/iterator.go b/rlp/iterator.go index 6be574572e..95bd3f2582 100644 --- a/rlp/iterator.go +++ b/rlp/iterator.go @@ -23,7 +23,6 @@ type listIterator struct { } // NewListIterator creates an iterator for the (list) represented by data -// TODO: Consider removing this implementation, as it is no longer used. func NewListIterator(data RawValue) (*listIterator, error) { k, t, c, err := readKind(data) if err != nil { diff --git a/rlp/unsafe.go b/rlp/unsafe.go index 2152ba35fc..10868caaf2 100644 --- a/rlp/unsafe.go +++ b/rlp/unsafe.go @@ -26,10 +26,5 @@ import ( // byteArrayBytes returns a slice of the byte array v. func byteArrayBytes(v reflect.Value, length int) []byte { - var s []byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&s)) - hdr.Data = v.UnsafeAddr() - hdr.Cap = length - hdr.Len = length - return s + return unsafe.Slice((*byte)(unsafe.Pointer(v.UnsafeAddr())), length) } diff --git a/rpc/client.go b/rpc/client.go index 2b0016db8f..eef6ee21cf 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -70,7 +70,7 @@ type BatchElem struct { // discarded. Result interface{} // Error is set if the server returns an error for this request, or if - // unmarshaling into Result fails. It is not set for I/O errors. + // unmarshalling into Result fails. It is not set for I/O errors. Error error } diff --git a/rpc/handler.go b/rpc/handler.go index f44e4d7b01..792581cbc0 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -324,7 +324,7 @@ func (h *handler) addRequestOp(op *requestOp) { } } -// removeRequestOps stops waiting for the given request IDs. +// removeRequestOp stops waiting for the given request IDs. func (h *handler) removeRequestOp(op *requestOp) { for _, id := range op.ids { delete(h.respWait, string(id)) diff --git a/rpc/http.go b/rpc/http.go index 741fa1c0eb..f4b99429ef 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -33,8 +33,8 @@ import ( ) const ( - maxRequestContentLength = 1024 * 1024 * 5 - contentType = "application/json" + defaultBodyLimit = 5 * 1024 * 1024 + contentType = "application/json" ) // https://www.jsonrpc.org/historical/json-rpc-over-http.html#id13 @@ -236,7 +236,7 @@ func (hc *httpConn) doRequest(ctx context.Context, msg interface{}) (io.ReadClos if _, err := buf.ReadFrom(resp.Body); err == nil { body = buf.Bytes() } - + resp.Body.Close() return nil, HTTPError{ Status: resp.Status, StatusCode: resp.StatusCode, @@ -253,8 +253,8 @@ type httpServerConn struct { r *http.Request } -func newHTTPServerConn(r *http.Request, w http.ResponseWriter) ServerCodec { - body := io.LimitReader(r.Body, maxRequestContentLength) +func (s *Server) newHTTPServerConn(r *http.Request, w http.ResponseWriter) ServerCodec { + body := io.LimitReader(r.Body, int64(s.httpBodyLimit)) conn := &httpServerConn{Reader: body, Writer: w, r: r} encoder := func(v any, isErrorResponse bool) error { @@ -312,7 +312,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) return } - if code, err := validateRequest(r); err != nil { + if code, err := s.validateRequest(r); err != nil { http.Error(w, err.Error(), code) return } @@ -330,19 +330,19 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // until EOF, writes the response to w, and orders the server to process a // single request. w.Header().Set("content-type", contentType) - codec := newHTTPServerConn(r, w) + codec := s.newHTTPServerConn(r, w) defer codec.close() s.serveSingleRequest(ctx, codec) } // validateRequest returns a non-zero response code and error message if the // request is invalid. -func validateRequest(r *http.Request) (int, error) { +func (s *Server) validateRequest(r *http.Request) (int, error) { if r.Method == http.MethodPut || r.Method == http.MethodDelete { return http.StatusMethodNotAllowed, errors.New("method not allowed") } - if r.ContentLength > maxRequestContentLength { - err := fmt.Errorf("content length too large (%d>%d)", r.ContentLength, maxRequestContentLength) + if r.ContentLength > int64(s.httpBodyLimit) { + err := fmt.Errorf("content length too large (%d>%d)", r.ContentLength, s.httpBodyLimit) return http.StatusRequestEntityTooLarge, err } // Allow OPTIONS (regardless of content-type) diff --git a/rpc/http_test.go b/rpc/http_test.go index 584842a9aa..ad86ca15ae 100644 --- a/rpc/http_test.go +++ b/rpc/http_test.go @@ -40,11 +40,13 @@ func confirmStatusCode(t *testing.T, got, want int) { func confirmRequestValidationCode(t *testing.T, method, contentType, body string, expectedStatusCode int) { t.Helper() + + s := NewServer() request := httptest.NewRequest(method, "http://url.com", strings.NewReader(body)) if len(contentType) > 0 { request.Header.Set("Content-Type", contentType) } - code, err := validateRequest(request) + code, err := s.validateRequest(request) if code == 0 { if err != nil { t.Errorf("validation: got error %v, expected nil", err) @@ -64,7 +66,7 @@ func TestHTTPErrorResponseWithPut(t *testing.T) { } func TestHTTPErrorResponseWithMaxContentLength(t *testing.T) { - body := make([]rune, maxRequestContentLength+1) + body := make([]rune, defaultBodyLimit+1) confirmRequestValidationCode(t, http.MethodPost, contentType, string(body), http.StatusRequestEntityTooLarge) } @@ -104,7 +106,7 @@ func TestHTTPResponseWithEmptyGet(t *testing.T) { // This checks that maxRequestContentLength is not applied to the response of a request. func TestHTTPRespBodyUnlimited(t *testing.T) { - const respLength = maxRequestContentLength * 3 + const respLength = defaultBodyLimit * 3 s := NewServer() defer s.Stop() diff --git a/rpc/server.go b/rpc/server.go index 2742adf07b..e2f9120aa2 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -51,13 +51,15 @@ type Server struct { run atomic.Bool batchItemLimit int batchResponseLimit int + httpBodyLimit int } // NewServer creates a new server instance with no registered handlers. func NewServer() *Server { server := &Server{ - idgen: randomIDGenerator(), - codecs: make(map[ServerCodec]struct{}), + idgen: randomIDGenerator(), + codecs: make(map[ServerCodec]struct{}), + httpBodyLimit: defaultBodyLimit, } server.run.Store(true) // Register the default service providing meta information about the RPC service such @@ -78,6 +80,13 @@ func (s *Server) SetBatchLimits(itemLimit, maxResponseSize int) { s.batchResponseLimit = maxResponseSize } +// SetHTTPBodyLimit sets the size limit for HTTP requests. +// +// This method should be called before processing any requests via ServeHTTP. +func (s *Server) SetHTTPBodyLimit(limit int) { + s.httpBodyLimit = limit +} + // RegisterName creates a service for the given receiver type under the given name. When no // methods on the given receiver match the criteria to be either a RPC method or a // subscription an error is returned. Otherwise a new service is created and added to the diff --git a/rpc/subscription.go b/rpc/subscription.go index 9cb0727547..d3dff32a27 100644 --- a/rpc/subscription.go +++ b/rpc/subscription.go @@ -145,12 +145,6 @@ func (n *Notifier) Notify(id ID, data any) error { return nil } -// Closed returns a channel that is closed when the RPC connection is closed. -// Deprecated: use subscription error channel -func (n *Notifier) Closed() <-chan interface{} { - return n.h.conn.closed() -} - // takeSubscription returns the subscription (if one has been created). No subscription can // be created after this call. func (n *Notifier) takeSubscription() *Subscription { diff --git a/rpc/testservice_test.go b/rpc/testservice_test.go index 7d873af667..69199e21b7 100644 --- a/rpc/testservice_test.go +++ b/rpc/testservice_test.go @@ -195,10 +195,7 @@ func (s *notificationTestService) SomeSubscription(ctx context.Context, n, val i return } } - select { - case <-notifier.Closed(): - case <-subscription.Err(): - } + <-subscription.Err() if s.unsubscribed != nil { s.unsubscribed <- string(subscription.ID) } diff --git a/rpc/types.go b/rpc/types.go index f88c37c59d..d124081786 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -19,6 +19,7 @@ package rpc import ( "context" "encoding/json" + "errors" "fmt" "math" "strings" @@ -104,7 +105,7 @@ func (bn *BlockNumber) UnmarshalJSON(data []byte) error { return err } if blckNum > math.MaxInt64 { - return fmt.Errorf("block number larger than int64") + return errors.New("block number larger than int64") } *bn = BlockNumber(blckNum) return nil @@ -154,7 +155,7 @@ func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { err := json.Unmarshal(data, &e) if err == nil { if e.BlockNumber != nil && e.BlockHash != nil { - return fmt.Errorf("cannot specify both BlockHash and BlockNumber, choose one or the other") + return errors.New("cannot specify both BlockHash and BlockNumber, choose one or the other") } bnh.BlockNumber = e.BlockNumber bnh.BlockHash = e.BlockHash @@ -202,7 +203,7 @@ func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { return err } if blckNum > math.MaxInt64 { - return fmt.Errorf("blocknumber too high") + return errors.New("blocknumber too high") } bn := BlockNumber(blckNum) bnh.BlockNumber = &bn diff --git a/rpc/types_test.go b/rpc/types_test.go index 617f441d91..2fa74f9899 100644 --- a/rpc/types_test.go +++ b/rpc/types_test.go @@ -45,9 +45,11 @@ func TestBlockNumberJSONUnmarshal(t *testing.T) { 11: {`"pending"`, false, PendingBlockNumber}, 12: {`"latest"`, false, LatestBlockNumber}, 13: {`"earliest"`, false, EarliestBlockNumber}, - 14: {`someString`, true, BlockNumber(0)}, - 15: {`""`, true, BlockNumber(0)}, - 16: {``, true, BlockNumber(0)}, + 14: {`"safe"`, false, SafeBlockNumber}, + 15: {`"finalized"`, false, FinalizedBlockNumber}, + 16: {`someString`, true, BlockNumber(0)}, + 17: {`""`, true, BlockNumber(0)}, + 18: {``, true, BlockNumber(0)}, } for i, test := range tests { @@ -87,18 +89,22 @@ func TestBlockNumberOrHash_UnmarshalJSON(t *testing.T) { 11: {`"pending"`, false, BlockNumberOrHashWithNumber(PendingBlockNumber)}, 12: {`"latest"`, false, BlockNumberOrHashWithNumber(LatestBlockNumber)}, 13: {`"earliest"`, false, BlockNumberOrHashWithNumber(EarliestBlockNumber)}, - 14: {`someString`, true, BlockNumberOrHash{}}, - 15: {`""`, true, BlockNumberOrHash{}}, - 16: {``, true, BlockNumberOrHash{}}, - 17: {`"0x0000000000000000000000000000000000000000000000000000000000000000"`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, - 18: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, - 19: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":false}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, - 20: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":true}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), true)}, - 21: {`{"blockNumber":"0x1"}`, false, BlockNumberOrHashWithNumber(1)}, - 22: {`{"blockNumber":"pending"}`, false, BlockNumberOrHashWithNumber(PendingBlockNumber)}, - 23: {`{"blockNumber":"latest"}`, false, BlockNumberOrHashWithNumber(LatestBlockNumber)}, - 24: {`{"blockNumber":"earliest"}`, false, BlockNumberOrHashWithNumber(EarliestBlockNumber)}, - 25: {`{"blockNumber":"0x1", "blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, true, BlockNumberOrHash{}}, + 14: {`"safe"`, false, BlockNumberOrHashWithNumber(SafeBlockNumber)}, + 15: {`"finalized"`, false, BlockNumberOrHashWithNumber(FinalizedBlockNumber)}, + 16: {`someString`, true, BlockNumberOrHash{}}, + 17: {`""`, true, BlockNumberOrHash{}}, + 18: {``, true, BlockNumberOrHash{}}, + 19: {`"0x0000000000000000000000000000000000000000000000000000000000000000"`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, + 20: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, + 21: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":false}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, + 22: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":true}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), true)}, + 23: {`{"blockNumber":"0x1"}`, false, BlockNumberOrHashWithNumber(1)}, + 24: {`{"blockNumber":"pending"}`, false, BlockNumberOrHashWithNumber(PendingBlockNumber)}, + 25: {`{"blockNumber":"latest"}`, false, BlockNumberOrHashWithNumber(LatestBlockNumber)}, + 26: {`{"blockNumber":"earliest"}`, false, BlockNumberOrHashWithNumber(EarliestBlockNumber)}, + 27: {`{"blockNumber":"safe"}`, false, BlockNumberOrHashWithNumber(SafeBlockNumber)}, + 28: {`{"blockNumber":"finalized"}`, false, BlockNumberOrHashWithNumber(FinalizedBlockNumber)}, + 29: {`{"blockNumber":"0x1", "blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, true, BlockNumberOrHash{}}, } for i, test := range tests { @@ -133,6 +139,8 @@ func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { {"pending", int64(PendingBlockNumber)}, {"latest", int64(LatestBlockNumber)}, {"earliest", int64(EarliestBlockNumber)}, + {"safe", int64(SafeBlockNumber)}, + {"finalized", int64(FinalizedBlockNumber)}, } for _, test := range tests { test := test @@ -160,6 +168,8 @@ func TestBlockNumberOrHash_StringAndUnmarshal(t *testing.T) { BlockNumberOrHashWithNumber(PendingBlockNumber), BlockNumberOrHashWithNumber(LatestBlockNumber), BlockNumberOrHashWithNumber(EarliestBlockNumber), + BlockNumberOrHashWithNumber(SafeBlockNumber), + BlockNumberOrHashWithNumber(FinalizedBlockNumber), BlockNumberOrHashWithNumber(32), BlockNumberOrHashWithHash(common.Hash{0xaa}, false), } diff --git a/rpc/websocket_test.go b/rpc/websocket_test.go index d3e15d94c9..8d2bd9d802 100644 --- a/rpc/websocket_test.go +++ b/rpc/websocket_test.go @@ -97,7 +97,7 @@ func TestWebsocketLargeCall(t *testing.T) { // This call sends slightly less than the limit and should work. var result echoResult - arg := strings.Repeat("x", maxRequestContentLength-200) + arg := strings.Repeat("x", defaultBodyLimit-200) if err := client.Call(&result, "test_echo", arg, 1); err != nil { t.Fatalf("valid call didn't work: %v", err) } @@ -106,7 +106,7 @@ func TestWebsocketLargeCall(t *testing.T) { } // This call sends twice the allowed size and shouldn't work. - arg = strings.Repeat("x", maxRequestContentLength*2) + arg = strings.Repeat("x", defaultBodyLimit*2) err = client.Call(&result, "test_echo", arg) if err == nil { t.Fatal("no error for too large call") diff --git a/signer/core/api.go b/signer/core/api.go index ef8c136625..a32f24cb18 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -631,7 +631,7 @@ func (api *SignerAPI) SignGnosisSafeTx(ctx context.Context, signerAddress common } } typedData := gnosisTx.ToTypedData() - // might aswell error early. + // might as well error early. // we are expected to sign. If our calculated hash does not match what they want, // The gnosis safetx input contains a 'safeTxHash' which is the expected safeTxHash that sighash, _, err := apitypes.TypedDataAndHash(typedData) diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 6bfcd2a727..e28f059106 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -708,7 +708,7 @@ func formatPrimitiveValue(encType string, encValue interface{}) (string, error) func (t Types) validate() error { for typeKey, typeArr := range t { if len(typeKey) == 0 { - return fmt.Errorf("empty type key") + return errors.New("empty type key") } for i, typeObj := range typeArr { if len(typeObj.Type) == 0 { diff --git a/signer/core/signed_data.go b/signer/core/signed_data.go index 3c2b6f5d45..f8b3c9d86d 100644 --- a/signer/core/signed_data.go +++ b/signer/core/signed_data.go @@ -260,7 +260,7 @@ func fromHex(data any) ([]byte, error) { return nil, fmt.Errorf("wrong type %T", data) } -// typeDataRequest tries to convert the data into a SignDataRequest. +// typedDataRequest tries to convert the data into a SignDataRequest. func typedDataRequest(data any) (*SignDataRequest, error) { var typedData apitypes.TypedData if td, ok := data.(apitypes.TypedData); ok { @@ -302,7 +302,7 @@ func (api *SignerAPI) EcRecover(ctx context.Context, data hexutil.Bytes, sig hex // Note, the signature must conform to the secp256k1 curve R, S and V values, where // the V value must be 27 or 28 for legacy reasons. // - // https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover + // https://geth.ethereum.org/docs/tools/clef/apis#account-ecrecover if len(sig) != 65 { return common.Address{}, errors.New("signature must be 65 bytes long") } diff --git a/tests/block_test.go b/tests/block_test.go index aa6f27b8f3..fb355085fd 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -61,14 +61,14 @@ func TestBlockchain(t *testing.T) { // which run natively, so there's no reason to run them here. } -// TestExecutionSpec runs the test fixtures from execution-spec-tests. -func TestExecutionSpec(t *testing.T) { - if !common.FileExist(executionSpecDir) { - t.Skipf("directory %s does not exist", executionSpecDir) +// TestExecutionSpecBlocktests runs the test fixtures from execution-spec-tests. +func TestExecutionSpecBlocktests(t *testing.T) { + if !common.FileExist(executionSpecBlockchainTestDir) { + t.Skipf("directory %s does not exist", executionSpecBlockchainTestDir) } bt := new(testMatcher) - bt.walk(t, executionSpecDir, func(t *testing.T, name string, test *BlockTest) { + bt.walk(t, executionSpecBlockchainTestDir, func(t *testing.T, name string, test *BlockTest) { execBlockTest(t, bt, test) }) } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 2b6ba6db03..04a04fdc28 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -34,14 +34,15 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) // A BlockTest checks handling of entire blocks. @@ -57,8 +58,8 @@ func (t *BlockTest) UnmarshalJSON(in []byte) error { type btJSON struct { Blocks []btBlock `json:"blocks"` Genesis btHeader `json:"genesisBlockHeader"` - Pre core.GenesisAlloc `json:"pre"` - Post core.GenesisAlloc `json:"postState"` + Pre types.GenesisAlloc `json:"pre"` + Post types.GenesisAlloc `json:"postState"` BestBlock common.UnprefixedHash `json:"lastblockhash"` Network string `json:"network"` SealEngine string `json:"sealEngine"` @@ -109,7 +110,7 @@ type btHeaderMarshaling struct { ExcessBlobGas *math.HexOrDecimal64 } -func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, postCheck func(error, *core.BlockChain)) (result error) { +func (t *BlockTest) Run(snapshotter bool, scheme string, tracer *tracing.Hooks, postCheck func(error, *core.BlockChain)) (result error) { config, ok := Forks[t.json.Network] if !ok { return UnsupportedForkError{t.json.Network} @@ -117,7 +118,7 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, po // import pre accounts & construct test genesis block & state root var ( db = rawdb.NewMemoryDatabase() - tconf = &trie.Config{ + tconf = &triedb.Config{ Preimages: true, } ) @@ -128,7 +129,7 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, po } // Commit genesis state gspec := t.genesis(config) - triedb := trie.NewDatabase(db, tconf) + triedb := triedb.NewDatabase(db, tconf) gblock, err := gspec.Commit(db, triedb) if err != nil { return err diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 6b5ca90880..dcafebb265 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" "golang.org/x/exp/slices" ) @@ -56,7 +57,7 @@ func (f *fuzzer) readInt() uint64 { } func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { - trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) size := f.readInt() // Fill it with some fluff diff --git a/tests/init_test.go b/tests/init_test.go index 3ab15e7658..e9bb99dc7d 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -34,15 +34,16 @@ import ( ) var ( - baseDir = filepath.Join(".", "testdata") - blockTestDir = filepath.Join(baseDir, "BlockchainTests") - stateTestDir = filepath.Join(baseDir, "GeneralStateTests") - legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests") - transactionTestDir = filepath.Join(baseDir, "TransactionTests") - rlpTestDir = filepath.Join(baseDir, "RLPTests") - difficultyTestDir = filepath.Join(baseDir, "BasicTests") - executionSpecDir = filepath.Join(".", "spec-tests", "fixtures") - benchmarksDir = filepath.Join(".", "evm-benchmarks", "benchmarks") + baseDir = filepath.Join(".", "testdata") + blockTestDir = filepath.Join(baseDir, "BlockchainTests") + stateTestDir = filepath.Join(baseDir, "GeneralStateTests") + legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests") + transactionTestDir = filepath.Join(baseDir, "TransactionTests") + rlpTestDir = filepath.Join(baseDir, "RLPTests") + difficultyTestDir = filepath.Join(baseDir, "BasicTests") + executionSpecBlockchainTestDir = filepath.Join(".", "spec-tests", "fixtures", "blockchain_tests") + executionSpecStateTestDir = filepath.Join(".", "spec-tests", "fixtures", "state_tests") + benchmarksDir = filepath.Join(".", "evm-benchmarks", "benchmarks") ) func readJSON(reader io.Reader, value interface{}) error { diff --git a/tests/state_test.go b/tests/state_test.go index 3a7e83ae3d..c6f4f8ed61 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -30,20 +30,16 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/holiman/uint256" ) -func TestState(t *testing.T) { - t.Parallel() - - st := new(testMatcher) +func initMatcher(st *testMatcher) { // Long tests: st.slow(`^stAttackTest/ContractCreationSpam`) st.slow(`^stBadOpcode/badOpcodes`) @@ -62,72 +58,102 @@ func TestState(t *testing.T) { // Broken tests: // EOF is not part of cancun st.skipLoad(`^stEOF/`) +} - // For Istanbul, older tests were moved into LegacyTests +func TestState(t *testing.T) { + t.Parallel() + + st := new(testMatcher) + initMatcher(st) for _, dir := range []string{ filepath.Join(baseDir, "EIPTests", "StateTests"), stateTestDir, - legacyStateTestDir, benchmarksDir, } { st.walk(t, dir, func(t *testing.T, name string, test *StateTest) { - if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 { - t.Skip("test (randomly) skipped on 32-bit windows") - return - } - for _, subtest := range test.Subtests() { - subtest := subtest - key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) + execStateTest(t, st, test) + }) + } +} - t.Run(key+"/hash/trie", func(t *testing.T) { - withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - var result error - test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { - result = st.checkFailure(t, err) - }) - return result - }) +// TestLegacyState tests some older tests, which were moved to the folder +// 'LegacyTests' for the Istanbul fork. +func TestLegacyState(t *testing.T) { + st := new(testMatcher) + initMatcher(st) + st.walk(t, legacyStateTestDir, func(t *testing.T, name string, test *StateTest) { + execStateTest(t, st, test) + }) +} + +// TestExecutionSpecState runs the test fixtures from execution-spec-tests. +func TestExecutionSpecState(t *testing.T) { + if !common.FileExist(executionSpecStateTestDir) { + t.Skipf("directory %s does not exist", executionSpecStateTestDir) + } + st := new(testMatcher) + + st.walk(t, executionSpecStateTestDir, func(t *testing.T, name string, test *StateTest) { + execStateTest(t, st, test) + }) +} + +func execStateTest(t *testing.T, st *testMatcher, test *StateTest) { + if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 { + t.Skip("test (randomly) skipped on 32-bit windows") + return + } + for _, subtest := range test.Subtests() { + subtest := subtest + key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) + + t.Run(key+"/hash/trie", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, state *StateTestState) { + result = st.checkFailure(t, err) }) - t.Run(key+"/hash/snap", func(t *testing.T) { - withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - var result error - test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { - if snaps != nil && state != nil { - if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil { - result = err - return - } - } - result = st.checkFailure(t, err) - }) - return result - }) + return result + }) + }) + t.Run(key+"/hash/snap", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, state *StateTestState) { + if state.Snapshots != nil && state.StateDB != nil { + if _, err := state.Snapshots.Journal(state.StateDB.IntermediateRoot(false)); err != nil { + result = err + return + } + } + result = st.checkFailure(t, err) }) - t.Run(key+"/path/trie", func(t *testing.T) { - withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - var result error - test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { - result = st.checkFailure(t, err) - }) - return result - }) + return result + }) + }) + t.Run(key+"/path/trie", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, state *StateTestState) { + result = st.checkFailure(t, err) }) - t.Run(key+"/path/snap", func(t *testing.T) { - withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - var result error - test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { - if snaps != nil && state != nil { - if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil { - result = err - return - } - } - result = st.checkFailure(t, err) - }) - return result - }) + return result + }) + }) + t.Run(key+"/path/snap", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, state *StateTestState) { + if state.Snapshots != nil && state.StateDB != nil { + if _, err := state.Snapshots.Journal(state.StateDB.IntermediateRoot(false)); err != nil { + result = err + return + } + } + result = st.checkFailure(t, err) }) - } + return result + }) }) } } @@ -151,7 +177,7 @@ func withTrace(t *testing.T, gasLimit uint64, test func(vm.Config) error) { } buf := new(bytes.Buffer) w := bufio.NewWriter(buf) - config.Tracer = logger.NewJSONLogger(&logger.Config{}, w) + config.Tracer = logger.NewJSONLogger(&logger.Config{}, w).Hooks() err2 := test(config) if !reflect.DeepEqual(err, err2) { t.Errorf("different error for second run: %v", err2) @@ -222,8 +248,8 @@ func runBenchmark(b *testing.B, t *StateTest) { vmconfig.ExtraEips = eips block := t.genesis(config).ToBlock() - triedb, _, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false, rawdb.HashScheme) - defer triedb.Close() + state := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false, rawdb.HashScheme) + defer state.Close() var baseFee *big.Int if rules.IsLondon { @@ -261,7 +287,7 @@ func runBenchmark(b *testing.B, t *StateTest) { context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash context.BaseFee = baseFee - evm := vm.NewEVM(context, txContext, statedb, config, vmconfig) + evm := vm.NewEVM(context, txContext, state.StateDB, config, vmconfig) // Create "contract" for sender to cache code analysis. sender := vm.NewContract(vm.AccountRef(msg.From), vm.AccountRef(msg.From), @@ -274,8 +300,8 @@ func runBenchmark(b *testing.B, t *StateTest) { ) b.ResetTimer() for n := 0; n < b.N; n++ { - snapshot := statedb.Snapshot() - statedb.Prepare(rules, msg.From, context.Coinbase, msg.To, vm.ActivePrecompiles(rules), msg.AccessList) + snapshot := state.StateDB.Snapshot() + state.StateDB.Prepare(rules, msg.From, context.Coinbase, msg.To, vm.ActivePrecompiles(rules), msg.AccessList) b.StartTimer() start := time.Now() @@ -288,10 +314,10 @@ func runBenchmark(b *testing.B, t *StateTest) { b.StopTimer() elapsed += uint64(time.Since(start)) - refund += statedb.GetRefund() + refund += state.StateDB.GetRefund() gasUsed += msg.GasLimit - leftOverGas - statedb.RevertToSnapshot(snapshot) + state.StateDB.RevertToSnapshot(snapshot) } if elapsed < 1 { elapsed = 1 diff --git a/tests/state_test_util.go b/tests/state_test_util.go index cd34c9355e..367688e57f 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -33,15 +33,16 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) @@ -64,7 +65,7 @@ func (t *StateTest) UnmarshalJSON(in []byte) error { type stJSON struct { Env stEnv `json:"env"` - Pre core.GenesisAlloc `json:"pre"` + Pre types.GenesisAlloc `json:"pre"` Tx stTransaction `json:"transaction"` Out hexutil.Bytes `json:"out"` Post map[string][]stPostState `json:"post"` @@ -194,20 +195,14 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error { } // Run executes a specific subtest and verifies the post-state and logs -func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, snaps *snapshot.Tree, state *state.StateDB)) (result error) { - triedb, snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme) - +func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, st *StateTestState)) (result error) { + st, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme) // Invoke the callback at the end of function for further analysis. defer func() { - postCheck(result, snaps, statedb) - - if triedb != nil { - triedb.Close() - } - if snaps != nil { - snaps.Release() - } + postCheck(result, &st) + st.Close() }() + checkedErr := t.checkError(subtest, err) if checkedErr != nil { return checkedErr @@ -224,23 +219,24 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bo if root != common.Hash(post.Root) { return fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) } - if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { + if logs := rlpHash(st.StateDB.Logs()); logs != common.Hash(post.Logs) { return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) } - statedb, _ = state.New(root, statedb.Database(), snaps) + st.StateDB, _ = state.New(root, st.StateDB.Database(), st.Snapshots) return nil } -// RunNoVerify runs a specific subtest and returns the statedb and post-state root -func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB, common.Hash, error) { +// RunNoVerify runs a specific subtest and returns the statedb and post-state root. +// Remember to call state.Close after verifying the test result! +func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (st StateTestState, root common.Hash, err error) { config, eips, err := GetChainConfig(subtest.Fork) if err != nil { - return nil, nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} + return st, common.Hash{}, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips block := t.genesis(config).ToBlock() - triedb, snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter, scheme) + st = MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter, scheme) var baseFee *big.Int if config.IsLondon(new(big.Int)) { @@ -254,8 +250,18 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post, baseFee) if err != nil { - triedb.Close() - return nil, nil, nil, common.Hash{}, err + return st, common.Hash{}, err + } + + { // Blob transactions may be present after the Cancun fork. + // In production, + // - the header is verified against the max in eip4844.go:VerifyEIP4844Header + // - the block body is verified against the header in block_validator.go:ValidateBody + // Here, we just do this shortcut smaller fix, since state tests do not + // utilize those codepaths + if len(msg.BlobHashes)*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { + return st, common.Hash{}, errors.New("blob gas exceeds maximum") + } } // Try to recover tx with current signer @@ -263,13 +269,10 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh var ttx types.Transaction err := ttx.UnmarshalBinary(post.TxBytes) if err != nil { - triedb.Close() - return nil, nil, nil, common.Hash{}, err + return st, common.Hash{}, err } - if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil { - triedb.Close() - return nil, nil, nil, common.Hash{}, err + return st, common.Hash{}, err } } @@ -290,78 +293,32 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh if config.IsCancun(new(big.Int), block.Time()) && t.json.Env.ExcessBlobGas != nil { context.BlobBaseFee = eip4844.CalcBlobFee(*t.json.Env.ExcessBlobGas) } - evm := vm.NewEVM(context, txContext, statedb, config, vmconfig) - - { // Blob transactions may be present after the Cancun fork. - // In production, - // - the header is verified against the max in eip4844.go:VerifyEIP4844Header - // - the block body is verified against the header in block_validator.go:ValidateBody - // Here, we just do this shortcut smaller fix, since state tests do not - // utilize those codepaths - if len(msg.BlobHashes)*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { - return nil, nil, nil, common.Hash{}, errors.New("blob gas exceeds maximum") - } - } + evm := vm.NewEVM(context, txContext, st.StateDB, config, vmconfig) // Execute the message. - snapshot := statedb.Snapshot() + snapshot := st.StateDB.Snapshot() gaspool := new(core.GasPool) gaspool.AddGas(block.GasLimit()) _, err = core.ApplyMessage(evm, msg, gaspool) if err != nil { - statedb.RevertToSnapshot(snapshot) + st.StateDB.RevertToSnapshot(snapshot) } // Add 0-value mining reward. This only makes a difference in the cases // where // - the coinbase self-destructed, or // - there are only 'bad' transactions, which aren't executed. In those cases, // the coinbase gets no txfee, so isn't created, and thus needs to be touched - statedb.AddBalance(block.Coinbase(), new(uint256.Int), state.BalanceChangeUnspecified) + st.StateDB.AddBalance(block.Coinbase(), new(uint256.Int), tracing.BalanceChangeUnspecified) // Commit state mutations into database. - root, _ := statedb.Commit(block.NumberU64(), config.IsEIP158(block.Number())) - return triedb, snaps, statedb, root, err + root, _ = st.StateDB.Commit(block.NumberU64(), config.IsEIP158(block.Number())) + return st, root, err } func (t *StateTest) gasLimit(subtest StateSubtest) uint64 { return t.json.Tx.GasLimit[t.json.Post[subtest.Fork][subtest.Index].Indexes.Gas] } -func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB) { - tconf := &trie.Config{Preimages: true} - if scheme == rawdb.HashScheme { - tconf.HashDB = hashdb.Defaults - } else { - tconf.PathDB = pathdb.Defaults - } - triedb := trie.NewDatabase(db, tconf) - sdb := state.NewDatabaseWithNodeDB(db, triedb) - statedb, _ := state.New(types.EmptyRootHash, sdb, nil) - for addr, a := range accounts { - statedb.SetCode(addr, a.Code) - statedb.SetNonce(addr, a.Nonce) - statedb.SetBalance(addr, uint256.MustFromBig(a.Balance), state.BalanceChangeUnspecified) - for k, v := range a.Storage { - statedb.SetState(addr, k, v) - } - } - // Commit and re-open to start with a clean state. - root, _ := statedb.Commit(0, false) - - var snaps *snapshot.Tree - if snapshotter { - snapconfig := snapshot.Config{ - CacheSize: 1, - Recovery: false, - NoBuild: false, - AsyncBuild: false, - } - snaps, _ = snapshot.New(snapconfig, db, triedb, root) - } - statedb, _ = state.New(root, sdb, snaps) - return triedb, snaps, statedb -} - func (t *StateTest) genesis(config *params.ChainConfig) *core.Genesis { genesis := &core.Genesis{ Config: config, @@ -478,3 +435,61 @@ func rlpHash(x interface{}) (h common.Hash) { func vmTestBlockHash(n uint64) common.Hash { return common.BytesToHash(crypto.Keccak256([]byte(big.NewInt(int64(n)).String()))) } + +// StateTestState groups all the state database objects together for use in tests. +type StateTestState struct { + StateDB *state.StateDB + TrieDB *triedb.Database + Snapshots *snapshot.Tree +} + +// MakePreState creates a state containing the given allocation. +func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, snapshotter bool, scheme string) StateTestState { + tconf := &triedb.Config{Preimages: true} + if scheme == rawdb.HashScheme { + tconf.HashDB = hashdb.Defaults + } else { + tconf.PathDB = pathdb.Defaults + } + triedb := triedb.NewDatabase(db, tconf) + sdb := state.NewDatabaseWithNodeDB(db, triedb) + statedb, _ := state.New(types.EmptyRootHash, sdb, nil) + for addr, a := range accounts { + statedb.SetCode(addr, a.Code) + statedb.SetNonce(addr, a.Nonce) + statedb.SetBalance(addr, uint256.MustFromBig(a.Balance), tracing.BalanceChangeUnspecified) + for k, v := range a.Storage { + statedb.SetState(addr, k, v) + } + } + // Commit and re-open to start with a clean state. + root, _ := statedb.Commit(0, false) + + // If snapshot is requested, initialize the snapshotter and use it in state. + var snaps *snapshot.Tree + if snapshotter { + snapconfig := snapshot.Config{ + CacheSize: 1, + Recovery: false, + NoBuild: false, + AsyncBuild: false, + } + snaps, _ = snapshot.New(snapconfig, db, triedb, root) + } + statedb, _ = state.New(root, sdb, snaps) + return StateTestState{statedb, triedb, snaps} +} + +// Close should be called when the state is no longer needed, ie. after running the test. +func (st *StateTestState) Close() { + if st.TrieDB != nil { + st.TrieDB.Close() + st.TrieDB = nil + } + if st.Snapshots != nil { + // Need to call Disable here to quit the snapshot generator goroutine. + st.Snapshots.Disable() + st.Snapshots.Release() + st.Snapshots = nil + } +} diff --git a/trie/committer.go b/trie/committer.go index 92163cdb3b..4e2f7b8bd6 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -154,12 +154,12 @@ func (c *committer) store(path []byte, n node) node { return hash } -// mptResolver the children resolver in merkle-patricia-tree. -type mptResolver struct{} +// MerkleResolver the children resolver in merkle-patricia-tree. +type MerkleResolver struct{} // ForEach implements childResolver, decodes the provided node and // traverses the children inside. -func (resolver mptResolver) ForEach(node []byte, onChild func(common.Hash)) { +func (resolver MerkleResolver) ForEach(node []byte, onChild func(common.Hash)) { forGatherChildren(mustDecodeNodeUnsafe(nil, node), onChild) } diff --git a/trie/database_test.go b/trie/database_test.go index d508c65533..aed508b368 100644 --- a/trie/database_test.go +++ b/trie/database_test.go @@ -17,24 +17,136 @@ package trie import ( + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb/database" ) -// newTestDatabase initializes the trie database with specified scheme. -func newTestDatabase(diskdb ethdb.Database, scheme string) *Database { - config := &Config{Preimages: false} - if scheme == rawdb.HashScheme { - config.HashDB = &hashdb.Config{ - CleanCacheSize: 0, - } // disable clean cache - } else { - config.PathDB = &pathdb.Config{ - CleanCacheSize: 0, - DirtyCacheSize: 0, - } // disable clean/dirty cache - } - return NewDatabase(diskdb, config) +// testReader implements database.Reader interface, providing function to +// access trie nodes. +type testReader struct { + db ethdb.Database + scheme string + nodes []*trienode.MergedNodeSet // sorted from new to old +} + +// Node implements database.Reader interface, retrieving trie node with +// all available cached layers. +func (r *testReader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) { + // Check the node presence with the cached layer, from latest to oldest. + for _, nodes := range r.nodes { + if _, ok := nodes.Sets[owner]; !ok { + continue + } + n, ok := nodes.Sets[owner].Nodes[string(path)] + if !ok { + continue + } + if n.IsDeleted() || n.Hash != hash { + return nil, &MissingNodeError{Owner: owner, Path: path, NodeHash: hash} + } + return n.Blob, nil + } + // Check the node presence in database. + return rawdb.ReadTrieNode(r.db, owner, path, hash, r.scheme), nil +} + +// testDb implements database.Database interface, using for testing purpose. +type testDb struct { + disk ethdb.Database + root common.Hash + scheme string + nodes map[common.Hash]*trienode.MergedNodeSet + parents map[common.Hash]common.Hash +} + +func newTestDatabase(diskdb ethdb.Database, scheme string) *testDb { + return &testDb{ + disk: diskdb, + root: types.EmptyRootHash, + scheme: scheme, + nodes: make(map[common.Hash]*trienode.MergedNodeSet), + parents: make(map[common.Hash]common.Hash), + } +} + +func (db *testDb) Reader(stateRoot common.Hash) (database.Reader, error) { + nodes, _ := db.dirties(stateRoot, true) + return &testReader{db: db.disk, scheme: db.scheme, nodes: nodes}, nil +} + +func (db *testDb) Preimage(hash common.Hash) []byte { + return rawdb.ReadPreimage(db.disk, hash) +} + +func (db *testDb) InsertPreimage(preimages map[common.Hash][]byte) { + rawdb.WritePreimages(db.disk, preimages) +} + +func (db *testDb) Scheme() string { return db.scheme } + +func (db *testDb) Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet) error { + if root == parent { + return nil + } + if _, ok := db.nodes[root]; ok { + return nil + } + db.parents[root] = parent + db.nodes[root] = nodes + return nil +} + +func (db *testDb) dirties(root common.Hash, topToBottom bool) ([]*trienode.MergedNodeSet, []common.Hash) { + var ( + pending []*trienode.MergedNodeSet + roots []common.Hash + ) + for { + if root == db.root { + break + } + nodes, ok := db.nodes[root] + if !ok { + break + } + if topToBottom { + pending = append(pending, nodes) + roots = append(roots, root) + } else { + pending = append([]*trienode.MergedNodeSet{nodes}, pending...) + roots = append([]common.Hash{root}, roots...) + } + root = db.parents[root] + } + return pending, roots +} + +func (db *testDb) Commit(root common.Hash) error { + if root == db.root { + return nil + } + pending, roots := db.dirties(root, false) + for i, nodes := range pending { + for owner, set := range nodes.Sets { + if owner == (common.Hash{}) { + continue + } + set.ForEachWithOrder(func(path string, n *trienode.Node) { + rawdb.WriteTrieNode(db.disk, owner, []byte(path), n.Hash, n.Blob, db.scheme) + }) + } + nodes.Sets[common.Hash{}].ForEachWithOrder(func(path string, n *trienode.Node) { + rawdb.WriteTrieNode(db.disk, common.Hash{}, []byte(path), n.Hash, n.Blob, db.scheme) + }) + db.root = roots[i] + } + for _, root := range roots { + delete(db.nodes, root) + delete(db.parents, root) + } + return nil } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 9679b49ca7..41e83f6cb6 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -30,7 +30,7 @@ import ( ) func TestEmptyIterator(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) iter := trie.MustNodeIterator(nil) seen := make(map[string]struct{}) @@ -43,7 +43,7 @@ func TestEmptyIterator(t *testing.T) { } func TestIterator(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase(), nil) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -60,7 +60,7 @@ func TestIterator(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) found := make(map[string]string) @@ -86,7 +86,7 @@ func (k *kv) cmp(other *kv) int { } func TestIteratorLargeData(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) vals := make(map[string]*kv) for i := byte(0); i < 255; i++ { @@ -205,7 +205,7 @@ var testdata2 = []kvs{ } func TestIteratorSeek(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for _, val := range testdata1 { trie.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -246,22 +246,22 @@ func checkIteratorOrder(want []kvs, it *Iterator) error { } func TestDifferenceIterator(t *testing.T) { - dba := NewDatabase(rawdb.NewMemoryDatabase(), nil) + dba := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) triea := NewEmpty(dba) for _, val := range testdata1 { triea.MustUpdate([]byte(val.k), []byte(val.v)) } rootA, nodesA, _ := triea.Commit(false) - dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) + dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)) triea, _ = New(TrieID(rootA), dba) - dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil) + dbb := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } rootB, nodesB, _ := trieb.Commit(false) - dbb.Update(rootB, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesB), nil) + dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB)) trieb, _ = New(TrieID(rootB), dbb) found := make(map[string]string) @@ -288,22 +288,22 @@ func TestDifferenceIterator(t *testing.T) { } func TestUnionIterator(t *testing.T) { - dba := NewDatabase(rawdb.NewMemoryDatabase(), nil) + dba := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) triea := NewEmpty(dba) for _, val := range testdata1 { triea.MustUpdate([]byte(val.k), []byte(val.v)) } rootA, nodesA, _ := triea.Commit(false) - dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) + dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)) triea, _ = New(TrieID(rootA), dba) - dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil) + dbb := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } rootB, nodesB, _ := trieb.Commit(false) - dbb.Update(rootB, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesB), nil) + dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB)) trieb, _ = New(TrieID(rootB), dbb) di, _ := NewUnionIterator([]NodeIterator{triea.MustNodeIterator(nil), trieb.MustNodeIterator(nil)}) @@ -341,7 +341,8 @@ func TestUnionIterator(t *testing.T) { } func TestIteratorNoDups(t *testing.T) { - tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + tr := NewEmpty(db) for _, val := range testdata1 { tr.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -365,9 +366,9 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { tr.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := tr.Commit(false) - tdb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) if !memonly { - tdb.Commit(root, false) + tdb.Commit(root) } tr, _ = New(TrieID(root), tdb) wantNodeCount := checkIteratorNoDups(t, tr.MustNodeIterator(nil), nil) @@ -481,9 +482,9 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool, scheme strin break } } - triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) if !memonly { - triedb.Commit(root, false) + triedb.Commit(root) } var ( barNodeBlob []byte @@ -555,8 +556,8 @@ func testIteratorNodeBlob(t *testing.T, scheme string) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) - triedb.Commit(root, false) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Commit(root) var found = make(map[common.Hash][]byte) trie, _ = New(TrieID(root), triedb) diff --git a/trie/proof.go b/trie/proof.go index a526a53402..fd892fb4be 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -389,7 +389,7 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error } else { if bytes.Compare(cld.Key, key[pos:]) > 0 { // The key of fork shortnode is greater than the - // path(it belongs to the range), unset the entrie + // path(it belongs to the range), unset the entries // branch. The parent must be a fullnode. fn := parent.(*fullNode) fn.Children[key[pos-1]] = nil diff --git a/trie/proof_test.go b/trie/proof_test.go index 59ae201cea..5471d0efa6 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -94,7 +94,7 @@ func TestProof(t *testing.T) { } func TestOneElementProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) updateString(trie, "k", "v") for i, prover := range makeProvers(trie) { proof := prover([]byte("k")) @@ -145,7 +145,7 @@ func TestBadProof(t *testing.T) { // Tests that missing keys can also be proven. The test explicitly uses a single // entry trie and checks for missing keys both before and after the single entry. func TestMissingKeyProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) updateString(trie, "k", "v") for i, key := range []string{"a", "j", "l", "z"} { @@ -343,7 +343,7 @@ func TestOneElementRangeProof(t *testing.T) { } // Test the mini trie with only a single element. - tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + tinyTrie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) entry := &kv{randBytes(32), randBytes(20), false} tinyTrie.MustUpdate(entry.k, entry.v) @@ -414,7 +414,7 @@ func TestAllElementsProof(t *testing.T) { // TestSingleSideRangeProof tests the range starts from zero. func TestSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -520,7 +520,7 @@ func TestBadRangeProof(t *testing.T) { // TestGappedRangeProof focuses on the small trie with embedded nodes. // If the gapped node is embedded in the trie, it should be detected too. func TestGappedRangeProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) var entries []*kv // Sorted entries for i := byte(0); i < 10; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -592,7 +592,7 @@ func TestSameSideProofs(t *testing.T) { } func TestHasRightElement(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -934,7 +934,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { } func randomTrie(n int) (*Trie, map[string]*kv) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) vals := make(map[string]*kv) for i := byte(0); i < 100; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -953,7 +953,7 @@ func randomTrie(n int) (*Trie, map[string]*kv) { } func nonRandomTrie(n int) (*Trie, map[string]*kv) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) vals := make(map[string]*kv) max := uint64(0xffffffffffffffff) for i := uint64(0); i < uint64(n); i++ { @@ -978,7 +978,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) { common.Hex2Bytes("02"), common.Hex2Bytes("03"), } - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for i, key := range keys { trie.MustUpdate(key, vals[i]) } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 7f0685e306..efd4dfb5d3 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb/database" ) // SecureTrie is the old name of StateTrie. @@ -29,7 +30,7 @@ type SecureTrie = StateTrie // NewSecure creates a new StateTrie. // Deprecated: use NewStateTrie. -func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { +func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db database.Database) (*SecureTrie, error) { id := &ID{ StateRoot: stateRoot, Owner: owner, @@ -50,7 +51,7 @@ func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db *D // StateTrie is not safe for concurrent use. type StateTrie struct { trie Trie - preimages *preimageStore + db database.Database hashKeyBuf [common.HashLength]byte secKeyCache map[string][]byte secKeyCacheOwner *StateTrie // Pointer to self, replace the key cache on mismatch @@ -61,7 +62,7 @@ type StateTrie struct { // If root is the zero hash or the sha3 hash of an empty string, the // trie is initially empty. Otherwise, New will panic if db is nil // and returns MissingNodeError if the root node cannot be found. -func NewStateTrie(id *ID, db *Database) (*StateTrie, error) { +func NewStateTrie(id *ID, db database.Database) (*StateTrie, error) { if db == nil { panic("trie.NewStateTrie called without a database") } @@ -69,7 +70,7 @@ func NewStateTrie(id *ID, db *Database) (*StateTrie, error) { if err != nil { return nil, err } - return &StateTrie{trie: *trie, preimages: db.preimages}, nil + return &StateTrie{trie: *trie, db: db}, nil } // MustGet returns the value for key stored in the trie. @@ -210,10 +211,7 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte { if key, ok := t.getSecKeyCache()[string(shaKey)]; ok { return key } - if t.preimages == nil { - return nil - } - return t.preimages.preimage(common.BytesToHash(shaKey)) + return t.db.Preimage(common.BytesToHash(shaKey)) } // Commit collects all dirty nodes in the trie and replaces them with the @@ -226,13 +224,11 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte { func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { - if t.preimages != nil { - preimages := make(map[common.Hash][]byte) - for hk, key := range t.secKeyCache { - preimages[common.BytesToHash([]byte(hk))] = key - } - t.preimages.insertPreimage(preimages) + preimages := make(map[common.Hash][]byte) + for hk, key := range t.secKeyCache { + preimages[common.BytesToHash([]byte(hk))] = key } + t.db.InsertPreimage(preimages) t.secKeyCache = make(map[string][]byte) } // Commit the trie and return its modified nodeset. @@ -249,7 +245,7 @@ func (t *StateTrie) Hash() common.Hash { func (t *StateTrie) Copy() *StateTrie { return &StateTrie{ trie: *t.trie.Copy(), - preimages: t.preimages, + db: t.db, secKeyCache: t.secKeyCache, } } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 2087866d38..0a6fd688b7 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -31,14 +31,14 @@ import ( ) func newEmptySecure() *StateTrie { - trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) return trie } // makeTestStateTrie creates a large enough secure trie for testing. -func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { +func makeTestStateTrie() (*testDb, *StateTrie, map[string][]byte) { // Create an empty trie - triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) + triedb := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb) // Fill it with some arbitrary data @@ -61,7 +61,7 @@ func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { } } root, nodes, _ := trie.Commit(false) - if err := triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } // Re-create the trie based on the new state diff --git a/trie/stacktrie_fuzzer_test.go b/trie/stacktrie_fuzzer_test.go index 1b3f9dbe9c..50b5c4de52 100644 --- a/trie/stacktrie_fuzzer_test.go +++ b/trie/stacktrie_fuzzer_test.go @@ -42,10 +42,10 @@ func fuzz(data []byte, debugging bool) { var ( input = bytes.NewReader(data) spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()} - dbA = NewDatabase(rawdb.NewDatabase(spongeA), nil) + dbA = newTestDatabase(rawdb.NewDatabase(spongeA), rawdb.HashScheme) trieA = NewEmpty(dbA) spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()} - dbB = NewDatabase(rawdb.NewDatabase(spongeB), nil) + dbB = newTestDatabase(rawdb.NewDatabase(spongeB), rawdb.HashScheme) options = NewStackTrieOptions().WithWriter(func(path []byte, hash common.Hash, blob []byte) { rawdb.WriteTrieNode(spongeB, common.Hash{}, path, hash, blob, dbB.Scheme()) @@ -87,10 +87,10 @@ func fuzz(data []byte, debugging bool) { panic(err) } if nodes != nil { - dbA.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + dbA.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) } // Flush memdb -> disk (sponge) - dbA.Commit(rootA, false) + dbA.Commit(rootA) // Stacktrie requires sorted insertion slices.SortFunc(vals, (*kv).cmp) diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index 909a77062a..3a0e1cb260 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -223,7 +223,7 @@ func TestStackTrieInsertAndHash(t *testing.T) { func TestSizeBug(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + nt := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -238,7 +238,7 @@ func TestSizeBug(t *testing.T) { func TestEmptyBug(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + nt := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -264,7 +264,7 @@ func TestEmptyBug(t *testing.T) { func TestValLength56(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + nt := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -289,7 +289,7 @@ func TestValLength56(t *testing.T) { // which causes a lot of node-within-node. This case was found via fuzzing. func TestUpdateSmallNodes(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + nt := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) kvs := []struct { K string V string @@ -317,7 +317,7 @@ func TestUpdateSmallNodes(t *testing.T) { func TestUpdateVariableKeys(t *testing.T) { t.SkipNow() st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + nt := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) kvs := []struct { K string V string diff --git a/trie/sync_test.go b/trie/sync_test.go index 585181b48c..7bc68c041f 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -32,7 +32,7 @@ import ( ) // makeTestTrie create a sample test trie to test node-wise reconstruction. -func makeTestTrie(scheme string) (ethdb.Database, *Database, *StateTrie, map[string][]byte) { +func makeTestTrie(scheme string) (ethdb.Database, *testDb, *StateTrie, map[string][]byte) { // Create an empty trie db := rawdb.NewMemoryDatabase() triedb := newTestDatabase(db, scheme) @@ -58,10 +58,10 @@ func makeTestTrie(scheme string) (ethdb.Database, *Database, *StateTrie, map[str } } root, nodes, _ := trie.Commit(false) - if err := triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } - if err := triedb.Commit(root, false); err != nil { + if err := triedb.Commit(root); err != nil { panic(err) } // Re-create the trie based on the new state @@ -143,7 +143,7 @@ func TestEmptySync(t *testing.T) { emptyD, _ := New(TrieID(types.EmptyRootHash), dbD) for i, trie := range []*Trie{emptyA, emptyB, emptyC, emptyD} { - sync := NewSync(trie.Hash(), memorydb.New(), nil, []*Database{dbA, dbB, dbC, dbD}[i].Scheme()) + sync := NewSync(trie.Hash(), memorydb.New(), nil, []*testDb{dbA, dbB, dbC, dbD}[i].Scheme()) if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, paths, nodes, codes) } @@ -684,11 +684,11 @@ func testSyncOrdering(t *testing.T, scheme string) { } } } -func syncWith(t *testing.T, root common.Hash, db ethdb.Database, srcDb *Database) { +func syncWith(t *testing.T, root common.Hash, db ethdb.Database, srcDb *testDb) { syncWithHookWriter(t, root, db, srcDb, nil) } -func syncWithHookWriter(t *testing.T, root common.Hash, db ethdb.Database, srcDb *Database, hookWriter ethdb.KeyValueWriter) { +func syncWithHookWriter(t *testing.T, root common.Hash, db ethdb.Database, srcDb *testDb, hookWriter ethdb.KeyValueWriter) { // Create a destination trie and sync with the scheduler sched := NewSync(root, db, nil, srcDb.Scheme()) @@ -771,10 +771,10 @@ func testSyncMovingTarget(t *testing.T, scheme string) { diff[string(key)] = val } root, nodes, _ := srcTrie.Commit(false) - if err := srcDb.Update(root, preRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil { panic(err) } - if err := srcDb.Commit(root, false); err != nil { + if err := srcDb.Commit(root); err != nil { panic(err) } preRoot = root @@ -796,10 +796,10 @@ func testSyncMovingTarget(t *testing.T, scheme string) { reverted[k] = val } root, nodes, _ = srcTrie.Commit(false) - if err := srcDb.Update(root, preRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil { panic(err) } - if err := srcDb.Commit(root, false); err != nil { + if err := srcDb.Commit(root); err != nil { panic(err) } srcTrie, _ = NewStateTrie(TrieID(root), srcDb) @@ -854,10 +854,10 @@ func testPivotMove(t *testing.T, scheme string, tiny bool) { writeFn([]byte{0x13, 0x44}, nil, srcTrie, stateA) rootA, nodesA, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil); err != nil { + if err := srcTrieDB.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootA, false); err != nil { + if err := srcTrieDB.Commit(rootA); err != nil { panic(err) } // Create a destination trie and sync with the scheduler @@ -873,10 +873,10 @@ func testPivotMove(t *testing.T, scheme string, tiny bool) { writeFn([]byte{0x01, 0x24}, nil, srcTrie, stateB) rootB, nodesB, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootB, rootA, 0, trienode.NewWithNodeSet(nodesB), nil); err != nil { + if err := srcTrieDB.Update(rootB, rootA, trienode.NewWithNodeSet(nodesB)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootB, false); err != nil { + if err := srcTrieDB.Commit(rootB); err != nil { panic(err) } syncWith(t, rootB, destDisk, srcTrieDB) @@ -891,10 +891,10 @@ func testPivotMove(t *testing.T, scheme string, tiny bool) { writeFn([]byte{0x13, 0x44}, nil, srcTrie, stateC) rootC, nodesC, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootC, rootB, 0, trienode.NewWithNodeSet(nodesC), nil); err != nil { + if err := srcTrieDB.Update(rootC, rootB, trienode.NewWithNodeSet(nodesC)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootC, false); err != nil { + if err := srcTrieDB.Commit(rootC); err != nil { panic(err) } syncWith(t, rootC, destDisk, srcTrieDB) @@ -960,10 +960,10 @@ func testSyncAbort(t *testing.T, scheme string) { writeFn(key, val, srcTrie, stateA) rootA, nodesA, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil); err != nil { + if err := srcTrieDB.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootA, false); err != nil { + if err := srcTrieDB.Commit(rootA); err != nil { panic(err) } // Create a destination trie and sync with the scheduler @@ -977,10 +977,10 @@ func testSyncAbort(t *testing.T, scheme string) { deleteFn(key, srcTrie, stateB) rootB, nodesB, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootB, rootA, 0, trienode.NewWithNodeSet(nodesB), nil); err != nil { + if err := srcTrieDB.Update(rootB, rootA, trienode.NewWithNodeSet(nodesB)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootB, false); err != nil { + if err := srcTrieDB.Commit(rootB); err != nil { panic(err) } @@ -1004,10 +1004,10 @@ func testSyncAbort(t *testing.T, scheme string) { writeFn(key, val, srcTrie, stateC) rootC, nodesC, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootC, rootB, 0, trienode.NewWithNodeSet(nodesC), nil); err != nil { + if err := srcTrieDB.Update(rootC, rootB, trienode.NewWithNodeSet(nodesC)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootC, false); err != nil { + if err := srcTrieDB.Commit(rootC); err != nil { panic(err) } syncWith(t, rootC, destDisk, srcTrieDB) diff --git a/trie/tracer_test.go b/trie/tracer_test.go index acb8c2f6bf..27e42d497a 100644 --- a/trie/tracer_test.go +++ b/trie/tracer_test.go @@ -61,7 +61,7 @@ func TestTrieTracer(t *testing.T) { // Tests if the trie diffs are tracked correctly. Tracer should capture // all non-leaf dirty nodes, no matter the node is embedded or not. func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { - db := NewDatabase(rawdb.NewMemoryDatabase(), nil) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie := NewEmpty(db) // Determine all new nodes are tracked @@ -71,7 +71,7 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { insertSet := copySet(trie.tracer.inserts) // copy before commit deleteSet := copySet(trie.tracer.deletes) // copy before commit root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) seen := setKeys(iterNodes(db, root)) if !compareSet(insertSet, seen) { @@ -104,7 +104,8 @@ func TestTrieTracerNoop(t *testing.T) { } func testTrieTracerNoop(t *testing.T, vals []struct{ k, v string }) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + trie := NewEmpty(db) for _, val := range vals { trie.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -128,7 +129,7 @@ func TestAccessList(t *testing.T) { func testAccessList(t *testing.T, vals []struct{ k, v string }) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase(), nil) + db = newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie = NewEmpty(db) orig = trie.Copy() ) @@ -137,7 +138,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -152,7 +153,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), randBytes(32)) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -170,7 +171,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate(key, randBytes(32)) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -185,7 +186,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(key), nil) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -200,7 +201,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), nil) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -211,7 +212,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { // Tests origin values won't be tracked in Iterator or Prover func TestAccessListLeak(t *testing.T) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase(), nil) + db = newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie = NewEmpty(db) ) // Create trie from scratch @@ -219,7 +220,7 @@ func TestAccessListLeak(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) var cases = []struct { op func(tr *Trie) @@ -262,14 +263,14 @@ func TestAccessListLeak(t *testing.T) { // in its parent due to the smaller size of the original tree node. func TestTinyTree(t *testing.T) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase(), nil) + db = newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie = NewEmpty(db) ) for _, val := range tiny { trie.MustUpdate([]byte(val.k), randBytes(32)) } root, set, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(set), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(set)) parent := root trie, _ = New(TrieID(root), db) @@ -278,7 +279,7 @@ func TestTinyTree(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, set, _ = trie.Commit(false) - db.Update(root, parent, 0, trienode.NewWithNodeSet(set), nil) + db.Update(root, parent, trienode.NewWithNodeSet(set)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, set); err != nil { @@ -312,7 +313,7 @@ func forNodes(tr *Trie) map[string][]byte { return nodes } -func iterNodes(db *Database, root common.Hash) map[string][]byte { +func iterNodes(db *testDb, root common.Hash) map[string][]byte { tr, _ := New(TrieID(root), db) return forNodes(tr) } diff --git a/trie/trie.go b/trie/trie.go index 07467ac69c..12764e18d1 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb/database" ) // Trie is a Merkle Patricia Trie. Use New to create a trie that sits on @@ -79,7 +80,7 @@ func (t *Trie) Copy() *Trie { // zero hash or the sha3 hash of an empty string, then trie is initially // empty, otherwise, the root node must be present in database or returns // a MissingNodeError if not. -func New(id *ID, db *Database) (*Trie, error) { +func New(id *ID, db database.Database) (*Trie, error) { reader, err := newTrieReader(id.StateRoot, id.Owner, db) if err != nil { return nil, err @@ -100,7 +101,7 @@ func New(id *ID, db *Database) (*Trie, error) { } // NewEmpty is a shortcut to create empty tree. It's mostly used in tests. -func NewEmpty(db *Database) *Trie { +func NewEmpty(db database.Database) *Trie { tr, _ := New(TrieID(types.EmptyRootHash), db) return tr } diff --git a/trie/trie_reader.go b/trie/trie_reader.go index 4215964559..42bc4316fe 100644 --- a/trie/trie_reader.go +++ b/trie/trie_reader.go @@ -21,31 +21,19 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/ethereum/go-ethereum/triedb/database" ) -// Reader wraps the Node method of a backing trie store. -type Reader interface { - // Node retrieves the trie node blob with the provided trie identifier, node path and - // the corresponding node hash. No error will be returned if the node is not found. - // - // When looking up nodes in the account trie, 'owner' is the zero hash. For contract - // storage trie nodes, 'owner' is the hash of the account address that containing the - // storage. - // - // TODO(rjl493456442): remove the 'hash' parameter, it's redundant in PBSS. - Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) -} - // trieReader is a wrapper of the underlying node reader. It's not safe // for concurrent usage. type trieReader struct { owner common.Hash - reader Reader + reader database.Reader banned map[string]struct{} // Marker to prevent node from being accessed, for tests } // newTrieReader initializes the trie reader with the given node reader. -func newTrieReader(stateRoot, owner common.Hash, db *Database) (*trieReader, error) { +func newTrieReader(stateRoot, owner common.Hash, db database.Database) (*trieReader, error) { if stateRoot == (common.Hash{}) || stateRoot == types.EmptyRootHash { if stateRoot == (common.Hash{}) { log.Error("Zero state root hash!") @@ -85,17 +73,22 @@ func (r *trieReader) node(path []byte, hash common.Hash) ([]byte, error) { return blob, nil } -// trieLoader implements triestate.TrieLoader for constructing tries. -type trieLoader struct { - db *Database +// MerkleLoader implements triestate.TrieLoader for constructing tries. +type MerkleLoader struct { + db database.Database +} + +// NewMerkleLoader creates the merkle trie loader. +func NewMerkleLoader(db database.Database) *MerkleLoader { + return &MerkleLoader{db: db} } // OpenTrie opens the main account trie. -func (l *trieLoader) OpenTrie(root common.Hash) (triestate.Trie, error) { +func (l *MerkleLoader) OpenTrie(root common.Hash) (triestate.Trie, error) { return New(TrieID(root), l.db) } // OpenStorageTrie opens the storage trie of an account. -func (l *trieLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) { +func (l *MerkleLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) { return New(StorageTrieID(stateRoot, addrHash, root), l.db) } diff --git a/trie/trie_test.go b/trie/trie_test.go index fcbd552e22..920594fdd2 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -25,6 +25,7 @@ import ( "io" "math/rand" "reflect" + "sort" "testing" "testing/quick" @@ -46,7 +47,7 @@ func init() { } func TestEmptyTrie(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) res := trie.Hash() exp := types.EmptyRootHash if res != exp { @@ -55,7 +56,7 @@ func TestEmptyTrie(t *testing.T) { } func TestNull(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) key := make([]byte, 32) value := []byte("test") trie.MustUpdate(key, value) @@ -95,10 +96,10 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) { updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) if !memonly { - triedb.Commit(root, false) + triedb.Commit(root) } trie, _ = New(TrieID(root), triedb) @@ -167,7 +168,7 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) { } func TestInsert(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -179,7 +180,7 @@ func TestInsert(t *testing.T) { t.Errorf("case 1: exp %x got %x", exp, root) } - trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie = NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") @@ -190,7 +191,7 @@ func TestInsert(t *testing.T) { } func TestGet(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase(), nil) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie := NewEmpty(db) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -209,13 +210,14 @@ func TestGet(t *testing.T) { return } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) } } func TestDelete(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -242,7 +244,7 @@ func TestDelete(t *testing.T) { } func TestEmptyValues(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -266,7 +268,7 @@ func TestEmptyValues(t *testing.T) { } func TestReplication(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase(), nil) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -281,7 +283,7 @@ func TestReplication(t *testing.T) { updateString(trie, val.k, val.v) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) // create a new trie on top of the database and check that lookups work. trie2, err := New(TrieID(root), db) @@ -300,7 +302,7 @@ func TestReplication(t *testing.T) { // recreate the trie after commit if nodes != nil { - db.Update(hash, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(hash, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) } trie2, err = New(TrieID(hash), db) if err != nil { @@ -327,13 +329,13 @@ func TestReplication(t *testing.T) { } func TestLargeValue(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) trie.MustUpdate([]byte("key1"), []byte{99, 99, 99, 99}) trie.MustUpdate([]byte("key2"), bytes.Repeat([]byte{1}, 32)) trie.Hash() } -// TestRandomCases tests som cases that were found via random fuzzing +// TestRandomCases tests some cases that were found via random fuzzing func TestRandomCases(t *testing.T) { var rt = []randTestStep{ {op: 6, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 0 @@ -531,7 +533,7 @@ func runRandTest(rt randTest) error { case opCommit: root, nodes, _ := tr.Commit(true) if nodes != nil { - triedb.Update(root, origin, 0, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, origin, trienode.NewWithNodeSet(nodes)) } newtr, err := New(TrieID(root), triedb) if err != nil { @@ -554,7 +556,7 @@ func runRandTest(rt randTest) error { checktr.MustUpdate(it.Key, it.Value) } if tr.Hash() != checktr.Hash() { - rt[i].err = fmt.Errorf("hash mismatch in opItercheckhash") + rt[i].err = errors.New("hash mismatch in opItercheckhash") } case opNodeDiff: var ( @@ -592,19 +594,19 @@ func runRandTest(rt randTest) error { } } if len(insertExp) != len(tr.tracer.inserts) { - rt[i].err = fmt.Errorf("insert set mismatch") + rt[i].err = errors.New("insert set mismatch") } if len(deleteExp) != len(tr.tracer.deletes) { - rt[i].err = fmt.Errorf("delete set mismatch") + rt[i].err = errors.New("delete set mismatch") } for insert := range tr.tracer.inserts { if _, present := insertExp[insert]; !present { - rt[i].err = fmt.Errorf("missing inserted node") + rt[i].err = errors.New("missing inserted node") } } for del := range tr.tracer.deletes { if _, present := deleteExp[del]; !present { - rt[i].err = fmt.Errorf("missing deleted node") + rt[i].err = errors.New("missing deleted node") } } } @@ -632,7 +634,7 @@ func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) } const benchElemCount = 20000 func benchGet(b *testing.B) { - triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) + triedb := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie := NewEmpty(triedb) k := make([]byte, 32) for i := 0; i < benchElemCount; i++ { @@ -651,7 +653,7 @@ func benchGet(b *testing.B) { } func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) k := make([]byte, 32) b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -683,7 +685,7 @@ func BenchmarkHash(b *testing.B) { // entries, then adding N more. addresses, accounts := makeAccounts(2 * b.N) // Insert the accounts into the trie and hash it - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) i := 0 for ; i < len(addresses)/2; i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) @@ -714,7 +716,7 @@ func BenchmarkCommitAfterHash(b *testing.B) { func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { // Make the random benchmark deterministic addresses, accounts := makeAccounts(b.N) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -728,7 +730,7 @@ func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { func TestTinyTrie(t *testing.T) { // Create a realistic account trie to hash _, accounts := makeAccounts(5) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) trie.MustUpdate(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3]) if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root { t.Errorf("1: got %x, exp %x", root, exp) @@ -741,7 +743,7 @@ func TestTinyTrie(t *testing.T) { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { t.Errorf("3: got %x, exp %x", root, exp) } - checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + checktr := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) it := NewIterator(trie.MustNodeIterator(nil)) for it.Next() { checktr.MustUpdate(it.Key, it.Value) @@ -754,7 +756,7 @@ func TestTinyTrie(t *testing.T) { func TestCommitAfterHash(t *testing.T) { // Create a realistic account trie to hash addresses, accounts := makeAccounts(1000) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -808,6 +810,8 @@ type spongeDb struct { sponge hash.Hash id string journal []string + keys []string + values map[string]string } func (s *spongeDb) Has(key []byte) (bool, error) { panic("implement me") } @@ -831,12 +835,27 @@ func (s *spongeDb) Put(key []byte, value []byte) error { valbrief = valbrief[:8] } s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, keybrief, len(value), valbrief)) - s.sponge.Write(key) - s.sponge.Write(value) + + if s.values == nil { + s.sponge.Write(key) + s.sponge.Write(value) + } else { + s.keys = append(s.keys, string(key)) + s.values[string(key)] = string(value) + } return nil } func (s *spongeDb) NewIterator(prefix []byte, start []byte) ethdb.Iterator { panic("implement me") } +func (s *spongeDb) Flush() { + // Bottom-up, the longest path first + sort.Sort(sort.Reverse(sort.StringSlice(s.keys))) + for _, key := range s.keys { + s.sponge.Write([]byte(key)) + s.sponge.Write([]byte(s.values[key])) + } +} + // spongeBatch is a dummy batch which immediately writes to the underlying spongedb type spongeBatch struct { db *spongeDb @@ -861,14 +880,14 @@ func TestCommitSequence(t *testing.T) { count int expWriteSeqHash []byte }{ - {20, common.FromHex("873c78df73d60e59d4a2bcf3716e8bfe14554549fea2fc147cb54129382a8066")}, - {200, common.FromHex("ba03d891bb15408c940eea5ee3d54d419595102648d02774a0268d892add9c8e")}, - {2000, common.FromHex("f7a184f20df01c94f09537401d11e68d97ad0c00115233107f51b9c287ce60c7")}, + {20, common.FromHex("330b0afae2853d96b9f015791fbe0fb7f239bf65f335f16dfc04b76c7536276d")}, + {200, common.FromHex("5162b3735c06b5d606b043a3ee8adbdbbb408543f4966bca9dcc63da82684eeb")}, + {2000, common.FromHex("4574cd8e6b17f3fe8ad89140d1d0bf4f1bd7a87a8ac3fb623b33550544c77635")}, } { addresses, accounts := makeAccounts(tc.count) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} - db := NewDatabase(rawdb.NewDatabase(s), nil) + db := newTestDatabase(rawdb.NewDatabase(s), rawdb.HashScheme) trie := NewEmpty(db) // Fill the trie with elements for i := 0; i < tc.count; i++ { @@ -876,9 +895,9 @@ func TestCommitSequence(t *testing.T) { } // Flush trie -> database root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) - db.Commit(root, false) + db.Commit(root) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { t.Errorf("test %d, disk write sequence wrong:\ngot %x exp %x\n", i, got, exp) } @@ -892,14 +911,14 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { count int expWriteSeqHash []byte }{ - {20, common.FromHex("8e4a01548551d139fa9e833ebc4e66fc1ba40a4b9b7259d80db32cff7b64ebbc")}, - {200, common.FromHex("6869b4e7b95f3097a19ddb30ff735f922b915314047e041614df06958fc50554")}, - {2000, common.FromHex("444200e6f4e2df49f77752f629a96ccf7445d4698c164f962bbd85a0526ef424")}, + {20, common.FromHex("8016650c7a50cf88485fd06cde52d634a89711051107f00d21fae98234f2f13d")}, + {200, common.FromHex("dde92ca9812e068e6982d04b40846dc65a61a9fd4996fc0f55f2fde172a8e13c")}, + {2000, common.FromHex("ab553a7f9aff82e3929c382908e30ef7dd17a332933e92ba3fe873fc661ef382")}, } { prng := rand.New(rand.NewSource(int64(i))) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} - db := NewDatabase(rawdb.NewDatabase(s), nil) + db := newTestDatabase(rawdb.NewDatabase(s), rawdb.HashScheme) trie := NewEmpty(db) // Fill the trie with elements for i := 0; i < tc.count; i++ { @@ -917,9 +936,9 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { } // Flush trie -> database root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) - db.Commit(root, false) + db.Commit(root) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { t.Fatalf("test %d, disk write sequence wrong:\ngot %x exp %x\n", i, got, exp) } @@ -930,17 +949,26 @@ func TestCommitSequenceStackTrie(t *testing.T) { for count := 1; count < 200; count++ { prng := rand.New(rand.NewSource(int64(count))) // This spongeDb is used to check the sequence of disk-db-writes - s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} - db := NewDatabase(rawdb.NewDatabase(s), nil) + s := &spongeDb{ + sponge: sha3.NewLegacyKeccak256(), + id: "a", + values: make(map[string]string), + } + db := newTestDatabase(rawdb.NewDatabase(s), rawdb.HashScheme) trie := NewEmpty(db) - // Another sponge is used for the stacktrie commits - stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} + // Another sponge is used for the stacktrie commits + stackTrieSponge := &spongeDb{ + sponge: sha3.NewLegacyKeccak256(), + id: "b", + values: make(map[string]string), + } options := NewStackTrieOptions() options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { rawdb.WriteTrieNode(stackTrieSponge, common.Hash{}, path, hash, blob, db.Scheme()) }) stTrie := NewStackTrie(options) + // Fill the trie with elements for i := 0; i < count; i++ { // For the stack trie, we need to do inserts in proper order @@ -960,13 +988,16 @@ func TestCommitSequenceStackTrie(t *testing.T) { // Flush trie -> database root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) - db.Commit(root, false) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Commit(root) + s.Flush() + // And flush stacktrie -> disk stRoot := stTrie.Commit() if stRoot != root { t.Fatalf("root wrong, got %x exp %x", stRoot, root) } + stackTrieSponge.Flush() if got, exp := stackTrieSponge.sponge.Sum(nil), s.sponge.Sum(nil); !bytes.Equal(got, exp) { // Show the journal t.Logf("Expected:") @@ -989,34 +1020,47 @@ func TestCommitSequenceStackTrie(t *testing.T) { // that even a small trie which contains a leaf will have an extension making it // not fit into 32 bytes, rlp-encoded. However, it's still the correct thing to do. func TestCommitSequenceSmallRoot(t *testing.T) { - s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} - db := NewDatabase(rawdb.NewDatabase(s), nil) + s := &spongeDb{ + sponge: sha3.NewLegacyKeccak256(), + id: "a", + values: make(map[string]string), + } + db := newTestDatabase(rawdb.NewDatabase(s), rawdb.HashScheme) trie := NewEmpty(db) - // Another sponge is used for the stacktrie commits - stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} + // Another sponge is used for the stacktrie commits + stackTrieSponge := &spongeDb{ + sponge: sha3.NewLegacyKeccak256(), + id: "b", + values: make(map[string]string), + } options := NewStackTrieOptions() options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { rawdb.WriteTrieNode(stackTrieSponge, common.Hash{}, path, hash, blob, db.Scheme()) }) stTrie := NewStackTrie(options) + // Add a single small-element to the trie(s) key := make([]byte, 5) key[0] = 1 trie.Update(key, []byte{0x1}) stTrie.Update(key, []byte{0x1}) + // Flush trie -> database root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) - db.Commit(root, false) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Commit(root) + // And flush stacktrie -> disk stRoot := stTrie.Commit() if stRoot != root { t.Fatalf("root wrong, got %x exp %x", stRoot, root) } - t.Logf("root: %x\n", stRoot) + + s.Flush() + stackTrieSponge.Flush() if got, exp := stackTrieSponge.sponge.Sum(nil), s.sponge.Sum(nil); !bytes.Equal(got, exp) { t.Fatalf("test, disk write sequence wrong:\ngot %x exp %x\n", got, exp) } @@ -1067,7 +1111,7 @@ func BenchmarkHashFixedSize(b *testing.B) { func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1118,7 +1162,7 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) { func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1129,60 +1173,6 @@ func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accou b.StopTimer() } -func BenchmarkDerefRootFixedSize(b *testing.B) { - b.Run("10", func(b *testing.B) { - b.StopTimer() - acc, add := makeAccounts(20) - for i := 0; i < b.N; i++ { - benchmarkDerefRootFixedSize(b, acc, add) - } - }) - b.Run("100", func(b *testing.B) { - b.StopTimer() - acc, add := makeAccounts(100) - for i := 0; i < b.N; i++ { - benchmarkDerefRootFixedSize(b, acc, add) - } - }) - - b.Run("1K", func(b *testing.B) { - b.StopTimer() - acc, add := makeAccounts(1000) - for i := 0; i < b.N; i++ { - benchmarkDerefRootFixedSize(b, acc, add) - } - }) - b.Run("10K", func(b *testing.B) { - b.StopTimer() - acc, add := makeAccounts(10000) - for i := 0; i < b.N; i++ { - benchmarkDerefRootFixedSize(b, acc, add) - } - }) - b.Run("100K", func(b *testing.B) { - b.StopTimer() - acc, add := makeAccounts(100000) - for i := 0; i < b.N; i++ { - benchmarkDerefRootFixedSize(b, acc, add) - } - }) -} - -func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { - b.ReportAllocs() - triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) - trie := NewEmpty(triedb) - for i := 0; i < len(addresses); i++ { - trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) - } - h := trie.Hash() - root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) - b.StartTimer() - triedb.Dereference(h) - b.StopTimer() -} - func getString(trie *Trie, k string) []byte { return trie.MustGet([]byte(k)) } diff --git a/trie/triestate/state.go b/trie/triestate/state.go index 4c47e9c397..aa4d32f852 100644 --- a/trie/triestate/state.go +++ b/trie/triestate/state.go @@ -59,18 +59,16 @@ type TrieLoader interface { // The value refers to the original content of state before the transition // is made. Nil means that the state was not present previously. type Set struct { - Accounts map[common.Address][]byte // Mutated account set, nil means the account was not present - Storages map[common.Address]map[common.Hash][]byte // Mutated storage set, nil means the slot was not present - Incomplete map[common.Address]struct{} // Indicator whether the storage is incomplete due to large deletion - size common.StorageSize // Approximate size of set + Accounts map[common.Address][]byte // Mutated account set, nil means the account was not present + Storages map[common.Address]map[common.Hash][]byte // Mutated storage set, nil means the slot was not present + size common.StorageSize // Approximate size of set } // New constructs the state set with provided data. -func New(accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte, incomplete map[common.Address]struct{}) *Set { +func New(accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte) *Set { return &Set{ - Accounts: accounts, - Storages: storages, - Incomplete: incomplete, + Accounts: accounts, + Storages: storages, } } @@ -88,7 +86,6 @@ func (s *Set) Size() common.StorageSize { } s.size += common.StorageSize(common.AddressLength) } - s.size += common.StorageSize(common.AddressLength * len(s.Incomplete)) return s.size } diff --git a/trie/verkle.go b/trie/verkle.go index c21a796a0f..01d813d9ec 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/triedb/database" "github.com/gballet/go-verkle" "github.com/holiman/uint256" ) @@ -39,13 +40,12 @@ var ( // interface so that Verkle trees can be reused verbatim. type VerkleTrie struct { root verkle.VerkleNode - db *Database cache *utils.PointCache reader *trieReader } // NewVerkleTrie constructs a verkle tree based on the specified root hash. -func NewVerkleTrie(root common.Hash, db *Database, cache *utils.PointCache) (*VerkleTrie, error) { +func NewVerkleTrie(root common.Hash, db database.Database, cache *utils.PointCache) (*VerkleTrie, error) { reader, err := newTrieReader(root, common.Hash{}, db) if err != nil { return nil, err @@ -64,7 +64,6 @@ func NewVerkleTrie(root common.Hash, db *Database, cache *utils.PointCache) (*Ve } return &VerkleTrie{ root: node, - db: db, cache: cache, reader: reader, }, nil @@ -261,7 +260,6 @@ func (t *VerkleTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { func (t *VerkleTrie) Copy() *VerkleTrie { return &VerkleTrie{ root: t.root.Copy(), - db: t.db, cache: t.cache, reader: t.reader, } diff --git a/trie/verkle_test.go b/trie/verkle_test.go index 1c65b673aa..0cbe28bf01 100644 --- a/trie/verkle_test.go +++ b/trie/verkle_test.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/utils" "github.com/holiman/uint256" ) @@ -57,12 +56,7 @@ var ( ) func TestVerkleTreeReadWrite(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase(), &Config{ - IsVerkle: true, - PathDB: pathdb.Defaults, - }) - defer db.Close() - + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) tr, _ := NewVerkleTrie(types.EmptyVerkleHash, db, utils.NewPointCache(100)) for addr, acct := range accounts { diff --git a/trie/database.go b/triedb/database.go similarity index 91% rename from trie/database.go rename to triedb/database.go index e20f7ef903..939a21f147 100644 --- a/trie/database.go +++ b/triedb/database.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package trie +package triedb import ( "errors" @@ -22,10 +22,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/ethereum/go-ethereum/triedb/database" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) // Config defines all necessary options for database. @@ -108,14 +110,21 @@ func NewDatabase(diskdb ethdb.Database, config *Config) *Database { if config.PathDB != nil { db.backend = pathdb.New(diskdb, config.PathDB) } else { - db.backend = hashdb.New(diskdb, config.HashDB, mptResolver{}) + var resolver hashdb.ChildResolver + if config.IsVerkle { + // TODO define verkle resolver + log.Crit("Verkle node resolver is not defined") + } else { + resolver = trie.MerkleResolver{} + } + db.backend = hashdb.New(diskdb, config.HashDB, resolver) } return db } // Reader returns a reader for accessing all trie nodes with provided state root. // An error will be returned if the requested state is not available. -func (db *Database) Reader(blockRoot common.Hash) (Reader, error) { +func (db *Database) Reader(blockRoot common.Hash) (database.Reader, error) { switch b := db.backend.(type) { case *hashdb.Database: return b.Reader(blockRoot) @@ -190,8 +199,7 @@ func (db *Database) WritePreimages() { } } -// Preimage retrieves a cached trie node pre-image from memory. If it cannot be -// found cached, the method queries the persistent database for the content. +// Preimage retrieves a cached trie node pre-image from preimage store. func (db *Database) Preimage(hash common.Hash) []byte { if db.preimages == nil { return nil @@ -199,6 +207,14 @@ func (db *Database) Preimage(hash common.Hash) []byte { return db.preimages.preimage(hash) } +// InsertPreimage writes pre-images of trie node to the preimage store. +func (db *Database) InsertPreimage(preimages map[common.Hash][]byte) { + if db.preimages == nil { + return + } + db.preimages.insertPreimage(preimages) +} + // Cap iteratively flushes old but still referenced trie nodes until the total // memory usage goes below the given threshold. The held pre-images accumulated // up to this point will be flushed in case the size exceeds the threshold. @@ -249,7 +265,14 @@ func (db *Database) Recover(target common.Hash) error { if !ok { return errors.New("not supported") } - return pdb.Recover(target, &trieLoader{db: db}) + var loader triestate.TrieLoader + if db.config.IsVerkle { + // TODO define verkle loader + log.Crit("Verkle loader is not defined") + } else { + loader = trie.NewMerkleLoader(db) + } + return pdb.Recover(target, loader) } // Recoverable returns the indicator if the specified state is enabled to be diff --git a/triedb/database/database.go b/triedb/database/database.go new file mode 100644 index 0000000000..18a8f454e2 --- /dev/null +++ b/triedb/database/database.go @@ -0,0 +1,48 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package database + +import ( + "github.com/ethereum/go-ethereum/common" +) + +// Reader wraps the Node method of a backing trie reader. +type Reader interface { + // Node retrieves the trie node blob with the provided trie identifier, + // node path and the corresponding node hash. No error will be returned + // if the node is not found. + Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) +} + +// PreimageStore wraps the methods of a backing store for reading and writing +// trie node preimages. +type PreimageStore interface { + // Preimage retrieves the preimage of the specified hash. + Preimage(hash common.Hash) []byte + + // InsertPreimage commits a set of preimages along with their hashes. + InsertPreimage(preimages map[common.Hash][]byte) +} + +// Database wraps the methods of a backing trie store. +type Database interface { + PreimageStore + + // Reader returns a node reader associated with the specific state. + // An error will be returned if the specified state is not available. + Reader(stateRoot common.Hash) (Reader, error) +} diff --git a/trie/triedb/hashdb/database.go b/triedb/hashdb/database.go similarity index 100% rename from trie/triedb/hashdb/database.go rename to triedb/hashdb/database.go diff --git a/trie/triedb/pathdb/database.go b/triedb/pathdb/database.go similarity index 98% rename from trie/triedb/pathdb/database.go rename to triedb/pathdb/database.go index f2d6cea635..b1e01abac4 100644 --- a/trie/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -203,7 +203,6 @@ func New(diskdb ethdb.Database, config *Config) *Database { log.Crit("Failed to disable database", "err", err) // impossible to happen } } - log.Warn("Path-based state scheme is an experimental feature") return db } @@ -391,17 +390,20 @@ func (db *Database) Recoverable(root common.Hash) bool { if *id >= dl.stateID() { return false } + // This is a temporary workaround for the unavailability of the freezer in + // dev mode. As a consequence, the Pathdb loses the ability for deep reorg + // in certain cases. + // TODO(rjl493456442): Implement the in-memory ancient store. + if db.freezer == nil { + return false + } // Ensure the requested state is a canonical state and all state // histories in range [id+1, disklayer.ID] are present and complete. - parent := root return checkHistories(db.freezer, *id+1, dl.stateID()-*id, func(m *meta) error { - if m.parent != parent { + if m.parent != root { return errors.New("unexpected state history") } - if len(m.incomplete) > 0 { - return errors.New("incomplete state history") - } - parent = m.root + root = m.root return nil }) == nil } diff --git a/trie/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go similarity index 99% rename from trie/triedb/pathdb/database_test.go rename to triedb/pathdb/database_test.go index e7bd469993..2e7e1bef05 100644 --- a/trie/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -299,7 +299,7 @@ func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNode } } } - return root, ctx.nodes, triestate.New(ctx.accountOrigin, ctx.storageOrigin, nil) + return root, ctx.nodes, triestate.New(ctx.accountOrigin, ctx.storageOrigin) } // lastRoot returns the latest root hash, or empty if nothing is cached. @@ -397,7 +397,11 @@ func TestDatabaseRollback(t *testing.T) { if err := tester.db.Recover(parent, loader); err != nil { t.Fatalf("Failed to revert db, err: %v", err) } - tester.verifyState(parent) + if i > 0 { + if err := tester.verifyState(parent); err != nil { + t.Fatalf("Failed to verify state, err: %v", err) + } + } } if tester.db.tree.len() != 1 { t.Fatal("Only disk layer is expected") @@ -539,7 +543,7 @@ func TestCorruptedJournal(t *testing.T) { // Mutate the journal in disk, it should be regarded as invalid blob := rawdb.ReadTrieJournal(tester.db.diskdb) - blob[0] = 1 + blob[0] = 0xa rawdb.WriteTrieJournal(tester.db.diskdb, blob) // Verify states, all not-yet-written states should be discarded diff --git a/trie/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go similarity index 100% rename from trie/triedb/pathdb/difflayer.go rename to triedb/pathdb/difflayer.go diff --git a/trie/triedb/pathdb/difflayer_test.go b/triedb/pathdb/difflayer_test.go similarity index 100% rename from trie/triedb/pathdb/difflayer_test.go rename to triedb/pathdb/difflayer_test.go diff --git a/trie/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go similarity index 97% rename from trie/triedb/pathdb/disklayer.go rename to triedb/pathdb/disklayer.go index ef697cbce8..777e4ec8a7 100644 --- a/trie/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -17,7 +17,6 @@ package pathdb import ( - "errors" "fmt" "sync" @@ -239,12 +238,6 @@ func (dl *diskLayer) revert(h *history, loader triestate.TrieLoader) (*diskLayer if h.meta.root != dl.rootHash() { return nil, errUnexpectedHistory } - // Reject if the provided state history is incomplete. It's due to - // a large construct SELF-DESTRUCT which can't be handled because - // of memory limitation. - if len(h.meta.incomplete) > 0 { - return nil, errors.New("incomplete state history") - } if dl.id == 0 { return nil, fmt.Errorf("%w: zero state id", errStateUnrecoverable) } diff --git a/trie/triedb/pathdb/errors.go b/triedb/pathdb/errors.go similarity index 100% rename from trie/triedb/pathdb/errors.go rename to triedb/pathdb/errors.go diff --git a/trie/triedb/pathdb/history.go b/triedb/pathdb/history.go similarity index 94% rename from trie/triedb/pathdb/history.go rename to triedb/pathdb/history.go index 6e3f3faaed..68fb4809f0 100644 --- a/trie/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -66,7 +66,7 @@ import ( const ( accountIndexSize = common.AddressLength + 13 // The length of encoded account index slotIndexSize = common.HashLength + 5 // The length of encoded slot index - historyMetaSize = 9 + 2*common.HashLength // The length of fixed size part of meta object + historyMetaSize = 9 + 2*common.HashLength // The length of encoded history meta stateHistoryVersion = uint8(0) // initial version of state history structure. ) @@ -192,47 +192,36 @@ func (i *slotIndex) decode(blob []byte) { // meta describes the meta data of state history object. type meta struct { - version uint8 // version tag of history object - parent common.Hash // prev-state root before the state transition - root common.Hash // post-state root after the state transition - block uint64 // associated block number - incomplete []common.Address // list of address whose storage set is incomplete + version uint8 // version tag of history object + parent common.Hash // prev-state root before the state transition + root common.Hash // post-state root after the state transition + block uint64 // associated block number } // encode packs the meta object into byte stream. func (m *meta) encode() []byte { - buf := make([]byte, historyMetaSize+len(m.incomplete)*common.AddressLength) + buf := make([]byte, historyMetaSize) buf[0] = m.version copy(buf[1:1+common.HashLength], m.parent.Bytes()) copy(buf[1+common.HashLength:1+2*common.HashLength], m.root.Bytes()) binary.BigEndian.PutUint64(buf[1+2*common.HashLength:historyMetaSize], m.block) - for i, h := range m.incomplete { - copy(buf[i*common.AddressLength+historyMetaSize:], h.Bytes()) - } return buf[:] } // decode unpacks the meta object from byte stream. func (m *meta) decode(blob []byte) error { if len(blob) < 1 { - return fmt.Errorf("no version tag") + return errors.New("no version tag") } switch blob[0] { case stateHistoryVersion: - if len(blob) < historyMetaSize { + if len(blob) != historyMetaSize { return fmt.Errorf("invalid state history meta, len: %d", len(blob)) } - if (len(blob)-historyMetaSize)%common.AddressLength != 0 { - return fmt.Errorf("corrupted state history meta, len: %d", len(blob)) - } m.version = blob[0] m.parent = common.BytesToHash(blob[1 : 1+common.HashLength]) m.root = common.BytesToHash(blob[1+common.HashLength : 1+2*common.HashLength]) m.block = binary.BigEndian.Uint64(blob[1+2*common.HashLength : historyMetaSize]) - for pos := historyMetaSize; pos < len(blob); { - m.incomplete = append(m.incomplete, common.BytesToAddress(blob[pos:pos+common.AddressLength])) - pos += common.AddressLength - } return nil default: return fmt.Errorf("unknown version %d", blob[0]) @@ -257,7 +246,6 @@ func newHistory(root common.Hash, parent common.Hash, block uint64, states *trie var ( accountList []common.Address storageList = make(map[common.Address][]common.Hash) - incomplete []common.Address ) for addr := range states.Accounts { accountList = append(accountList, addr) @@ -272,18 +260,12 @@ func newHistory(root common.Hash, parent common.Hash, block uint64, states *trie slices.SortFunc(slist, common.Hash.Cmp) storageList[addr] = slist } - for addr := range states.Incomplete { - incomplete = append(incomplete, addr) - } - slices.SortFunc(incomplete, common.Address.Cmp) - return &history{ meta: &meta{ - version: stateHistoryVersion, - parent: parent, - root: root, - block: block, - incomplete: incomplete, + version: stateHistoryVersion, + parent: parent, + root: root, + block: block, }, accounts: states.Accounts, accountList: accountList, diff --git a/trie/triedb/pathdb/history_test.go b/triedb/pathdb/history_test.go similarity index 99% rename from trie/triedb/pathdb/history_test.go rename to triedb/pathdb/history_test.go index a3257441de..ab0d44777d 100644 --- a/trie/triedb/pathdb/history_test.go +++ b/triedb/pathdb/history_test.go @@ -47,7 +47,7 @@ func randomStateSet(n int) *triestate.Set { account := generateAccount(types.EmptyRootHash) accounts[addr] = types.SlimAccountRLP(account) } - return triestate.New(accounts, storages, nil) + return triestate.New(accounts, storages) } func makeHistory() *history { diff --git a/trie/triedb/pathdb/journal.go b/triedb/pathdb/journal.go similarity index 95% rename from trie/triedb/pathdb/journal.go rename to triedb/pathdb/journal.go index ac770763e3..3a0b7ebae2 100644 --- a/trie/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -41,7 +41,13 @@ var ( errUnmatchedJournal = errors.New("unmatched journal") ) -const journalVersion uint64 = 0 +// journalVersion ensures that an incompatible journal is detected and discarded. +// +// Changelog: +// +// - Version 0: initial version +// - Version 1: storage.Incomplete field is removed +const journalVersion uint64 = 1 // journalNode represents a trie node persisted in the journal. type journalNode struct { @@ -64,10 +70,9 @@ type journalAccounts struct { // journalStorage represents a list of storage slots belong to an account. type journalStorage struct { - Incomplete bool - Account common.Address - Hashes []common.Hash - Slots [][]byte + Account common.Address + Hashes []common.Hash + Slots [][]byte } // loadJournal tries to parse the layer journal from the disk. @@ -209,11 +214,10 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) { } // Read state changes from journal var ( - jaccounts journalAccounts - jstorages []journalStorage - accounts = make(map[common.Address][]byte) - storages = make(map[common.Address]map[common.Hash][]byte) - incomplete = make(map[common.Address]struct{}) + jaccounts journalAccounts + jstorages []journalStorage + accounts = make(map[common.Address][]byte) + storages = make(map[common.Address]map[common.Hash][]byte) ) if err := r.Decode(&jaccounts); err != nil { return nil, fmt.Errorf("load diff accounts: %v", err) @@ -233,12 +237,9 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) { set[h] = nil } } - if entry.Incomplete { - incomplete[entry.Account] = struct{}{} - } storages[entry.Account] = set } - return db.loadDiffLayer(newDiffLayer(parent, root, parent.stateID()+1, block, nodes, triestate.New(accounts, storages, incomplete)), r) + return db.loadDiffLayer(newDiffLayer(parent, root, parent.stateID()+1, block, nodes, triestate.New(accounts, storages)), r) } // journal implements the layer interface, marshaling the un-flushed trie nodes @@ -316,9 +317,6 @@ func (dl *diffLayer) journal(w io.Writer) error { storage := make([]journalStorage, 0, len(dl.states.Storages)) for addr, slots := range dl.states.Storages { entry := journalStorage{Account: addr} - if _, ok := dl.states.Incomplete[addr]; ok { - entry.Incomplete = true - } for slotHash, slot := range slots { entry.Hashes = append(entry.Hashes, slotHash) entry.Slots = append(entry.Slots, slot) diff --git a/trie/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go similarity index 100% rename from trie/triedb/pathdb/layertree.go rename to triedb/pathdb/layertree.go diff --git a/trie/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go similarity index 100% rename from trie/triedb/pathdb/metrics.go rename to triedb/pathdb/metrics.go diff --git a/trie/triedb/pathdb/nodebuffer.go b/triedb/pathdb/nodebuffer.go similarity index 94% rename from trie/triedb/pathdb/nodebuffer.go rename to triedb/pathdb/nodebuffer.go index 4a7d328b9a..8f84c2b442 100644 --- a/trie/triedb/pathdb/nodebuffer.go +++ b/triedb/pathdb/nodebuffer.go @@ -204,6 +204,19 @@ func (b *nodebuffer) setSize(size int, db ethdb.KeyValueStore, clean *fastcache. return b.flush(db, clean, id, false) } +// allocBatch returns a database batch with pre-allocated buffer. +func (b *nodebuffer) allocBatch(db ethdb.KeyValueStore) ethdb.Batch { + var metasize int + for owner, nodes := range b.nodes { + if owner == (common.Hash{}) { + metasize += len(nodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix + } else { + metasize += len(nodes) * (len(rawdb.TrieNodeStoragePrefix) + common.HashLength) // database key prefix + owner + } + } + return db.NewBatchWithSize((metasize + int(b.size)) * 11 / 10) // extra 10% for potential pebble internal stuff +} + // flush persists the in-memory dirty trie node into the disk if the configured // memory threshold is reached. Note, all data must be written atomically. func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force bool) error { @@ -217,7 +230,7 @@ func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id ui } var ( start = time.Now() - batch = db.NewBatchWithSize(int(b.size)) + batch = b.allocBatch(db) ) nodes := writeNodes(batch, b.nodes, clean) rawdb.WritePersistentStateID(batch, id) diff --git a/trie/triedb/pathdb/testutils.go b/triedb/pathdb/testutils.go similarity index 100% rename from trie/triedb/pathdb/testutils.go rename to triedb/pathdb/testutils.go diff --git a/trie/preimages.go b/triedb/preimages.go similarity index 99% rename from trie/preimages.go rename to triedb/preimages.go index 66f34117c1..a5384910f7 100644 --- a/trie/preimages.go +++ b/triedb/preimages.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package trie +package triedb import ( "sync"