diff --git a/Makefile b/Makefile index d736ef61c0..99b8ba54b4 100644 --- a/Makefile +++ b/Makefile @@ -8,20 +8,25 @@ GOBIN = ./build/bin GO ?= latest GORUN = go run +#? geth: Build geth geth: $(GORUN) build/ci.go install ./cmd/geth @echo "Done building." @echo "Run \"$(GOBIN)/geth\" to launch geth." +#? all: Build all packages and executables all: $(GORUN) build/ci.go install +#? test: Run the tests test: all $(GORUN) build/ci.go test +#? lint: Run certain pre-selected linters lint: ## Run linters. $(GORUN) build/ci.go lint +#? clean: Clean go cache, built executables, and the auto generated folder clean: go clean -cache rm -fr build/_workspace/pkg/ $(GOBIN)/* @@ -29,6 +34,7 @@ clean: # The devtools target installs tools required for 'go generate'. # You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'. +#? devtools: Install recommended developer tools devtools: env GOBIN= go install golang.org/x/tools/cmd/stringer@latest env GOBIN= go install github.com/fjl/gencodec@latest @@ -36,3 +42,9 @@ devtools: env GOBIN= go install ./cmd/abigen @type "solc" 2> /dev/null || echo 'Please install solc' @type "protoc" 2> /dev/null || echo 'Please install protoc' + +#? help: Get more info on make commands. +help: Makefile + @echo " Choose a command run in go-ethereum:" + @sed -n 's/^#?//p' $< | column -t -s ':' | sort | sed -e 's/^/ /' +.PHONY: help diff --git a/README.md b/README.md index d6bc1af05c..1e8dba8090 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ ## Go Ethereum -Official Golang execution layer implementation of the Ethereum protocol. +Golang execution layer implementation of the Ethereum protocol. [![API Reference]( https://pkg.go.dev/badge/github.com/ethereum/go-ethereum )](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc) [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) -[![Travis](https://travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.com/ethereum/go-ethereum) +[![Travis](https://app.travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://app.travis-ci.com/github/ethereum/go-ethereum) [![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv) Automated builds are available for stable releases and the unstable master branch. Binary diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index 4abf298068..c7bc2b4541 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -29,7 +29,7 @@ import ( ) // The ABI holds information about a contract's context and available -// invokable methods. It will allow you to type check function calls and +// invocable methods. It will allow you to type check function calls and // packs data accordingly. type ABI struct { Constructor Method diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index 9fd919a295..cce71d26e0 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -65,7 +65,7 @@ func TestWaitDeployed(t *testing.T) { // Create the transaction head, _ := backend.Client().HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(params.GWei)) tx := types.NewContractCreation(0, big.NewInt(0), test.gas, gasPrice, common.FromHex(test.code)) tx, _ = types.SignTx(tx, types.LatestSignerForChainID(big.NewInt(1337)), testKey) diff --git a/accounts/scwallet/hub.go b/accounts/scwallet/hub.go index f9dcf58e19..5f1f369ca2 100644 --- a/accounts/scwallet/hub.go +++ b/accounts/scwallet/hub.go @@ -241,7 +241,7 @@ func (hub *Hub) refreshWallets() { card.Disconnect(pcsc.LeaveCard) continue } - // Card connected, start tracking in amongs the wallets + // Card connected, start tracking among the wallets hub.wallets[reader] = wallet events = append(events, accounts.WalletEvent{Wallet: wallet, Kind: accounts.WalletArrived}) } diff --git a/accounts/usbwallet/trezor/trezor.go b/accounts/usbwallet/trezor/trezor.go index 7e756e609b..cdca6b4e0b 100644 --- a/accounts/usbwallet/trezor/trezor.go +++ b/accounts/usbwallet/trezor/trezor.go @@ -16,7 +16,7 @@ // This file contains the implementation for interacting with the Trezor hardware // wallets. The wire protocol spec can be found on the SatoshiLabs website: -// https://wiki.trezor.io/Developers_guide-Message_Workflows +// https://docs.trezor.io/trezor-firmware/common/message-workflows.html // !!! STAHP !!! // diff --git a/beacon/engine/types.go b/beacon/engine/types.go index 67f30d4455..f72319ad50 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -26,6 +26,16 @@ import ( "github.com/ethereum/go-ethereum/trie" ) +// PayloadVersion denotes the version of PayloadAttributes used to request the +// building of the payload to commence. +type PayloadVersion byte + +var ( + PayloadV1 PayloadVersion = 0x1 + PayloadV2 PayloadVersion = 0x2 + PayloadV3 PayloadVersion = 0x3 +) + //go:generate go run github.com/fjl/gencodec -type PayloadAttributes -field-override payloadAttributesMarshaling -out gen_blockparams.go // PayloadAttributes describes the environment context in which a block should @@ -115,6 +125,21 @@ type TransitionConfigurationV1 struct { // PayloadID is an identifier of the payload build process type PayloadID [8]byte +// Version returns the payload version associated with the identifier. +func (b PayloadID) Version() PayloadVersion { + return PayloadVersion(b[0]) +} + +// Is returns whether the identifier matches any of provided payload versions. +func (b PayloadID) Is(versions ...PayloadVersion) bool { + for _, v := range versions { + if v == b.Version() { + return true + } + } + return false +} + func (b PayloadID) String() string { return hexutil.Encode(b[:]) } diff --git a/build/checksums.txt b/build/checksums.txt index b9d322aa1a..96815ff791 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -5,22 +5,22 @@ # https://github.com/ethereum/execution-spec-tests/releases/download/v1.0.6/ 485af7b66cf41eb3a8c1bd46632913b8eb95995df867cf665617bbc9b4beedd1 fixtures_develop.tar.gz -# version:golang 1.21.5 +# version:golang 1.21.6 # https://go.dev/dl/ -285cbbdf4b6e6e62ed58f370f3f6d8c30825d6e56c5853c66d3c23bcdb09db19 go1.21.5.src.tar.gz -a2e1d5743e896e5fe1e7d96479c0a769254aed18cf216cf8f4c3a2300a9b3923 go1.21.5.darwin-amd64.tar.gz -d0f8ac0c4fb3efc223a833010901d02954e3923cfe2c9a2ff0e4254a777cc9cc go1.21.5.darwin-arm64.tar.gz -2c05bbe0dc62456b90b7ddd354a54f373b7c377a98f8b22f52ab694b4f6cca58 go1.21.5.freebsd-386.tar.gz -30b6c64e9a77129605bc12f836422bf09eec577a8c899ee46130aeff81567003 go1.21.5.freebsd-amd64.tar.gz -8f4dba9cf5c61757bbd7e9ebdb93b6a30a1b03f4a636a1ba0cc2f27b907ab8e1 go1.21.5.linux-386.tar.gz -e2bc0b3e4b64111ec117295c088bde5f00eeed1567999ff77bc859d7df70078e go1.21.5.linux-amd64.tar.gz -841cced7ecda9b2014f139f5bab5ae31785f35399f236b8b3e75dff2a2978d96 go1.21.5.linux-arm64.tar.gz -837f4bf4e22fcdf920ffeaa4abf3d02d1314e03725431065f4d44c46a01b42fe go1.21.5.linux-armv6l.tar.gz -907b8c6ec4be9b184952e5d3493be66b1746442394a8bc78556c56834cd7c38b go1.21.5.linux-ppc64le.tar.gz -9c4a81b72ebe44368813cd03684e1080a818bf915d84163abae2ed325a1b2dc0 go1.21.5.linux-s390x.tar.gz -6da2418889dfb37763d0eb149c4a8d728c029e12f0cd54fbca0a31ae547e2d34 go1.21.5.windows-386.zip -bbe603cde7c9dee658f45164b4d06de1eff6e6e6b800100824e7c00d56a9a92f go1.21.5.windows-amd64.zip -9b7acca50e674294e43202df4fbc26d5af4d8bc3170a3342a1514f09a2dab5e9 go1.21.5.windows-arm64.zip +124926a62e45f78daabbaedb9c011d97633186a33c238ffc1e25320c02046248 go1.21.6.src.tar.gz +31d6ecca09010ab351e51343a5af81d678902061fee871f912bdd5ef4d778850 go1.21.6.darwin-amd64.tar.gz +0ff541fb37c38e5e5c5bcecc8f4f43c5ffd5e3a6c33a5d3e4003ded66fcfb331 go1.21.6.darwin-arm64.tar.gz +a1d1a149b34bf0f53965a237682c6da1140acabb131bf0e597240e4a140b0e5e go1.21.6.freebsd-386.tar.gz +de59e1217e4398b1522eed8dddabab2fa1b97aecbdca3af08e34832b4f0e3f81 go1.21.6.freebsd-amd64.tar.gz +05d09041b5a1193c14e4b2db3f7fcc649b236c567f5eb93305c537851b72dd95 go1.21.6.linux-386.tar.gz +3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 go1.21.6.linux-amd64.tar.gz +e2e8aa88e1b5170a0d495d7d9c766af2b2b6c6925a8f8956d834ad6b4cacbd9a go1.21.6.linux-arm64.tar.gz +6a8eda6cc6a799ff25e74ce0c13fdc1a76c0983a0bb07c789a2a3454bf6ec9b2 go1.21.6.linux-armv6l.tar.gz +e872b1e9a3f2f08fd4554615a32ca9123a4ba877ab6d19d36abc3424f86bc07f go1.21.6.linux-ppc64le.tar.gz +92894d0f732d3379bc414ffdd617eaadad47e1d72610e10d69a1156db03fc052 go1.21.6.linux-s390x.tar.gz +65b38857135cf45c80e1d267e0ce4f80fe149326c68835217da4f2da9b7943fe go1.21.6.windows-386.zip +27ac9dd6e66fb3fd0acfa6792ff053c86e7d2c055b022f4b5d53bfddec9e3301 go1.21.6.windows-amd64.zip +b93aff8f3c882c764c66a39b7a1483b0460e051e9992bf3435479129e5051bcd go1.21.6.windows-arm64.zip # version:golangci 1.55.2 # https://github.com/golangci/golangci-lint/releases/ diff --git a/build/ci.go b/build/ci.go index 1ffbf3074d..4d8dba6ce2 100644 --- a/build/ci.go +++ b/build/ci.go @@ -121,14 +121,13 @@ var ( // Note: vivid is unsupported because there is no golang-1.6 package for it. // Note: the following Ubuntu releases have been officially deprecated on Launchpad: // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite, impish, - // kinetic + // kinetic, lunar debDistroGoBoots = map[string]string{ "trusty": "golang-1.11", // 14.04, EOL: 04/2024 "xenial": "golang-go", // 16.04, EOL: 04/2026 "bionic": "golang-go", // 18.04, EOL: 04/2028 "focal": "golang-go", // 20.04, EOL: 04/2030 "jammy": "golang-go", // 22.04, EOL: 04/2032 - "lunar": "golang-go", // 23.04, EOL: 01/2024 "mantic": "golang-go", // 23.10, EOL: 07/2024 } diff --git a/cmd/clef/datatypes.md b/cmd/clef/datatypes.md index dd8cda5846..8456edfa35 100644 --- a/cmd/clef/datatypes.md +++ b/cmd/clef/datatypes.md @@ -75,7 +75,7 @@ Example: }, { "type": "Info", - "message": "User should see this aswell" + "message": "User should see this as well" } ], "meta": { diff --git a/cmd/devp2p/internal/ethtest/conn.go b/cmd/devp2p/internal/ethtest/conn.go index 2d36ccb423..ba3c0585fd 100644 --- a/cmd/devp2p/internal/ethtest/conn.go +++ b/cmd/devp2p/internal/ethtest/conn.go @@ -166,7 +166,7 @@ func (c *Conn) ReadEth() (any, error) { case eth.TransactionsMsg: msg = new(eth.TransactionsPacket) case eth.NewPooledTransactionHashesMsg: - msg = new(eth.NewPooledTransactionHashesPacket68) + msg = new(eth.NewPooledTransactionHashesPacket) case eth.GetPooledTransactionsMsg: msg = new(eth.GetPooledTransactionsPacket) case eth.PooledTransactionsMsg: diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 4f499d41d8..9409d6f083 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -710,7 +710,7 @@ func (s *Suite) TestNewPooledTxs(t *utesting.T) { } // Send announcement. - ann := eth.NewPooledTransactionHashesPacket68{Types: txTypes, Sizes: sizes, Hashes: hashes} + ann := eth.NewPooledTransactionHashesPacket{Types: txTypes, Sizes: sizes, Hashes: hashes} err = conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann) if err != nil { t.Fatalf("failed to write to connection: %v", err) @@ -728,7 +728,7 @@ func (s *Suite) TestNewPooledTxs(t *utesting.T) { t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsRequest)) } return - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: continue case *eth.TransactionsPacket: continue @@ -796,12 +796,12 @@ func (s *Suite) TestBlobViolations(t *utesting.T) { t2 = s.makeBlobTxs(2, 3, 0x2) ) for _, test := range []struct { - ann eth.NewPooledTransactionHashesPacket68 + ann eth.NewPooledTransactionHashesPacket resp eth.PooledTransactionsResponse }{ // Invalid tx size. { - ann: eth.NewPooledTransactionHashesPacket68{ + ann: eth.NewPooledTransactionHashesPacket{ Types: []byte{types.BlobTxType, types.BlobTxType}, Sizes: []uint32{uint32(t1[0].Size()), uint32(t1[1].Size() + 10)}, Hashes: []common.Hash{t1[0].Hash(), t1[1].Hash()}, @@ -810,7 +810,7 @@ func (s *Suite) TestBlobViolations(t *utesting.T) { }, // Wrong tx type. { - ann: eth.NewPooledTransactionHashesPacket68{ + ann: eth.NewPooledTransactionHashesPacket{ Types: []byte{types.DynamicFeeTxType, types.BlobTxType}, Sizes: []uint32{uint32(t2[0].Size()), uint32(t2[1].Size())}, Hashes: []common.Hash{t2[0].Hash(), t2[1].Hash()}, diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index 0ea7c32752..acf93a041e 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -70,7 +70,7 @@ func (s *Suite) sendTxs(txs []*types.Transaction) error { for _, tx := range *msg { got[tx.Hash()] = true } - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: for _, hash := range msg.Hashes { got[hash] = true } @@ -146,7 +146,7 @@ func (s *Suite) sendInvalidTxs(txs []*types.Transaction) error { return fmt.Errorf("received bad tx: %s", tx.Hash()) } } - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: for _, hash := range msg.Hashes { if _, ok := invalids[hash]; ok { return fmt.Errorf("received bad tx: %s", hash) diff --git a/cmd/era/main.go b/cmd/era/main.go new file mode 100644 index 0000000000..e27d8ccec6 --- /dev/null +++ b/cmd/era/main.go @@ -0,0 +1,324 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "encoding/json" + "fmt" + "math/big" + "os" + "path" + "strconv" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" + "github.com/urfave/cli/v2" +) + +var app = flags.NewApp("go-ethereum era tool") + +var ( + dirFlag = &cli.StringFlag{ + Name: "dir", + Usage: "directory storing all relevant era1 files", + Value: "eras", + } + networkFlag = &cli.StringFlag{ + Name: "network", + Usage: "network name associated with era1 files", + Value: "mainnet", + } + eraSizeFlag = &cli.IntFlag{ + Name: "size", + Usage: "number of blocks per era", + Value: era.MaxEra1Size, + } + txsFlag = &cli.BoolFlag{ + Name: "txs", + Usage: "print full transaction values", + } +) + +var ( + blockCommand = &cli.Command{ + Name: "block", + Usage: "get block data", + ArgsUsage: "", + Action: block, + Flags: []cli.Flag{ + txsFlag, + }, + } + infoCommand = &cli.Command{ + Name: "info", + ArgsUsage: "", + Usage: "get epoch information", + Action: info, + } + verifyCommand = &cli.Command{ + Name: "verify", + ArgsUsage: "", + Usage: "verifies each era1 against expected accumulator root", + Action: verify, + } +) + +func init() { + app.Commands = []*cli.Command{ + blockCommand, + infoCommand, + verifyCommand, + } + app.Flags = []cli.Flag{ + dirFlag, + networkFlag, + eraSizeFlag, + } +} + +func main() { + if err := app.Run(os.Args); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +// block prints the specified block from an era1 store. +func block(ctx *cli.Context) error { + num, err := strconv.ParseUint(ctx.Args().First(), 10, 64) + if err != nil { + return fmt.Errorf("invalid block number: %w", err) + } + e, err := open(ctx, num/uint64(ctx.Int(eraSizeFlag.Name))) + if err != nil { + return fmt.Errorf("error opening era1: %w", err) + } + defer e.Close() + // Read block with number. + block, err := e.GetBlockByNumber(num) + if err != nil { + return fmt.Errorf("error reading block %d: %w", num, err) + } + // Convert block to JSON and print. + val := ethapi.RPCMarshalBlock(block, ctx.Bool(txsFlag.Name), ctx.Bool(txsFlag.Name), params.MainnetChainConfig) + b, err := json.MarshalIndent(val, "", " ") + if err != nil { + return fmt.Errorf("error marshaling json: %w", err) + } + fmt.Println(string(b)) + return nil +} + +// info prints some high-level information about the era1 file. +func info(ctx *cli.Context) error { + epoch, err := strconv.ParseUint(ctx.Args().First(), 10, 64) + if err != nil { + return fmt.Errorf("invalid epoch number: %w", err) + } + e, err := open(ctx, epoch) + if err != nil { + return err + } + defer e.Close() + acc, err := e.Accumulator() + if err != nil { + return fmt.Errorf("error reading accumulator: %w", err) + } + td, err := e.InitialTD() + if err != nil { + return fmt.Errorf("error reading total difficulty: %w", err) + } + info := struct { + Accumulator common.Hash `json:"accumulator"` + TotalDifficulty *big.Int `json:"totalDifficulty"` + StartBlock uint64 `json:"startBlock"` + Count uint64 `json:"count"` + }{ + acc, td, e.Start(), e.Count(), + } + b, _ := json.MarshalIndent(info, "", " ") + fmt.Println(string(b)) + return nil +} + +// open opens an era1 file at a certain epoch. +func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { + var ( + dir = ctx.String(dirFlag.Name) + network = ctx.String(networkFlag.Name) + ) + entries, err := era.ReadDir(dir, network) + if err != nil { + return nil, fmt.Errorf("error reading era dir: %w", err) + } + if epoch >= uint64(len(entries)) { + return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch) + } + return era.Open(path.Join(dir, entries[epoch])) +} + +// verify checks each era1 file in a directory to ensure it is well-formed and +// that the accumulator matches the expected value. +func verify(ctx *cli.Context) error { + if ctx.Args().Len() != 1 { + return fmt.Errorf("missing accumulators file") + } + + roots, err := readHashes(ctx.Args().First()) + if err != nil { + return fmt.Errorf("unable to read expected roots file: %w", err) + } + + var ( + dir = ctx.String(dirFlag.Name) + network = ctx.String(networkFlag.Name) + start = time.Now() + reported = time.Now() + ) + + entries, err := era.ReadDir(dir, network) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + + if len(entries) != len(roots) { + return fmt.Errorf("number of era1 files should match the number of accumulator hashes") + } + + // Verify each epoch matches the expected root. + for i, want := range roots { + // Wrap in function so defers don't stack. + err := func() error { + name := entries[i] + e, err := era.Open(path.Join(dir, name)) + if err != nil { + return fmt.Errorf("error opening era1 file %s: %w", name, err) + } + defer e.Close() + // Read accumulator and check against expected. + if got, err := e.Accumulator(); err != nil { + return fmt.Errorf("error retrieving accumulator for %s: %w", name, err) + } else if got != want { + return fmt.Errorf("invalid root %s: got %s, want %s", name, got, want) + } + // Recompute accumulator. + if err := checkAccumulator(e); err != nil { + return fmt.Errorf("error verify era1 file %s: %w", name, err) + } + // Give the user some feedback that something is happening. + if time.Since(reported) >= 8*time.Second { + fmt.Printf("Verifying Era1 files \t\t verified=%d,\t elapsed=%s\n", i, common.PrettyDuration(time.Since(start))) + reported = time.Now() + } + return nil + }() + if err != nil { + return err + } + } + + return nil +} + +// checkAccumulator verifies the accumulator matches the data in the Era. +func checkAccumulator(e *era.Era) error { + var ( + err error + want common.Hash + td *big.Int + tds = make([]*big.Int, 0) + hashes = make([]common.Hash, 0) + ) + if want, err = e.Accumulator(); err != nil { + return fmt.Errorf("error reading accumulator: %w", err) + } + if td, err = e.InitialTD(); err != nil { + return fmt.Errorf("error reading total difficulty: %w", err) + } + it, err := era.NewIterator(e) + if err != nil { + return fmt.Errorf("error making era iterator: %w", err) + } + // To fully verify an era the following attributes must be checked: + // 1) the block index is constructed correctly + // 2) the tx root matches the value in the block + // 3) the receipts root matches the value in the block + // 4) the starting total difficulty value is correct + // 5) the accumulator is correct by recomputing it locally, which verifies + // the blocks are all correct (via hash) + // + // The attributes 1), 2), and 3) are checked for each block. 4) and 5) require + // accumulation across the entire set and are verified at the end. + for it.Next() { + // 1) next() walks the block index, so we're able to implicitly verify it. + if it.Error() != nil { + return fmt.Errorf("error reading block %d: %w", it.Number(), err) + } + block, receipts, err := it.BlockAndReceipts() + if it.Error() != nil { + return fmt.Errorf("error reading block %d: %w", it.Number(), err) + } + // 2) recompute tx root and verify against header. + tr := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)) + if tr != block.TxHash() { + return fmt.Errorf("tx root in block %d mismatch: want %s, got %s", block.NumberU64(), block.TxHash(), tr) + } + // 3) recompute receipt root and check value against block. + rr := types.DeriveSha(receipts, trie.NewStackTrie(nil)) + if rr != block.ReceiptHash() { + return fmt.Errorf("receipt root in block %d mismatch: want %s, got %s", block.NumberU64(), block.ReceiptHash(), rr) + } + hashes = append(hashes, block.Hash()) + td.Add(td, block.Difficulty()) + tds = append(tds, new(big.Int).Set(td)) + } + // 4+5) Verify accumulator and total difficulty. + got, err := era.ComputeAccumulator(hashes, tds) + if err != nil { + return fmt.Errorf("error computing accumulator: %w", err) + } + if got != want { + return fmt.Errorf("expected accumulator root does not match calculated: got %s, want %s", got, want) + } + return nil +} + +// readHashes reads a file of newline-delimited hashes. +func readHashes(f string) ([]common.Hash, error) { + b, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("unable to open accumulators file") + } + s := strings.Split(string(b), "\n") + // Remove empty last element, if present. + if s[len(s)-1] == "" { + s = s[:len(s)-1] + } + // Convert to hashes. + r := make([]common.Hash, len(s)) + for i := range s { + r[i] = common.HexToHash(s[i]) + } + return r, nil +} diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index b654cb2196..1ae093b61e 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) @@ -308,15 +309,15 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, reward.Sub(reward, new(big.Int).SetUint64(ommer.Delta)) reward.Mul(reward, blockReward) reward.Div(reward, big.NewInt(8)) - statedb.AddBalance(ommer.Address, reward) + statedb.AddBalance(ommer.Address, uint256.MustFromBig(reward)) } - statedb.AddBalance(pre.Env.Coinbase, minerReward) + statedb.AddBalance(pre.Env.Coinbase, uint256.MustFromBig(minerReward)) } // Apply withdrawals for _, w := range pre.Env.Withdrawals { // Amount is in gwei, turn into wei amount := new(big.Int).Mul(new(big.Int).SetUint64(w.Amount), big.NewInt(params.GWei)) - statedb.AddBalance(w.Address, amount) + statedb.AddBalance(w.Address, uint256.MustFromBig(amount)) } // Commit block root, err := statedb.Commit(vmContext.BlockNumber.Uint64(), chainConfig.IsEIP158(vmContext.BlockNumber)) @@ -359,7 +360,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB for addr, a := range accounts { statedb.SetCode(addr, a.Code) statedb.SetNonce(addr, a.Nonce) - statedb.SetBalance(addr, a.Balance) + statedb.SetBalance(addr, uint256.MustFromBig(a.Balance)) for k, v := range a.Storage { statedb.SetState(addr, k, v) } diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 4dc50e577f..31e96894dd 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -280,7 +280,7 @@ func (g Alloc) OnAccount(addr *common.Address, dumpAccount state.DumpAccount) { if addr == nil { return } - balance, _ := new(big.Int).SetString(dumpAccount.Balance, 10) + balance, _ := new(big.Int).SetString(dumpAccount.Balance, 0) var storage map[common.Hash]common.Hash if dumpAccount.Storage != nil { storage = make(map[common.Hash]common.Hash) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 3b4f516af7..d333c17559 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -35,10 +35,12 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/urfave/cli/v2" ) @@ -122,6 +124,33 @@ Optional second and third arguments control the first and last block to write. In this mode, the file will be appended if already existing. If the file ends with .gz, the output will be gzipped.`, + } + importHistoryCommand = &cli.Command{ + Action: importHistory, + Name: "import-history", + Usage: "Import an Era archive", + ArgsUsage: "", + Flags: flags.Merge([]cli.Flag{ + utils.TxLookupLimitFlag, + }, + utils.DatabaseFlags, + utils.NetworkFlags, + ), + Description: ` +The import-history command will import blocks and their corresponding receipts +from Era archives. +`, + } + exportHistoryCommand = &cli.Command{ + Action: exportHistory, + Name: "export-history", + Usage: "Export blockchain history to Era archives", + ArgsUsage: " ", + Flags: flags.Merge(utils.DatabaseFlags), + Description: ` +The export-history command will export blocks and their corresponding receipts +into Era archives. Eras are typically packaged in steps of 8192 blocks. +`, } importPreimagesCommand = &cli.Command{ Action: importPreimages, @@ -364,7 +393,97 @@ func exportChain(ctx *cli.Context) error { } err = utils.ExportAppendChain(chain, fp, uint64(first), uint64(last)) } + if err != nil { + utils.Fatalf("Export error: %v\n", err) + } + fmt.Printf("Export done in %v\n", time.Since(start)) + return nil +} +func importHistory(ctx *cli.Context) error { + if ctx.Args().Len() != 1 { + utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) + } + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, db := utils.MakeChain(ctx, stack, false) + defer db.Close() + + var ( + start = time.Now() + dir = ctx.Args().Get(0) + network string + ) + + // Determine network. + if utils.IsNetworkPreset(ctx) { + switch { + case ctx.Bool(utils.MainnetFlag.Name): + network = "mainnet" + case ctx.Bool(utils.SepoliaFlag.Name): + network = "sepolia" + case ctx.Bool(utils.GoerliFlag.Name): + network = "goerli" + } + } else { + // No network flag set, try to determine network based on files + // present in directory. + var networks []string + for _, n := range params.NetworkNames { + entries, err := era.ReadDir(dir, n) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + if len(entries) > 0 { + networks = append(networks, n) + } + } + if len(networks) == 0 { + return fmt.Errorf("no era1 files found in %s", dir) + } + if len(networks) > 1 { + return fmt.Errorf("multiple networks found, use a network flag to specify desired network") + } + network = networks[0] + } + + if err := utils.ImportHistory(chain, db, dir, network); err != nil { + return err + } + fmt.Printf("Import done in %v\n", time.Since(start)) + return nil +} + +// exportHistory exports chain history in Era archives at a specified +// directory. +func exportHistory(ctx *cli.Context) error { + if ctx.Args().Len() != 3 { + utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) + } + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, _ := utils.MakeChain(ctx, stack, true) + start := time.Now() + + var ( + dir = ctx.Args().Get(0) + first, ferr = strconv.ParseInt(ctx.Args().Get(1), 10, 64) + last, lerr = strconv.ParseInt(ctx.Args().Get(2), 10, 64) + ) + if ferr != nil || lerr != nil { + utils.Fatalf("Export error in parsing parameters: block number not an integer\n") + } + if first < 0 || last < 0 { + utils.Fatalf("Export error: block number must be greater than 0\n") + } + if head := chain.CurrentSnapBlock(); uint64(last) > head.Number.Uint64() { + utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.Number.Uint64()) + } + err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size)) if err != nil { utils.Fatalf("Export error: %v\n", err) } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 4438cef560..2f7d37fdd7 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . -// geth is the official command-line client for Ethereum. +// geth is a command-line client for Ethereum. package main import ( @@ -208,6 +208,8 @@ func init() { initCommand, importCommand, exportCommand, + importHistoryCommand, + exportHistoryCommand, importPreimagesCommand, removedbCommand, dumpCommand, diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 8b571be1ef..4b57164665 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -19,12 +19,15 @@ package utils import ( "bufio" + "bytes" "compress/gzip" + "crypto/sha256" "errors" "fmt" "io" "os" "os/signal" + "path" "runtime" "strings" "syscall" @@ -39,8 +42,10 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/urfave/cli/v2" ) @@ -228,6 +233,105 @@ func ImportChain(chain *core.BlockChain, fn string) error { return nil } +func readList(filename string) ([]string, error) { + b, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + return strings.Split(string(b), "\n"), nil +} + +// ImportHistory imports Era1 files containing historical block information, +// starting from genesis. +func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error { + if chain.CurrentSnapBlock().Number.BitLen() != 0 { + return fmt.Errorf("history import only supported when starting from genesis") + } + entries, err := era.ReadDir(dir, network) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + checksums, err := readList(path.Join(dir, "checksums.txt")) + if err != nil { + return fmt.Errorf("unable to read checksums.txt: %w", err) + } + if len(checksums) != len(entries) { + return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", len(checksums), len(entries)) + } + var ( + start = time.Now() + reported = time.Now() + imported = 0 + forker = core.NewForkChoice(chain, nil) + h = sha256.New() + buf = bytes.NewBuffer(nil) + ) + for i, filename := range entries { + err := func() error { + f, err := os.Open(path.Join(dir, filename)) + if err != nil { + return fmt.Errorf("unable to open era: %w", err) + } + defer f.Close() + + // Validate checksum. + if _, err := io.Copy(h, f); err != nil { + return fmt.Errorf("unable to recalculate checksum: %w", err) + } + if have, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; have != want { + return fmt.Errorf("checksum mismatch: have %s, want %s", have, want) + } + h.Reset() + buf.Reset() + + // Import all block data from Era1. + e, err := era.From(f) + if err != nil { + return fmt.Errorf("error opening era: %w", err) + } + it, err := era.NewIterator(e) + if err != nil { + return fmt.Errorf("error making era reader: %w", err) + } + for it.Next() { + block, err := it.Block() + if err != nil { + return fmt.Errorf("error reading block %d: %w", it.Number(), err) + } + if block.Number().BitLen() == 0 { + continue // skip genesis + } + receipts, err := it.Receipts() + if err != nil { + return fmt.Errorf("error reading receipts %d: %w", it.Number(), err) + } + if status, err := chain.HeaderChain().InsertHeaderChain([]*types.Header{block.Header()}, start, forker); err != nil { + return fmt.Errorf("error inserting header %d: %w", it.Number(), err) + } else if status != core.CanonStatTy { + return fmt.Errorf("error inserting header %d, not canon: %v", it.Number(), status) + } + if _, err := chain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{receipts}, 2^64-1); err != nil { + return fmt.Errorf("error inserting body %d: %w", it.Number(), err) + } + imported += 1 + + // Give the user some feedback that something is happening. + if time.Since(reported) >= 8*time.Second { + log.Info("Importing Era files", "head", it.Number(), "imported", imported, "elapsed", common.PrettyDuration(time.Since(start))) + imported = 0 + reported = time.Now() + } + } + return nil + }() + if err != nil { + return err + } + } + + return nil +} + func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block { head := chain.CurrentBlock() for i, block := range blocks { @@ -297,6 +401,93 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las return nil } +// ExportHistory exports blockchain history into the specified directory, +// following the Era format. +func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) error { + log.Info("Exporting blockchain history", "dir", dir) + if head := bc.CurrentBlock().Number.Uint64(); head < last { + log.Warn("Last block beyond head, setting last = head", "head", head, "last", last) + last = head + } + network := "unknown" + if name, ok := params.NetworkNames[bc.Config().ChainID.String()]; ok { + network = name + } + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return fmt.Errorf("error creating output directory: %w", err) + } + var ( + start = time.Now() + reported = time.Now() + h = sha256.New() + buf = bytes.NewBuffer(nil) + checksums []string + ) + for i := first; i <= last; i += step { + err := func() error { + filename := path.Join(dir, era.Filename(network, int(i/step), common.Hash{})) + f, err := os.Create(filename) + if err != nil { + return fmt.Errorf("could not create era file: %w", err) + } + defer f.Close() + + w := era.NewBuilder(f) + for j := uint64(0); j < step && j <= last-i; j++ { + var ( + n = i + j + block = bc.GetBlockByNumber(n) + ) + if block == nil { + return fmt.Errorf("export failed on #%d: not found", n) + } + receipts := bc.GetReceiptsByHash(block.Hash()) + if receipts == nil { + return fmt.Errorf("export failed on #%d: receipts not found", n) + } + td := bc.GetTd(block.Hash(), block.NumberU64()) + if td == nil { + return fmt.Errorf("export failed on #%d: total difficulty not found", n) + } + if err := w.Add(block, receipts, td); err != nil { + return err + } + } + root, err := w.Finalize() + if err != nil { + return fmt.Errorf("export failed to finalize %d: %w", step/i, err) + } + // Set correct filename with root. + os.Rename(filename, path.Join(dir, era.Filename(network, int(i/step), root))) + + // Compute checksum of entire Era1. + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + if _, err := io.Copy(h, f); err != nil { + return fmt.Errorf("unable to calculate checksum: %w", err) + } + checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex()) + h.Reset() + buf.Reset() + return nil + }() + if err != nil { + return err + } + if time.Since(reported) >= 8*time.Second { + log.Info("Exporting blocks", "exported", i, "elapsed", common.PrettyDuration(time.Since(start))) + reported = time.Now() + } + } + + os.WriteFile(path.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) + + log.Info("Exported blockchain to", "dir", dir) + + return nil +} + // ImportPreimages imports a batch of exported hash preimages into the database. // It's a part of the deprecated functionality, should be removed in the future. func ImportPreimages(db ethdb.Database, fn string) error { diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go new file mode 100644 index 0000000000..5a13f67aa9 --- /dev/null +++ b/cmd/utils/history_test.go @@ -0,0 +1,184 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package utils + +import ( + "bytes" + "crypto/sha256" + "io" + "math/big" + "os" + "path" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" +) + +var ( + count uint64 = 128 + step uint64 = 16 +) + +func TestHistoryImportAndExport(t *testing.T) { + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}}, + } + signer = types.LatestSigner(genesis.Config) + ) + + // Generate chain. + db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) { + if i == 0 { + return + } + tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(i - 1), + GasTipCap: common.Big0, + GasFeeCap: g.PrevBlock(0).BaseFee(), + Gas: 50000, + To: &common.Address{0xaa}, + Value: big.NewInt(int64(i)), + Data: nil, + AccessList: nil, + }) + if err != nil { + t.Fatalf("error creating tx: %v", err) + } + g.AddTx(tx) + }) + + // Initialize BlockChain. + chain, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if _, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("error insterting chain: %v", err) + } + + // Make temp directory for era files. + dir, err := os.MkdirTemp("", "history-export-test") + if err != nil { + t.Fatalf("error creating temp test directory: %v", err) + } + defer os.RemoveAll(dir) + + // Export history to temp directory. + if err := ExportHistory(chain, dir, 0, count, step); err != nil { + t.Fatalf("error exporting history: %v", err) + } + + // Read checksums. + b, err := os.ReadFile(path.Join(dir, "checksums.txt")) + if err != nil { + t.Fatalf("failed to read checksums: %v", err) + } + checksums := strings.Split(string(b), "\n") + + // Verify each Era. + entries, _ := era.ReadDir(dir, "mainnet") + for i, filename := range entries { + func() { + f, err := os.Open(path.Join(dir, filename)) + if err != nil { + t.Fatalf("error opening era file: %v", err) + } + var ( + h = sha256.New() + buf = bytes.NewBuffer(nil) + ) + if _, err := io.Copy(h, f); err != nil { + t.Fatalf("unable to recalculate checksum: %v", err) + } + if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want { + t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) + } + e, err := era.From(f) + if err != nil { + t.Fatalf("error opening era: %v", err) + } + defer e.Close() + it, err := era.NewIterator(e) + if err != nil { + t.Fatalf("error making era reader: %v", err) + } + for j := 0; it.Next(); j++ { + n := i*int(step) + j + if it.Error() != nil { + t.Fatalf("error reading block entry %d: %v", n, it.Error()) + } + block, receipts, err := it.BlockAndReceipts() + if err != nil { + t.Fatalf("error reading block entry %d: %v", n, err) + } + want := chain.GetBlockByNumber(uint64(n)) + if want, got := uint64(n), block.NumberU64(); want != got { + t.Fatalf("blocks out of order: want %d, got %d", want, got) + } + if want.Hash() != block.Hash() { + t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex()) + } + if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() { + t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got) + } + if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() { + t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got) + } + if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() { + t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got) + } + } + }() + } + + // Now import Era. + freezer := t.TempDir() + db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false) + if err != nil { + panic(err) + } + t.Cleanup(func() { + db2.Close() + }) + + genesis.MustCommit(db2, trie.NewDatabase(db, trie.HashDefaults)) + imported, err := core.NewBlockChain(db2, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if err := ImportHistory(imported, db2, dir, "mainnet"); err != nil { + t.Fatalf("failed to import chain: %v", err) + } + if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { + t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash()) + } +} diff --git a/common/big.go b/common/big.go index 65d4377bf7..cbb562a28e 100644 --- a/common/big.go +++ b/common/big.go @@ -16,7 +16,11 @@ package common -import "math/big" +import ( + "math/big" + + "github.com/holiman/uint256" +) // Common big integers often used var ( @@ -27,4 +31,6 @@ var ( Big32 = big.NewInt(32) Big256 = big.NewInt(256) Big257 = big.NewInt(257) + + U2560 = uint256.NewInt(0) ) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index e856f4e6ce..a350e383a2 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) // Proof-of-stake protocol constants. @@ -355,8 +356,8 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types. // Withdrawals processing. for _, w := range withdrawals { // Convert amount from gwei to wei. - amount := new(big.Int).SetUint64(w.Amount) - amount = amount.Mul(amount, big.NewInt(params.GWei)) + amount := new(uint256.Int).SetUint64(w.Amount) + amount = amount.Mul(amount, uint256.NewInt(params.GWei)) state.AddBalance(w.Address, amount) } // No block reward which is issued by consensus layer instead. diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 130dfdf213..c2936fd4b3 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -33,16 +33,17 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) // Ethash proof-of-work protocol constants. var ( - FrontierBlockReward = big.NewInt(5e+18) // Block reward in wei for successfully mining a block - ByzantiumBlockReward = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium - ConstantinopleBlockReward = big.NewInt(2e+18) // Block reward in wei for successfully mining a block upward from Constantinople - maxUncles = 2 // Maximum number of uncles allowed in a single block - allowedFutureBlockTimeSeconds = int64(15) // Max seconds from current time allowed for blocks, before they're considered future blocks + FrontierBlockReward = uint256.NewInt(5e+18) // Block reward in wei for successfully mining a block + ByzantiumBlockReward = uint256.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium + ConstantinopleBlockReward = uint256.NewInt(2e+18) // Block reward in wei for successfully mining a block upward from Constantinople + maxUncles = 2 // Maximum number of uncles allowed in a single block + allowedFutureBlockTimeSeconds = int64(15) // Max seconds from current time allowed for blocks, before they're considered future blocks // calcDifficultyEip5133 is the difficulty adjustment algorithm as specified by EIP 5133. // It offsets the bomb a total of 11.4M blocks. @@ -562,8 +563,8 @@ func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) { // Some weird constants to avoid constant memory allocs for them. var ( - big8 = big.NewInt(8) - big32 = big.NewInt(32) + u256_8 = uint256.NewInt(8) + u256_32 = uint256.NewInt(32) ) // AccumulateRewards credits the coinbase of the given block with the mining @@ -579,16 +580,18 @@ func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header blockReward = ConstantinopleBlockReward } // Accumulate the rewards for the miner and any included uncles - reward := new(big.Int).Set(blockReward) - r := new(big.Int) + reward := new(uint256.Int).Set(blockReward) + r := new(uint256.Int) + hNum, _ := uint256.FromBig(header.Number) for _, uncle := range uncles { - r.Add(uncle.Number, big8) - r.Sub(r, header.Number) + uNum, _ := uint256.FromBig(uncle.Number) + r.AddUint64(uNum, 8) + r.Sub(r, hNum) r.Mul(r, blockReward) - r.Div(r, big8) + r.Div(r, u256_8) state.AddBalance(uncle.Coinbase, r) - r.Div(blockReward, big32) + r.Div(blockReward, u256_32) reward.Add(reward, r) } state.AddBalance(header.Coinbase, reward) diff --git a/consensus/misc/dao.go b/consensus/misc/dao.go index 96995616de..e21a44f63d 100644 --- a/consensus/misc/dao.go +++ b/consensus/misc/dao.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) var ( @@ -81,6 +82,6 @@ func ApplyDAOHardFork(statedb *state.StateDB) { // Move every DAO account and extra-balance account funds into the refund contract for _, addr := range params.DAODrainList() { statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr)) - statedb.SetBalance(addr, new(big.Int)) + statedb.SetBalance(addr, new(uint256.Int)) } } diff --git a/core/bench_test.go b/core/bench_test.go index c5991f10e8..951ce2a08c 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -243,7 +243,7 @@ func BenchmarkChainWrite_full_500k(b *testing.B) { // makeChainForBench writes a given number of headers or empty blocks/receipts // into a database. -func makeChainForBench(db ethdb.Database, full bool, count uint64) { +func makeChainForBench(db ethdb.Database, genesis *Genesis, full bool, count uint64) { var hash common.Hash for n := uint64(0); n < count; n++ { header := &types.Header{ @@ -255,6 +255,9 @@ func makeChainForBench(db ethdb.Database, full bool, count uint64) { TxHash: types.EmptyTxsHash, ReceiptHash: types.EmptyReceiptsHash, } + if n == 0 { + header = genesis.ToBlock().Header() + } hash = header.Hash() rawdb.WriteHeader(db, header) @@ -262,7 +265,7 @@ func makeChainForBench(db ethdb.Database, full bool, count uint64) { rawdb.WriteTd(db, hash, n, big.NewInt(int64(n+1))) if n == 0 { - rawdb.WriteChainConfig(db, hash, params.AllEthashProtocolChanges) + rawdb.WriteChainConfig(db, hash, genesis.Config) } rawdb.WriteHeadHeaderHash(db, hash) @@ -276,13 +279,14 @@ func makeChainForBench(db ethdb.Database, full bool, count uint64) { } func benchWriteChain(b *testing.B, full bool, count uint64) { + genesis := &Genesis{Config: params.AllEthashProtocolChanges} for i := 0; i < b.N; i++ { dir := b.TempDir() db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - makeChainForBench(db, full, count) + makeChainForBench(db, genesis, full, count) db.Close() } } @@ -294,7 +298,8 @@ func benchReadChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - makeChainForBench(db, full, count) + genesis := &Genesis{Config: params.AllEthashProtocolChanges} + makeChainForBench(db, genesis, full, count) db.Close() cacheConfig := *defaultCacheConfig cacheConfig.TrieDirtyDisabled = true @@ -307,7 +312,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - chain, err := NewBlockChain(db, &cacheConfig, nil, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, &cacheConfig, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { b.Fatalf("error creating chain: %v", err) } diff --git a/core/blockchain.go b/core/blockchain.go index f458da8257..15a3bf5d05 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -185,6 +185,13 @@ func DefaultCacheConfigWithScheme(scheme string) *CacheConfig { return &config } +// txLookup is wrapper over transaction lookup along with the corresponding +// transaction object. +type txLookup struct { + lookup *rawdb.LegacyTxLookupEntry + transaction *types.Transaction +} + // BlockChain represents the canonical chain given a database with a genesis // block. The Blockchain manages chain imports, reverts, chain reorganisations. // @@ -211,13 +218,7 @@ type BlockChain struct { flushInterval atomic.Int64 // Time interval (processing time) after which to flush a state triedb *trie.Database // The database handler for maintaining trie nodes. stateCache state.Database // State database to reuse between imports (contains state cache) - - // txLookupLimit is the maximum number of blocks from head whose tx indices - // are reserved: - // * 0: means no limit and regenerate any missing indexes - // * N: means N block limit [HEAD-N+1, HEAD] and delete extra indexes - // * nil: disable tx reindexer/deleter, but still index new blocks - txLookupLimit uint64 + txIndexer *txIndexer // Transaction indexer, might be nil if not enabled hc *HeaderChain rmLogsFeed event.Feed @@ -242,15 +243,15 @@ type BlockChain struct { bodyRLPCache *lru.Cache[common.Hash, rlp.RawValue] receiptsCache *lru.Cache[common.Hash, []*types.Receipt] blockCache *lru.Cache[common.Hash, *types.Block] - txLookupCache *lru.Cache[common.Hash, *rawdb.LegacyTxLookupEntry] + txLookupCache *lru.Cache[common.Hash, txLookup] // future blocks are blocks added for later processing futureBlocks *lru.Cache[common.Hash, *types.Block] - wg sync.WaitGroup // - quit chan struct{} // shutdown signal, closed in Stop. - stopping atomic.Bool // false if chain is running, true when stopped - procInterrupt atomic.Bool // interrupt signaler for block processing + wg sync.WaitGroup + quit chan struct{} // shutdown signal, closed in Stop. + stopping atomic.Bool // false if chain is running, true when stopped + procInterrupt atomic.Bool // interrupt signaler for block processing engine consensus.Engine validator Validator // Block and state validator interface @@ -297,7 +298,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis bodyRLPCache: lru.NewCache[common.Hash, rlp.RawValue](bodyCacheLimit), receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), - txLookupCache: lru.NewCache[common.Hash, *rawdb.LegacyTxLookupEntry](txLookupCacheLimit), + txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit), futureBlocks: lru.NewCache[common.Hash, *types.Block](maxFutureBlocks), engine: engine, vmConfig: vmConfig, @@ -463,12 +464,9 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis } rawdb.WriteChainConfig(db, genesisHash, chainConfig) } - // Start tx indexer/unindexer if required. + // Start tx indexer if it's enabled. if txLookupLimit != nil { - bc.txLookupLimit = *txLookupLimit - - bc.wg.Add(1) - go bc.maintainTxIndex() + bc.txIndexer = newTxIndexer(*txLookupLimit, bc) } return bc, nil } @@ -958,7 +956,10 @@ func (bc *BlockChain) stopWithoutSaving() { if !bc.stopping.CompareAndSwap(false, true) { return } - + // Signal shutdown tx indexer. + if bc.txIndexer != nil { + bc.txIndexer.close() + } // Unsubscribe all subscriptions registered from blockchain. bc.scope.Close() @@ -1155,14 +1156,13 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ // Ensure genesis is in ancients. if first.NumberU64() == 1 { if frozen, _ := bc.db.Ancients(); frozen == 0 { - b := bc.genesisBlock td := bc.genesisBlock.Difficulty() - writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{b}, []types.Receipts{nil}, td) - size += writeSize + writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil}, td) if err != nil { log.Error("Error writing genesis to ancients", "err", err) return 0, err } + size += writeSize log.Info("Wrote genesis to ancients") } } @@ -1176,44 +1176,11 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ // Write all chain data to ancients. td := bc.GetTd(first.Hash(), first.NumberU64()) writeSize, err := rawdb.WriteAncientBlocks(bc.db, blockChain, receiptChain, td) - size += writeSize if err != nil { log.Error("Error importing chain data to ancients", "err", err) return 0, err } - - // Write tx indices if any condition is satisfied: - // * If user requires to reserve all tx indices(txlookuplimit=0) - // * If all ancient tx indices are required to be reserved(txlookuplimit is even higher than ancientlimit) - // * If block number is large enough to be regarded as a recent block - // It means blocks below the ancientLimit-txlookupLimit won't be indexed. - // - // But if the `TxIndexTail` is not nil, e.g. Geth is initialized with - // an external ancient database, during the setup, blockchain will start - // a background routine to re-indexed all indices in [ancients - txlookupLimit, ancients) - // range. In this case, all tx indices of newly imported blocks should be - // generated. - batch := bc.db.NewBatch() - for i, block := range blockChain { - if bc.txLookupLimit == 0 || ancientLimit <= bc.txLookupLimit || block.NumberU64() >= ancientLimit-bc.txLookupLimit { - rawdb.WriteTxLookupEntriesByBlock(batch, block) - } else if rawdb.ReadTxIndexTail(bc.db) != nil { - rawdb.WriteTxLookupEntriesByBlock(batch, block) - } - stats.processed++ - - if batch.ValueSize() > ethdb.IdealBatchSize || i == len(blockChain)-1 { - size += int64(batch.ValueSize()) - if err = batch.Write(); err != nil { - snapBlock := bc.CurrentSnapBlock().Number.Uint64() - if _, err := bc.db.TruncateHead(snapBlock + 1); err != nil { - log.Error("Can't truncate ancient store after failed insert", "err", err) - } - return 0, err - } - batch.Reset() - } - } + size += writeSize // Sync the ancient store explicitly to ensure all data has been flushed to disk. if err := bc.db.Sync(); err != nil { @@ -1231,8 +1198,10 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } // Delete block data from the main database. - batch.Reset() - canonHashes := make(map[common.Hash]struct{}) + var ( + batch = bc.db.NewBatch() + canonHashes = make(map[common.Hash]struct{}) + ) for _, block := range blockChain { canonHashes[block.Hash()] = struct{}{} if block.NumberU64() == 0 { @@ -1250,13 +1219,16 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if err := batch.Write(); err != nil { return 0, err } + stats.processed += int32(len(blockChain)) return 0, nil } // writeLive writes blockchain and corresponding receipt chain into active store. writeLive := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { - skipPresenceCheck := false - batch := bc.db.NewBatch() + var ( + skipPresenceCheck = false + batch = bc.db.NewBatch() + ) for i, block := range blockChain { // Short circuit insertion if shutting down or processing failed if bc.insertStopped() { @@ -1281,11 +1253,10 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ // Write all the data out into the database rawdb.WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()) rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i]) - rawdb.WriteTxLookupEntriesByBlock(batch, block) // Always write tx indices for live blocks, we assume they are needed // Write everything belongs to the blocks into the database. So that - // we can ensure all components of body is completed(body, receipts, - // tx indexes) + // we can ensure all components of body is completed(body, receipts) + // except transaction indexes(will be created once sync is finished). if batch.ValueSize() >= ethdb.IdealBatchSize { if err := batch.Write(); err != nil { return 0, err @@ -1317,19 +1288,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ return n, err } } - // Write the tx index tail (block number from where we index) before write any live blocks - if len(liveBlocks) > 0 && liveBlocks[0].NumberU64() == ancientLimit+1 { - // The tx index tail can only be one of the following two options: - // * 0: all ancient blocks have been indexed - // * ancient-limit: the indices of blocks before ancient-limit are ignored - if tail := rawdb.ReadTxIndexTail(bc.db); tail == nil { - if bc.txLookupLimit == 0 || ancientLimit <= bc.txLookupLimit { - rawdb.WriteTxIndexTail(bc.db, 0) - } else { - rawdb.WriteTxIndexTail(bc.db, ancientLimit-bc.txLookupLimit) - } - } - } if len(liveBlocks) > 0 { if n, err := writeLive(liveBlocks, liveReceipts); err != nil { if err == errInsertionInterrupted { @@ -1338,13 +1296,14 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ return n, err } } - - head := blockChain[len(blockChain)-1] - context := []interface{}{ - "count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)), - "number", head.Number(), "hash", head.Hash(), "age", common.PrettyAge(time.Unix(int64(head.Time()), 0)), - "size", common.StorageSize(size), - } + var ( + head = blockChain[len(blockChain)-1] + context = []interface{}{ + "count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)), + "number", head.Number(), "hash", head.Hash(), "age", common.PrettyAge(time.Unix(int64(head.Time()), 0)), + "size", common.StorageSize(size), + } + ) if stats.ignored > 0 { context = append(context, []interface{}{"ignored", stats.ignored}...) } @@ -1360,7 +1319,6 @@ func (bc *BlockChain) writeBlockWithoutState(block *types.Block, td *big.Int) (e if bc.insertStopped() { return errInsertionInterrupted } - batch := bc.db.NewBatch() rawdb.WriteTd(batch, block.Hash(), block.NumberU64(), td) rawdb.WriteBlock(batch, block) @@ -1715,7 +1673,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) // The chain importer is starting and stopping trie prefetchers. If a bad // block or other error is hit however, an early return may not properly // terminate the background threads. This defer ensures that we clean up - // and dangling prefetcher, without defering each and holding on live refs. + // and dangling prefetcher, without deferring each and holding on live refs. if activeState != nil { activeState.StopPrefetcher() } @@ -2230,6 +2188,12 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Block) error { // rewind the canonical chain to a lower point. log.Error("Impossible reorg, please file an issue", "oldnum", oldBlock.Number(), "oldhash", oldBlock.Hash(), "oldblocks", len(oldChain), "newnum", newBlock.Number(), "newhash", newBlock.Hash(), "newblocks", len(newChain)) } + // Reset the tx lookup cache in case to clear stale txlookups. + // This is done before writing any new chain data to avoid the + // weird scenario that canonical chain is changed while the + // stale lookups are still cached. + bc.txLookupCache.Purge() + // Insert the new chain(except the head block(reverse order)), // taking care of the proper incremental order. for i := len(newChain) - 1; i >= 1; i-- { @@ -2244,11 +2208,13 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Block) error { // Delete useless indexes right now which includes the non-canonical // transaction indexes, canonical chain indexes which above the head. - indexesBatch := bc.db.NewBatch() - for _, tx := range types.HashDifference(deletedTxs, addedTxs) { + var ( + indexesBatch = bc.db.NewBatch() + diffs = types.HashDifference(deletedTxs, addedTxs) + ) + for _, tx := range diffs { rawdb.DeleteTxLookupEntry(indexesBatch, tx) } - // Delete all hash markers that are not part of the new canonical chain. // Because the reorg function does not handle new chain head, all hash // markers greater than or equal to new chain head should be deleted. @@ -2423,102 +2389,6 @@ func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool { return false } -// indexBlocks reindexes or unindexes transactions depending on user configuration -func (bc *BlockChain) indexBlocks(tail *uint64, head uint64, done chan struct{}) { - defer func() { close(done) }() - - // If head is 0, it means the chain is just initialized and no blocks are inserted, - // so don't need to indexing anything. - if head == 0 { - return - } - - // The tail flag is not existent, it means the node is just initialized - // and all blocks(may from ancient store) are not indexed yet. - if tail == nil { - from := uint64(0) - if bc.txLookupLimit != 0 && head >= bc.txLookupLimit { - from = head - bc.txLookupLimit + 1 - } - rawdb.IndexTransactions(bc.db, from, head+1, bc.quit) - return - } - // The tail flag is existent, but the whole chain is required to be indexed. - if bc.txLookupLimit == 0 || head < bc.txLookupLimit { - if *tail > 0 { - // It can happen when chain is rewound to a historical point which - // is even lower than the indexes tail, recap the indexing target - // to new head to avoid reading non-existent block bodies. - end := *tail - if end > head+1 { - end = head + 1 - } - rawdb.IndexTransactions(bc.db, 0, end, bc.quit) - } - return - } - // Update the transaction index to the new chain state - if head-bc.txLookupLimit+1 < *tail { - // Reindex a part of missing indices and rewind index tail to HEAD-limit - rawdb.IndexTransactions(bc.db, head-bc.txLookupLimit+1, *tail, bc.quit) - } else { - // Unindex a part of stale indices and forward index tail to HEAD-limit - rawdb.UnindexTransactions(bc.db, *tail, head-bc.txLookupLimit+1, bc.quit) - } -} - -// maintainTxIndex is responsible for the construction and deletion of the -// transaction index. -// -// User can use flag `txlookuplimit` to specify a "recentness" block, below -// which ancient tx indices get deleted. If `txlookuplimit` is 0, it means -// all tx indices will be reserved. -// -// The user can adjust the txlookuplimit value for each launch after sync, -// Geth will automatically construct the missing indices or delete the extra -// indices. -func (bc *BlockChain) maintainTxIndex() { - defer bc.wg.Done() - - // Listening to chain events and manipulate the transaction indexes. - var ( - done chan struct{} // Non-nil if background unindexing or reindexing routine is active. - headCh = make(chan ChainHeadEvent, 1) // Buffered to avoid locking up the event feed - ) - sub := bc.SubscribeChainHeadEvent(headCh) - if sub == nil { - return - } - defer sub.Unsubscribe() - log.Info("Initialized transaction indexer", "limit", bc.TxLookupLimit()) - - // Launch the initial processing if chain is not empty. This step is - // useful in these scenarios that chain has no progress and indexer - // is never triggered. - if head := rawdb.ReadHeadBlock(bc.db); head != nil { - done = make(chan struct{}) - go bc.indexBlocks(rawdb.ReadTxIndexTail(bc.db), head.NumberU64(), done) - } - - for { - select { - case head := <-headCh: - if done == nil { - done = make(chan struct{}) - go bc.indexBlocks(rawdb.ReadTxIndexTail(bc.db), head.Block.NumberU64(), done) - } - case <-done: - done = nil - case <-bc.quit: - if done != nil { - log.Info("Waiting background transaction indexer to exit") - <-done - } - return - } - } -} - // reportBlock logs a bad block error. func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, err error) { rawdb.WriteBadBlock(bc.db, block) diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 466a86c144..706844171d 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -17,6 +17,7 @@ package core import ( + "errors" "math/big" "github.com/ethereum/go-ethereum/common" @@ -254,20 +255,46 @@ func (bc *BlockChain) GetAncestor(hash common.Hash, number, ancestor uint64, max return bc.hc.GetAncestor(hash, number, ancestor, maxNonCanonical) } -// GetTransactionLookup retrieves the lookup associate with the given transaction -// hash from the cache or database. -func (bc *BlockChain) GetTransactionLookup(hash common.Hash) *rawdb.LegacyTxLookupEntry { +// GetTransactionLookup retrieves the lookup along with the transaction +// itself associate with the given transaction hash. +// +// An error will be returned if the transaction is not found, and background +// indexing for transactions is still in progress. The transaction might be +// reachable shortly once it's indexed. +// +// A null will be returned in the transaction is not found and background +// transaction indexing is already finished. The transaction is not existent +// from the node's perspective. +func (bc *BlockChain) GetTransactionLookup(hash common.Hash) (*rawdb.LegacyTxLookupEntry, *types.Transaction, error) { // Short circuit if the txlookup already in the cache, retrieve otherwise - if lookup, exist := bc.txLookupCache.Get(hash); exist { - return lookup + if item, exist := bc.txLookupCache.Get(hash); exist { + return item.lookup, item.transaction, nil } tx, blockHash, blockNumber, txIndex := rawdb.ReadTransaction(bc.db, hash) if tx == nil { - return nil + progress, err := bc.TxIndexProgress() + if err != nil { + return nil, nil, nil + } + // The transaction indexing is not finished yet, returning an + // error to explicitly indicate it. + if !progress.Done() { + return nil, nil, errors.New("transaction indexing still in progress") + } + // The transaction is already indexed, the transaction is either + // not existent or not in the range of index, returning null. + return nil, nil, nil } - lookup := &rawdb.LegacyTxLookupEntry{BlockHash: blockHash, BlockIndex: blockNumber, Index: txIndex} - bc.txLookupCache.Add(hash, lookup) - return lookup + lookup := &rawdb.LegacyTxLookupEntry{ + BlockHash: blockHash, + BlockIndex: blockNumber, + Index: txIndex, + } + bc.txLookupCache.Add(hash, txLookup{ + lookup: lookup, + transaction: tx, + }) + return lookup, tx, nil } // GetTd retrieves a block's total difficulty in the canonical chain from the @@ -370,16 +397,12 @@ func (bc *BlockChain) GetVMConfig() *vm.Config { return &bc.vmConfig } -// SetTxLookupLimit is responsible for updating the txlookup limit to the -// original one stored in db if the new mismatches with the old one. -func (bc *BlockChain) SetTxLookupLimit(limit uint64) { - bc.txLookupLimit = limit -} - -// TxLookupLimit retrieves the txlookup limit used by blockchain to prune -// stale transaction indices. -func (bc *BlockChain) TxLookupLimit() uint64 { - return bc.txLookupLimit +// TxIndexProgress returns the transaction indexing progress. +func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) { + if bc.txIndexer == nil { + return TxIndexProgress{}, errors.New("tx indexer is not enabled") + } + return bc.txIndexer.txIndexProgress() } // TrieDB retrieves the low level trie database used for data storage. @@ -387,6 +410,11 @@ func (bc *BlockChain) TrieDB() *trie.Database { return bc.triedb } +// HeaderChain returns the underlying header chain. +func (bc *BlockChain) HeaderChain() *HeaderChain { + return bc.hc +} + // SubscribeRemovedLogsEvent registers a subscription of RemovedLogsEvent. func (bc *BlockChain) SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription { return bc.scope.Track(bc.rmLogsFeed.Subscribe(ch)) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index bc6f8112f0..46882f4098 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -40,6 +40,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) // So we can deterministically seed different blockchains @@ -2722,191 +2723,6 @@ func testReorgToShorterRemovesCanonMappingHeaderChain(t *testing.T, scheme strin } } -func TestTransactionIndices(t *testing.T) { - // Configure and generate a sample block chain - var ( - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(100000000000000000) - gspec = &Genesis{ - Config: params.TestChainConfig, - Alloc: GenesisAlloc{address: {Balance: funds}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - signer = types.LatestSigner(gspec.Config) - ) - _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 128, func(i int, block *BlockGen) { - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) - if err != nil { - panic(err) - } - block.AddTx(tx) - }) - - check := func(tail *uint64, chain *BlockChain) { - stored := rawdb.ReadTxIndexTail(chain.db) - if tail == nil && stored != nil { - t.Fatalf("Oldest indexded block mismatch, want nil, have %d", *stored) - } - if tail != nil && *stored != *tail { - t.Fatalf("Oldest indexded block mismatch, want %d, have %d", *tail, *stored) - } - if tail != nil { - for i := *tail; i <= chain.CurrentBlock().Number.Uint64(); i++ { - block := rawdb.ReadBlock(chain.db, rawdb.ReadCanonicalHash(chain.db, i), i) - if block.Transactions().Len() == 0 { - continue - } - for _, tx := range block.Transactions() { - if index := rawdb.ReadTxLookupEntry(chain.db, tx.Hash()); index == nil { - t.Fatalf("Miss transaction indice, number %d hash %s", i, tx.Hash().Hex()) - } - } - } - for i := uint64(0); i < *tail; i++ { - block := rawdb.ReadBlock(chain.db, rawdb.ReadCanonicalHash(chain.db, i), i) - if block.Transactions().Len() == 0 { - continue - } - for _, tx := range block.Transactions() { - if index := rawdb.ReadTxLookupEntry(chain.db, tx.Hash()); index != nil { - t.Fatalf("Transaction indice should be deleted, number %d hash %s", i, tx.Hash().Hex()) - } - } - } - } - } - // Init block chain with external ancients, check all needed indices has been indexed. - limit := []uint64{0, 32, 64, 128} - for _, l := range limit { - frdir := t.TempDir() - ancientDb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) - rawdb.WriteAncientBlocks(ancientDb, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) - - l := l - chain, err := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) - if err != nil { - t.Fatalf("failed to create tester chain: %v", err) - } - chain.indexBlocks(rawdb.ReadTxIndexTail(ancientDb), 128, make(chan struct{})) - - var tail uint64 - if l != 0 { - tail = uint64(128) - l + 1 - } - check(&tail, chain) - chain.Stop() - ancientDb.Close() - os.RemoveAll(frdir) - } - - // Reconstruct a block chain which only reserves HEAD-64 tx indices - ancientDb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) - defer ancientDb.Close() - - rawdb.WriteAncientBlocks(ancientDb, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) - limit = []uint64{0, 64 /* drop stale */, 32 /* shorten history */, 64 /* extend history */, 0 /* restore all */} - for _, l := range limit { - l := l - chain, err := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) - if err != nil { - t.Fatalf("failed to create tester chain: %v", err) - } - var tail uint64 - if l != 0 { - tail = uint64(128) - l + 1 - } - chain.indexBlocks(rawdb.ReadTxIndexTail(ancientDb), 128, make(chan struct{})) - check(&tail, chain) - chain.Stop() - } -} - -func TestSkipStaleTxIndicesInSnapSync(t *testing.T) { - testSkipStaleTxIndicesInSnapSync(t, rawdb.HashScheme) - testSkipStaleTxIndicesInSnapSync(t, rawdb.PathScheme) -} - -func testSkipStaleTxIndicesInSnapSync(t *testing.T, scheme string) { - // Configure and generate a sample block chain - var ( - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(100000000000000000) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} - signer = types.LatestSigner(gspec.Config) - ) - _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 128, func(i int, block *BlockGen) { - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) - if err != nil { - panic(err) - } - block.AddTx(tx) - }) - - check := func(tail *uint64, chain *BlockChain) { - stored := rawdb.ReadTxIndexTail(chain.db) - if tail == nil && stored != nil { - t.Fatalf("Oldest indexded block mismatch, want nil, have %d", *stored) - } - if tail != nil && *stored != *tail { - t.Fatalf("Oldest indexded block mismatch, want %d, have %d", *tail, *stored) - } - if tail != nil { - for i := *tail; i <= chain.CurrentBlock().Number.Uint64(); i++ { - block := rawdb.ReadBlock(chain.db, rawdb.ReadCanonicalHash(chain.db, i), i) - if block.Transactions().Len() == 0 { - continue - } - for _, tx := range block.Transactions() { - if index := rawdb.ReadTxLookupEntry(chain.db, tx.Hash()); index == nil { - t.Fatalf("Miss transaction indice, number %d hash %s", i, tx.Hash().Hex()) - } - } - } - for i := uint64(0); i < *tail; i++ { - block := rawdb.ReadBlock(chain.db, rawdb.ReadCanonicalHash(chain.db, i), i) - if block.Transactions().Len() == 0 { - continue - } - for _, tx := range block.Transactions() { - if index := rawdb.ReadTxLookupEntry(chain.db, tx.Hash()); index != nil { - t.Fatalf("Transaction indice should be deleted, number %d hash %s", i, tx.Hash().Hex()) - } - } - } - } - } - - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) - if err != nil { - t.Fatalf("failed to create temp freezer db: %v", err) - } - defer ancientDb.Close() - - // Import all blocks into ancient db, only HEAD-32 indices are kept. - l := uint64(32) - chain, err := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) - if err != nil { - t.Fatalf("failed to create tester chain: %v", err) - } - defer chain.Stop() - - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - if n, err := chain.InsertHeaderChain(headers); err != nil { - t.Fatalf("failed to insert header %d: %v", n, err) - } - // The indices before ancient-N(32) should be ignored. After that all blocks should be indexed. - if n, err := chain.InsertReceiptChain(blocks, receipts, 64); err != nil { - t.Fatalf("block %d: failed to insert into chain: %v", n, err) - } - tail := uint64(32) - check(&tail, chain) -} - // Benchmarks large blocks with value transfers to non-existing accounts func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks int, recipientFn func(uint64) common.Address, dataFn func(uint64) []byte) { var ( @@ -3652,7 +3468,7 @@ func testInitThenFailCreateContract(t *testing.T, scheme string) { defer chain.Stop() statedb, _ := chain.State() - if got, exp := statedb.GetBalance(aa), big.NewInt(100000); got.Cmp(exp) != 0 { + if got, exp := statedb.GetBalance(aa), uint256.NewInt(100000); got.Cmp(exp) != 0 { t.Fatalf("Genesis err, got %v exp %v", got, exp) } // First block tries to create, but fails @@ -3662,7 +3478,7 @@ func testInitThenFailCreateContract(t *testing.T, scheme string) { t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err) } statedb, _ = chain.State() - if got, exp := statedb.GetBalance(aa), big.NewInt(100000); got.Cmp(exp) != 0 { + if got, exp := statedb.GetBalance(aa), uint256.NewInt(100000); got.Cmp(exp) != 0 { t.Fatalf("block %d: got %v exp %v", block.NumberU64(), got, exp) } } @@ -3848,17 +3664,17 @@ func testEIP1559Transition(t *testing.T, scheme string) { state, _ := chain.State() // 3: Ensure that miner received only the tx's tip. - actual := state.GetBalance(block.Coinbase()) + actual := state.GetBalance(block.Coinbase()).ToBig() expected := new(big.Int).Add( new(big.Int).SetUint64(block.GasUsed()*block.Transactions()[0].GasTipCap().Uint64()), - ethash.ConstantinopleBlockReward, + ethash.ConstantinopleBlockReward.ToBig(), ) if actual.Cmp(expected) != 0 { t.Fatalf("miner balance incorrect: expected %d, got %d", expected, actual) } // 4: Ensure the tx sender paid for the gasUsed * (tip + block baseFee). - actual = new(big.Int).Sub(funds, state.GetBalance(addr1)) + actual = new(big.Int).Sub(funds, state.GetBalance(addr1).ToBig()) expected = new(big.Int).SetUint64(block.GasUsed() * (block.Transactions()[0].GasTipCap().Uint64() + block.BaseFee().Uint64())) if actual.Cmp(expected) != 0 { t.Fatalf("sender balance incorrect: expected %d, got %d", expected, actual) @@ -3888,17 +3704,17 @@ func testEIP1559Transition(t *testing.T, scheme string) { effectiveTip := block.Transactions()[0].GasTipCap().Uint64() - block.BaseFee().Uint64() // 6+5: Ensure that miner received only the tx's effective tip. - actual = state.GetBalance(block.Coinbase()) + actual = state.GetBalance(block.Coinbase()).ToBig() expected = new(big.Int).Add( new(big.Int).SetUint64(block.GasUsed()*effectiveTip), - ethash.ConstantinopleBlockReward, + ethash.ConstantinopleBlockReward.ToBig(), ) if actual.Cmp(expected) != 0 { t.Fatalf("miner balance incorrect: expected %d, got %d", expected, actual) } // 4: Ensure the tx sender paid for the gasUsed * (effectiveTip + block baseFee). - actual = new(big.Int).Sub(funds, state.GetBalance(addr2)) + actual = new(big.Int).Sub(funds, state.GetBalance(addr2).ToBig()) expected = new(big.Int).SetUint64(block.GasUsed() * (effectiveTip + block.BaseFee().Uint64())) if actual.Cmp(expected) != 0 { t.Fatalf("sender balance incorrect: expected %d, got %d", expected, actual) @@ -4103,212 +3919,6 @@ func testCanonicalHashMarker(t *testing.T, scheme string) { } } -// TestTxIndexer tests the tx indexes are updated correctly. -func TestTxIndexer(t *testing.T) { - var ( - testBankKey, _ = crypto.GenerateKey() - testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) - testBankFunds = big.NewInt(1000000000000000000) - - gspec = &Genesis{ - Config: params.TestChainConfig, - Alloc: GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - engine = ethash.NewFaker() - nonce = uint64(0) - ) - _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, 128, func(i int, gen *BlockGen) { - tx, _ := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("0xdeadbeef"), big.NewInt(1000), params.TxGas, big.NewInt(10*params.InitialBaseFee), nil), types.HomesteadSigner{}, testBankKey) - gen.AddTx(tx) - nonce += 1 - }) - - // verifyIndexes checks if the transaction indexes are present or not - // of the specified block. - verifyIndexes := func(db ethdb.Database, number uint64, exist bool) { - if number == 0 { - return - } - block := blocks[number-1] - for _, tx := range block.Transactions() { - lookup := rawdb.ReadTxLookupEntry(db, tx.Hash()) - if exist && lookup == nil { - t.Fatalf("missing %d %x", number, tx.Hash().Hex()) - } - if !exist && lookup != nil { - t.Fatalf("unexpected %d %x", number, tx.Hash().Hex()) - } - } - } - // verifyRange runs verifyIndexes for a range of blocks, from and to are included. - verifyRange := func(db ethdb.Database, from, to uint64, exist bool) { - for number := from; number <= to; number += 1 { - verifyIndexes(db, number, exist) - } - } - verify := func(db ethdb.Database, expTail uint64) { - tail := rawdb.ReadTxIndexTail(db) - if tail == nil { - t.Fatal("Failed to write tx index tail") - } - if *tail != expTail { - t.Fatalf("Unexpected tx index tail, want %v, got %d", expTail, *tail) - } - if *tail != 0 { - verifyRange(db, 0, *tail-1, false) - } - verifyRange(db, *tail, 128, true) - } - - var cases = []struct { - limitA uint64 - tailA uint64 - limitB uint64 - tailB uint64 - limitC uint64 - tailC uint64 - }{ - { - // LimitA: 0 - // TailA: 0 - // - // all blocks are indexed - limitA: 0, - tailA: 0, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, - }, - { - // LimitA: 64 - // TailA: 65 - // - // block [65, 128] are indexed - limitA: 64, - tailA: 65, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, - }, - { - // LimitA: 127 - // TailA: 2 - // - // block [2, 128] are indexed - limitA: 127, - tailA: 2, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, - }, - { - // LimitA: 128 - // TailA: 1 - // - // block [2, 128] are indexed - limitA: 128, - tailA: 1, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, - }, - { - // LimitA: 129 - // TailA: 0 - // - // block [0, 128] are indexed - limitA: 129, - tailA: 0, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, - }, - } - for _, c := range cases { - frdir := t.TempDir() - db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) - rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) - - // Index the initial blocks from ancient store - chain, _ := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, &c.limitA) - chain.indexBlocks(nil, 128, make(chan struct{})) - verify(db, c.tailA) - - chain.SetTxLookupLimit(c.limitB) - chain.indexBlocks(rawdb.ReadTxIndexTail(db), 128, make(chan struct{})) - verify(db, c.tailB) - - chain.SetTxLookupLimit(c.limitC) - chain.indexBlocks(rawdb.ReadTxIndexTail(db), 128, make(chan struct{})) - verify(db, c.tailC) - - // Recover all indexes - chain.SetTxLookupLimit(0) - chain.indexBlocks(rawdb.ReadTxIndexTail(db), 128, make(chan struct{})) - verify(db, 0) - - chain.Stop() - db.Close() - os.RemoveAll(frdir) - } -} - func TestCreateThenDeletePreByzantium(t *testing.T) { // We use Ropsten chain config instead of Testchain config, this is // deliberate: we want to use pre-byz rules where we have intermediate state roots @@ -4703,14 +4313,14 @@ func TestEIP3651(t *testing.T) { state, _ := chain.State() // 3: Ensure that miner received only the tx's tip. - actual := state.GetBalance(block.Coinbase()) + actual := state.GetBalance(block.Coinbase()).ToBig() expected := new(big.Int).SetUint64(block.GasUsed() * block.Transactions()[0].GasTipCap().Uint64()) if actual.Cmp(expected) != 0 { t.Fatalf("miner balance incorrect: expected %d, got %d", expected, actual) } // 4: Ensure the tx sender paid for the gasUsed * (tip + block baseFee). - actual = new(big.Int).Sub(funds, state.GetBalance(addr1)) + actual = new(big.Int).Sub(funds, state.GetBalance(addr1).ToBig()) expected = new(big.Int).SetUint64(block.GasUsed() * (block.Transactions()[0].GasTipCap().Uint64() + block.BaseFee().Uint64())) if actual.Cmp(expected) != 0 { t.Fatalf("sender balance incorrect: expected %d, got %d", expected, actual) diff --git a/core/chain_makers.go b/core/chain_makers.go index 31c111b73e..5b979dfc41 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) // BlockGen creates blocks for testing. @@ -82,7 +83,7 @@ func (b *BlockGen) SetDifficulty(diff *big.Int) { b.header.Difficulty = diff } -// SetPos makes the header a PoS-header (0 difficulty) +// SetPoS makes the header a PoS-header (0 difficulty) func (b *BlockGen) SetPoS() { b.header.Difficulty = new(big.Int) } @@ -157,7 +158,7 @@ func (b *BlockGen) AddTxWithVMConfig(tx *types.Transaction, config vm.Config) { } // GetBalance returns the balance of the given address at the generated block. -func (b *BlockGen) GetBalance(addr common.Address) *big.Int { +func (b *BlockGen) GetBalance(addr common.Address) *uint256.Int { return b.statedb.GetBalance(addr) } diff --git a/core/evm.go b/core/evm.go index c4801dc797..73f6d7bc20 100644 --- a/core/evm.go +++ b/core/evm.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/holiman/uint256" ) // ChainContext supports retrieving headers and consensus parameters from the @@ -129,12 +130,12 @@ func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash // CanTransfer checks whether there are enough funds in the address' account to make a transfer. // This does not take the necessary gas in to account to make the transfer valid. -func CanTransfer(db vm.StateDB, addr common.Address, amount *big.Int) bool { +func CanTransfer(db vm.StateDB, addr common.Address, amount *uint256.Int) bool { return db.GetBalance(addr).Cmp(amount) >= 0 } // Transfer subtracts amount from sender and adds amount to recipient using the given Db -func Transfer(db vm.StateDB, sender, recipient common.Address, amount *big.Int) { +func Transfer(db vm.StateDB, sender, recipient common.Address, amount *uint256.Int) { db.SubBalance(sender, amount) db.AddBalance(recipient, amount) } diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index 776c428f75..b9d346bd90 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -74,8 +74,10 @@ func TestCreation(t *testing.T) { {15049999, 0, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // Last Arrow Glacier block {15050000, 0, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 1681338455}}, // First Gray Glacier block {20000000, 1681338454, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 1681338455}}, // Last Gray Glacier block - {20000000, 1681338455, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}}, // First Shanghai block - {30000000, 2000000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}}, // Future Shanghai block + {20000000, 1681338455, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // First Shanghai block + {30000000, 1710338134, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // Last Shanghai block + {40000000, 1710338135, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // First Cancun block + {50000000, 2000000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // Future Cancun block }, }, // Goerli test cases @@ -141,6 +143,7 @@ func TestValidation(t *testing.T) { // Config that has not timestamp enabled legacyConfig := *params.MainnetChainConfig legacyConfig.ShanghaiTime = nil + legacyConfig.CancunTime = nil tests := []struct { config *params.ChainConfig @@ -213,14 +216,10 @@ func TestValidation(t *testing.T) { // at some future block 88888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - // - // TODO(karalabe): This testcase will fail once mainnet gets timestamped forks, make legacy chain config {&legacyConfig, 88888888, 0, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 88888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Byzantium. Remote is also in Byzantium, but announces Gopherium (non existing // fork) at block 7279999, before Petersburg. Local is incompatible. - // - // TODO(karalabe): This testcase will fail once mainnet gets timestamped forks, make legacy chain config {&legacyConfig, 7279999, 0, ID{Hash: checksumToBytes(0xa00bc324), Next: 7279999}, ErrLocalIncompatibleOrStale}, //------------------------------------ @@ -297,34 +296,25 @@ func TestValidation(t *testing.T) { // Local is mainnet currently in Shanghai only (so it's aware of Cancun), remote announces // also Shanghai, but it's not yet aware of Cancun (e.g. non updated node before the fork). // In this case we don't know if Cancun passed yet or not. - // - // TODO(karalabe): Enable this when Cancun is specced - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0x71147644), Next: 0}, nil}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}, nil}, // Local is mainnet currently in Shanghai only (so it's aware of Cancun), remote announces // also Shanghai, and it's also aware of Cancun (e.g. updated node before the fork). We // don't know if Cancun passed yet (will pass) or not. - // - // TODO(karalabe): Enable this when Cancun is specced and update next timestamp - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, nil}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}, nil}, // Local is mainnet currently in Shanghai only (so it's aware of Cancun), remote announces // also Shanghai, and it's also aware of some random fork (e.g. misconfigured Cancun). As // neither forks passed at neither nodes, they may mismatch, but we still connect for now. - // - // TODO(karalabe): Enable this when Cancun is specced - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0x71147644), Next: math.MaxUint64}, nil}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: math.MaxUint64}, nil}, // Local is mainnet exactly on Cancun, remote announces Shanghai + knowledge about Cancun. Remote // is simply out of sync, accept. - // - // TODO(karalabe): Enable this when Cancun is specced, update local head and time, next timestamp - // {params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, nil}, + {params.MainnetChainConfig, 21000000, 1710338135, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}, nil}, // Local is mainnet Cancun, remote announces Shanghai + knowledge about Cancun. Remote // is simply out of sync, accept. - // TODO(karalabe): Enable this when Cancun is specced, update local head and time, next timestamp - //{params.MainnetChainConfig, 21123456, 1678123456, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, nil}, + {params.MainnetChainConfig, 21123456, 1710338136, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}, nil}, // Local is mainnet Prague, remote announces Shanghai + knowledge about Cancun. Remote // is definitely out of sync. It may or may not need the Prague update, we don't know yet. @@ -333,9 +323,7 @@ func TestValidation(t *testing.T) { //{params.MainnetChainConfig, 0, 0, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}, nil}, // Local is mainnet Shanghai, remote announces Cancun. Local is out of sync, accept. - // - // TODO(karalabe): Enable this when Cancun is specced, update remote checksum - //{params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x00000000), Next: 0}, nil}, + {params.MainnetChainConfig, 21000000, 1700000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}, nil}, // Local is mainnet Shanghai, remote announces Cancun, but is not aware of Prague. Local // out of sync. Local also knows about a future fork, but that is uncertain yet. @@ -345,9 +333,7 @@ func TestValidation(t *testing.T) { // Local is mainnet Cancun. remote announces Shanghai but is not aware of further forks. // Remote needs software update. - // - // TODO(karalabe): Enable this when Cancun is specced, update local head and time - //{params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x71147644), Next: 0}, ErrRemoteStale}, + {params.MainnetChainConfig, 21000000, 1710338135, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}, ErrRemoteStale}, // Local is mainnet Shanghai, and isn't aware of more forks. Remote announces Shanghai + // 0xffffffff. Local needs software update, reject. @@ -355,24 +341,20 @@ func TestValidation(t *testing.T) { // Local is mainnet Shanghai, and is aware of Cancun. Remote announces Cancun + // 0xffffffff. Local needs software update, reject. - // - // TODO(karalabe): Enable this when Cancun is specced, update remote checksum - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(checksumUpdate(0x00000000, math.MaxUint64)), Next: 0}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(checksumUpdate(0x9f3d2254, math.MaxUint64)), Next: 0}, ErrLocalIncompatibleOrStale}, // Local is mainnet Shanghai, remote is random Shanghai. {params.MainnetChainConfig, 20000000, 1681338455, ID{Hash: checksumToBytes(0x12345678), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Shanghai, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Cancun, far in the future. Remote announces Gopherium (non existing fork) // at some future timestamp 8888888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0xdce96c2d), Next: 8888888888}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0x9f3d2254), Next: 8888888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Shanghai. Remote is also in Shanghai, but announces Gopherium (non existing // fork) at timestamp 1668000000, before Cancun. Local is incompatible. - // - // TODO(karalabe): Enable this when Cancun is specced - //{params.MainnetChainConfig, 20999999, 1677999999, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 20999999, 1699999999, ID{Hash: checksumToBytes(0x71147644), Next: 1700000000}, ErrLocalIncompatibleOrStale}, } genesis := core.DefaultGenesisBlock().ToBlock() for i, tt := range tests { diff --git a/core/genesis.go b/core/genesis.go index 634be9a9e0..7a7bd194a5 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/holiman/uint256" ) //go:generate go run github.com/fjl/gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go @@ -142,7 +143,7 @@ func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) { } for addr, account := range *ga { if account.Balance != nil { - statedb.AddBalance(addr, account.Balance) + statedb.AddBalance(addr, uint256.MustFromBig(account.Balance)) } statedb.SetCode(addr, account.Code) statedb.SetNonce(addr, account.Nonce) @@ -163,7 +164,7 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhas } for addr, account := range *ga { if account.Balance != nil { - statedb.AddBalance(addr, account.Balance) + statedb.AddBalance(addr, uint256.MustFromBig(account.Balance)) } statedb.SetCode(addr, account.Code) statedb.SetNonce(addr, account.Nonce) @@ -412,6 +413,8 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { return g.Config case ghash == params.MainnetGenesisHash: return params.MainnetChainConfig + case ghash == params.HoleskyGenesisHash: + return params.HoleskyChainConfig case ghash == params.SepoliaGenesisHash: return params.SepoliaChainConfig case ghash == params.GoerliGenesisHash: diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index d9a89fe90c..964b3a311d 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -278,23 +278,6 @@ func WriteTxIndexTail(db ethdb.KeyValueWriter, number uint64) { } } -// ReadFastTxLookupLimit retrieves the tx lookup limit used in fast sync. -func ReadFastTxLookupLimit(db ethdb.KeyValueReader) *uint64 { - data, _ := db.Get(fastTxLookupLimitKey) - if len(data) != 8 { - return nil - } - number := binary.BigEndian.Uint64(data) - return &number -} - -// WriteFastTxLookupLimit stores the txlookup limit used in fast sync into database. -func WriteFastTxLookupLimit(db ethdb.KeyValueWriter, number uint64) { - if err := db.Put(fastTxLookupLimitKey, encodeBlockNumber(number)); err != nil { - log.Crit("Failed to store transaction lookup limit for fast sync", "err", err) - } -} - // ReadHeaderRange returns the rlp-encoded headers, starting at 'number', and going // backwards towards genesis. This method assumes that the caller already has // placed a cap on count, to prevent DoS issues. diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 56bb15b718..759e5913d1 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -178,7 +178,7 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool // // There is a passed channel, the whole procedure will be interrupted if any // signal received. -func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { +func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool, report bool) { // short circuit for invalid range if from >= to { return @@ -188,13 +188,13 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan batch = db.NewBatch() start = time.Now() logged = start.Add(-7 * time.Second) + // Since we iterate in reverse, we expect the first number to come // in to be [to-1]. Therefore, setting lastNum to means that the - // prqueue gap-evaluation will work correctly - lastNum = to - queue = prque.New[int64, *blockTxHashes](nil) - // for stats reporting - blocks, txs = 0, 0 + // queue gap-evaluation will work correctly + lastNum = to + queue = prque.New[int64, *blockTxHashes](nil) + blocks, txs = 0, 0 // for stats reporting ) for chanDelivery := range hashesCh { // Push the delivery into the queue and process contiguous ranges. @@ -240,11 +240,15 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan log.Crit("Failed writing batch to db", "error", err) return } + logger := log.Debug + if report { + logger = log.Info + } select { case <-interrupt: - log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) default: - log.Debug("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) } } @@ -257,20 +261,20 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan // // There is a passed channel, the whole procedure will be interrupted if any // signal received. -func IndexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { - indexTransactions(db, from, to, interrupt, nil) +func IndexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, report bool) { + indexTransactions(db, from, to, interrupt, nil, report) } // indexTransactionsForTesting is the internal debug version with an additional hook. func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { - indexTransactions(db, from, to, interrupt, hook) + indexTransactions(db, from, to, interrupt, hook, false) } // unindexTransactions removes txlookup indices of the specified block range. // // There is a passed channel, the whole procedure will be interrupted if any // signal received. -func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { +func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool, report bool) { // short circuit for invalid range if from >= to { return @@ -280,12 +284,12 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch batch = db.NewBatch() start = time.Now() logged = start.Add(-7 * time.Second) + // we expect the first number to come in to be [from]. Therefore, setting - // nextNum to from means that the prqueue gap-evaluation will work correctly - nextNum = from - queue = prque.New[int64, *blockTxHashes](nil) - // for stats reporting - blocks, txs = 0, 0 + // nextNum to from means that the queue gap-evaluation will work correctly + nextNum = from + queue = prque.New[int64, *blockTxHashes](nil) + blocks, txs = 0, 0 // for stats reporting ) // Otherwise spin up the concurrent iterator and unindexer for delivery := range hashesCh { @@ -332,11 +336,15 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch log.Crit("Failed writing batch to db", "error", err) return } + logger := log.Debug + if report { + logger = log.Info + } select { case <-interrupt: - log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) default: - log.Debug("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) } } @@ -345,11 +353,11 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch // // There is a passed channel, the whole procedure will be interrupted if any // signal received. -func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { - unindexTransactions(db, from, to, interrupt, nil) +func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, report bool) { + unindexTransactions(db, from, to, interrupt, nil, report) } // unindexTransactionsForTesting is the internal debug version with an additional hook. func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { - unindexTransactions(db, from, to, interrupt, hook) + unindexTransactions(db, from, to, interrupt, hook, false) } diff --git a/core/rawdb/chain_iterator_test.go b/core/rawdb/chain_iterator_test.go index 9580cd92a8..78b0a82e10 100644 --- a/core/rawdb/chain_iterator_test.go +++ b/core/rawdb/chain_iterator_test.go @@ -162,18 +162,18 @@ func TestIndexTransactions(t *testing.T) { t.Fatalf("Transaction tail mismatch") } } - IndexTransactions(chainDb, 5, 11, nil) + IndexTransactions(chainDb, 5, 11, nil, false) verify(5, 11, true, 5) verify(0, 5, false, 5) - IndexTransactions(chainDb, 0, 5, nil) + IndexTransactions(chainDb, 0, 5, nil, false) verify(0, 11, true, 0) - UnindexTransactions(chainDb, 0, 5, nil) + UnindexTransactions(chainDb, 0, 5, nil, false) verify(5, 11, true, 5) verify(0, 5, false, 5) - UnindexTransactions(chainDb, 5, 11, nil) + UnindexTransactions(chainDb, 5, 11, nil, false) verify(0, 11, false, 11) // Testing corner cases @@ -190,7 +190,7 @@ func TestIndexTransactions(t *testing.T) { }) verify(9, 11, true, 9) verify(0, 9, false, 9) - IndexTransactions(chainDb, 0, 9, nil) + IndexTransactions(chainDb, 0, 9, nil, false) signal = make(chan struct{}) var once2 sync.Once diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 18b5bccb51..27a9ec7412 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -657,7 +657,6 @@ func ReadChainMetadata(db ethdb.KeyValueStore) [][]string { {"snapshotRecoveryNumber", pp(ReadSnapshotRecoveryNumber(db))}, {"snapshotRoot", fmt.Sprintf("%v", ReadSnapshotRoot(db))}, {"txIndexTail", pp(ReadTxIndexTail(db))}, - {"fastTxLookupLimit", pp(ReadFastTxLookupLimit(db))}, } if b := ReadSkeletonSyncStatus(db); b != nil { data = append(data, []string{"SkeletonSyncStatus", string(b)}) diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 4471463932..91b4943e59 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -894,7 +894,7 @@ func getChunk(size int, b int) []byte { } // TODO (?) -// - test that if we remove several head-files, aswell as data last data-file, +// - test that if we remove several head-files, as well as data last data-file, // the index is truncated accordingly // Right now, the freezer would fail on these conditions: // 1. have data files d0, d1, d2, d3 diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index be03723553..11cf5b40fe 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -80,6 +80,8 @@ var ( txIndexTailKey = []byte("TransactionIndexTail") // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync. + // This flag is deprecated, it's kept to avoid reporting errors when inspect + // database. fastTxLookupLimitKey = []byte("FastTransactionLookupLimit") // badBlockKey tracks the list of bad blocks seen by local diff --git a/core/state/journal.go b/core/state/journal.go index 137ec76395..6cdc1fc868 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -17,9 +17,8 @@ package state import ( - "math/big" - "github.com/ethereum/go-ethereum/common" + "github.com/holiman/uint256" ) // journalEntry is a modification entry in the state change journal that can be @@ -103,13 +102,13 @@ type ( selfDestructChange struct { account *common.Address prev bool // whether account had already self-destructed - prevbalance *big.Int + prevbalance *uint256.Int } // Changes to individual accounts. balanceChange struct { account *common.Address - prev *big.Int + prev *uint256.Int } nonceChange struct { account *common.Address diff --git a/core/state/pruner/bloom.go b/core/state/pruner/bloom.go index 9f068eaf2d..dad2b5b2a8 100644 --- a/core/state/pruner/bloom.go +++ b/core/state/pruner/bloom.go @@ -27,17 +27,10 @@ import ( bloomfilter "github.com/holiman/bloomfilter/v2" ) -// stateBloomHasher is a wrapper around a byte blob to satisfy the interface API -// requirements of the bloom library used. It's used to convert a trie hash or -// contract code hash into a 64 bit mini hash. -type stateBloomHasher []byte - -func (f stateBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } -func (f stateBloomHasher) Sum(b []byte) []byte { panic("not implemented") } -func (f stateBloomHasher) Reset() { panic("not implemented") } -func (f stateBloomHasher) BlockSize() int { panic("not implemented") } -func (f stateBloomHasher) Size() int { return 8 } -func (f stateBloomHasher) Sum64() uint64 { return binary.BigEndian.Uint64(f) } +// stateBloomHash is used to convert a trie hash or contract code hash into a 64 bit mini hash. +func stateBloomHash(f []byte) uint64 { + return binary.BigEndian.Uint64(f) +} // stateBloom is a bloom filter used during the state conversion(snapshot->state). // The keys of all generated entries will be recorded here so that in the pruning @@ -113,10 +106,10 @@ func (bloom *stateBloom) Put(key []byte, value []byte) error { if !isCode { return errors.New("invalid entry") } - bloom.bloom.Add(stateBloomHasher(codeKey)) + bloom.bloom.AddHash(stateBloomHash(codeKey)) return nil } - bloom.bloom.Add(stateBloomHasher(key)) + bloom.bloom.AddHash(stateBloomHash(key)) return nil } @@ -128,5 +121,5 @@ func (bloom *stateBloom) Delete(key []byte) error { panic("not supported") } // - If it says yes, the key may be contained // - If it says no, the key is definitely not contained. func (bloom *stateBloom) Contain(key []byte) bool { - return bloom.bloom.Contains(stateBloomHasher(key)) + return bloom.bloom.ContainsHash(stateBloomHash(key)) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index a0f95078d0..b7398f2138 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -121,7 +121,7 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta // the trie nodes(and codes) belong to the active state will be filtered // out. A very small part of stale tries will also be filtered because of // the false-positive rate of bloom filter. But the assumption is held here - // that the false-positive is low enough(~0.05%). The probablity of the + // that the false-positive is low enough(~0.05%). The probability of the // dangling node is the state root is super low. So the dangling nodes in // theory will never ever be visited again. var ( diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index b6aca599c5..70c9f44189 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -43,7 +43,7 @@ var ( aggregatorMemoryLimit = uint64(4 * 1024 * 1024) // aggregatorItemLimit is an approximate number of items that will end up - // in the agregator layer before it's flushed out to disk. A plain account + // in the aggregator layer before it's flushed out to disk. A plain account // weighs around 14B (+hash), a storage slot 32B (+hash), a deleted slot // 0B (+hash). Slots are mostly set/unset in lockstep, so that average at // 16B (+hash). All in all, the average entry seems to be 15+32=47B. Use a @@ -124,47 +124,20 @@ type diffLayer struct { lock sync.RWMutex } -// destructBloomHasher is a wrapper around a common.Hash to satisfy the interface -// API requirements of the bloom library used. It's used to convert a destruct -// event into a 64 bit mini hash. -type destructBloomHasher common.Hash - -func (h destructBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } -func (h destructBloomHasher) Sum(b []byte) []byte { panic("not implemented") } -func (h destructBloomHasher) Reset() { panic("not implemented") } -func (h destructBloomHasher) BlockSize() int { panic("not implemented") } -func (h destructBloomHasher) Size() int { return 8 } -func (h destructBloomHasher) Sum64() uint64 { +// destructBloomHash is used to convert a destruct event into a 64 bit mini hash. +func destructBloomHash(h common.Hash) uint64 { return binary.BigEndian.Uint64(h[bloomDestructHasherOffset : bloomDestructHasherOffset+8]) } -// accountBloomHasher is a wrapper around a common.Hash to satisfy the interface -// API requirements of the bloom library used. It's used to convert an account -// hash into a 64 bit mini hash. -type accountBloomHasher common.Hash - -func (h accountBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } -func (h accountBloomHasher) Sum(b []byte) []byte { panic("not implemented") } -func (h accountBloomHasher) Reset() { panic("not implemented") } -func (h accountBloomHasher) BlockSize() int { panic("not implemented") } -func (h accountBloomHasher) Size() int { return 8 } -func (h accountBloomHasher) Sum64() uint64 { +// accountBloomHash is used to convert an account hash into a 64 bit mini hash. +func accountBloomHash(h common.Hash) uint64 { return binary.BigEndian.Uint64(h[bloomAccountHasherOffset : bloomAccountHasherOffset+8]) } -// storageBloomHasher is a wrapper around a [2]common.Hash to satisfy the interface -// API requirements of the bloom library used. It's used to convert an account -// hash into a 64 bit mini hash. -type storageBloomHasher [2]common.Hash - -func (h storageBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } -func (h storageBloomHasher) Sum(b []byte) []byte { panic("not implemented") } -func (h storageBloomHasher) Reset() { panic("not implemented") } -func (h storageBloomHasher) BlockSize() int { panic("not implemented") } -func (h storageBloomHasher) Size() int { return 8 } -func (h storageBloomHasher) Sum64() uint64 { - return binary.BigEndian.Uint64(h[0][bloomStorageHasherOffset:bloomStorageHasherOffset+8]) ^ - binary.BigEndian.Uint64(h[1][bloomStorageHasherOffset:bloomStorageHasherOffset+8]) +// storageBloomHash is used to convert an account hash and a storage hash into a 64 bit mini hash. +func storageBloomHash(h0, h1 common.Hash) uint64 { + return binary.BigEndian.Uint64(h0[bloomStorageHasherOffset:bloomStorageHasherOffset+8]) ^ + binary.BigEndian.Uint64(h1[bloomStorageHasherOffset:bloomStorageHasherOffset+8]) } // newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low @@ -233,14 +206,14 @@ func (dl *diffLayer) rebloom(origin *diskLayer) { } // Iterate over all the accounts and storage slots and index them for hash := range dl.destructSet { - dl.diffed.Add(destructBloomHasher(hash)) + dl.diffed.AddHash(destructBloomHash(hash)) } for hash := range dl.accountData { - dl.diffed.Add(accountBloomHasher(hash)) + dl.diffed.AddHash(accountBloomHash(hash)) } for accountHash, slots := range dl.storageData { for storageHash := range slots { - dl.diffed.Add(storageBloomHasher{accountHash, storageHash}) + dl.diffed.AddHash(storageBloomHash(accountHash, storageHash)) } } // Calculate the current false positive rate and update the error rate meter. @@ -301,9 +274,9 @@ func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) { } // Check the bloom filter first whether there's even a point in reaching into // all the maps in all the layers below - hit := dl.diffed.Contains(accountBloomHasher(hash)) + hit := dl.diffed.ContainsHash(accountBloomHash(hash)) if !hit { - hit = dl.diffed.Contains(destructBloomHasher(hash)) + hit = dl.diffed.ContainsHash(destructBloomHash(hash)) } var origin *diskLayer if !hit { @@ -372,9 +345,9 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro dl.lock.RUnlock() return nil, ErrSnapshotStale } - hit := dl.diffed.Contains(storageBloomHasher{accountHash, storageHash}) + hit := dl.diffed.ContainsHash(storageBloomHash(accountHash, storageHash)) if !hit { - hit = dl.diffed.Contains(destructBloomHasher(accountHash)) + hit = dl.diffed.ContainsHash(destructBloomHash(accountHash)) } var origin *diskLayer if !hit { diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index f95b798515..168458c405 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -139,7 +139,7 @@ func TestDiskMerge(t *testing.T) { // Retrieve all the data through the disk layer and validate it base = snaps.Snapshot(diffRoot) if _, ok := base.(*diskLayer); !ok { - t.Fatalf("update not flattend into the disk layer") + t.Fatalf("update not flattened into the disk layer") } // assertAccount ensures that an account matches the given blob. @@ -362,7 +362,7 @@ func TestDiskPartialMerge(t *testing.T) { // Retrieve all the data through the disk layer and validate it base = snaps.Snapshot(diffRoot) if _, ok := base.(*diskLayer); !ok { - t.Fatalf("test %d: update not flattend into the disk layer", i) + t.Fatalf("test %d: update not flattened into the disk layer", i) } assertAccount(accNoModNoCache, accNoModNoCache[:]) assertAccount(accNoModCache, accNoModCache[:]) diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index c25f3e7e8b..7d941f6285 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -18,7 +18,6 @@ package snapshot import ( "fmt" - "math/big" "os" "testing" "time" @@ -33,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) @@ -58,9 +58,9 @@ func testGeneration(t *testing.T, scheme string) { var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) @@ -97,16 +97,16 @@ func testGenerateExistentState(t *testing.T, scheme string) { var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) stRoot = helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) root, snap := helper.CommitAndGenerate() @@ -259,28 +259,28 @@ func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) { helper := newHelper(scheme) // Account one, empty root but non-empty database - helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) // Account two, non empty root but empty database stRoot := helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-2", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Miss slots { // Account three, non empty root but misses slots in the beginning helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-3", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"}) // Account four, non empty root but misses slots in the middle helper.makeStorageTrie(hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-4", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-4", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"}) // Account five, non empty root but misses slots in the end helper.makeStorageTrie(hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-5", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-5", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"}) } @@ -288,22 +288,22 @@ func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) { { // Account six, non empty root but wrong slots in the beginning helper.makeStorageTrie(hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-6", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-6", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"}) // Account seven, non empty root but wrong slots in the middle helper.makeStorageTrie(hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-7", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-7", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"}) // Account eight, non empty root but wrong slots in the end helper.makeStorageTrie(hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-8", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-8", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"}) // Account 9, non empty root but rotated slots helper.makeStorageTrie(hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-9", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-9", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"}) } @@ -311,17 +311,17 @@ func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) { { // Account 10, non empty root but extra slots in the beginning helper.makeStorageTrie(hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-10", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-10", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"}) // Account 11, non empty root but extra slots in the middle helper.makeStorageTrie(hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-11", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-11", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"}) // Account 12, non empty root but extra slots in the end helper.makeStorageTrie(hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-12", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-12", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"}) } @@ -366,25 +366,25 @@ func testGenerateExistentStateWithWrongAccounts(t *testing.T, scheme string) { // Missing accounts, only in the trie { - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Beginning - helper.addTrieAccount("acc-4", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Middle - helper.addTrieAccount("acc-6", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // End + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Beginning + helper.addTrieAccount("acc-4", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Middle + helper.addTrieAccount("acc-6", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // End } // Wrong accounts { - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-2", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")}) + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")}) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-3", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) } // Extra accounts, only in the snap { - helper.addSnapAccount("acc-0", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // before the beginning - helper.addSnapAccount("acc-5", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: common.Hex2Bytes("0x1234")}) // Middle - helper.addSnapAccount("acc-7", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // after the end + helper.addSnapAccount("acc-0", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // before the beginning + helper.addSnapAccount("acc-5", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: common.Hex2Bytes("0x1234")}) // Middle + helper.addSnapAccount("acc-7", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // after the end } root, snap := helper.CommitAndGenerate() @@ -418,9 +418,9 @@ func testGenerateCorruptAccountTrie(t *testing.T, scheme string) { // without any storage slots to keep the test smaller. helper := newHelper(scheme) - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 @@ -462,11 +462,11 @@ func testGenerateMissingStorageTrie(t *testing.T, scheme string) { acc3 = hashData([]byte("acc-3")) helper = newHelper(scheme) ) - stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 stRoot = helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 root := helper.Commit() @@ -502,11 +502,11 @@ func testGenerateCorruptStorageTrie(t *testing.T, scheme string) { // two of which also has the same 3-slot storage trie attached. helper := newHelper(scheme) - stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 stRoot = helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 root := helper.Commit() @@ -546,7 +546,7 @@ func testGenerateWithExtraAccounts(t *testing.T, scheme string) { []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, true, ) - acc := &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.MustUpdate([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e @@ -566,7 +566,7 @@ func testGenerateWithExtraAccounts(t *testing.T, scheme string) { []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, true, ) - acc := &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte("acc-2")) rawdb.WriteAccountSnapshot(helper.diskdb, key, val) @@ -622,7 +622,7 @@ func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) { []string{"val-1", "val-2", "val-3"}, true, ) - acc := &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.MustUpdate([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e @@ -636,7 +636,7 @@ func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) { { // 100 accounts exist only in snapshot for i := 0; i < 1000; i++ { - acc := &types.StateAccount{Balance: big.NewInt(int64(i)), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(uint64(i)), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte(fmt.Sprintf("acc-%d", i))) rawdb.WriteAccountSnapshot(helper.diskdb, key, val) @@ -678,7 +678,7 @@ func testGenerateWithExtraBeforeAndAfter(t *testing.T, scheme string) { } helper := newHelper(scheme) { - acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.MustUpdate(common.HexToHash("0x03").Bytes(), val) helper.accTrie.MustUpdate(common.HexToHash("0x07").Bytes(), val) @@ -720,7 +720,7 @@ func testGenerateWithMalformedSnapdata(t *testing.T, scheme string) { } helper := newHelper(scheme) { - acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.MustUpdate(common.HexToHash("0x03").Bytes(), val) @@ -764,7 +764,7 @@ func testGenerateFromEmptySnap(t *testing.T, scheme string) { for i := 0; i < 400; i++ { stRoot := helper.makeStorageTrie(hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount(fmt.Sprintf("acc-%d", i), - &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) } root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4 @@ -806,7 +806,7 @@ func testGenerateWithIncompleteStorage(t *testing.T, scheme string) { for i := 0; i < 8; i++ { accKey := fmt.Sprintf("acc-%d", i) stRoot := helper.makeStorageTrie(hashData([]byte(accKey)), stKeys, stVals, true) - helper.addAccount(accKey, &types.StateAccount{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount(accKey, &types.StateAccount{Balance: uint256.NewInt(uint64(i)), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) var moddedKeys []string var moddedVals []string for ii := 0; ii < 8; ii++ { @@ -903,11 +903,11 @@ func testGenerateCompleteSnapshotWithDanglingStorage(t *testing.T, scheme string var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addAccount("acc-2", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-3", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) @@ -943,11 +943,11 @@ func testGenerateBrokenSnapshotWithDanglingStorage(t *testing.T, scheme string) var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) populateDangling(helper.diskdb) diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index b66799757e..a9ab3eaea3 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -20,7 +20,6 @@ import ( crand "crypto/rand" "encoding/binary" "fmt" - "math/big" "math/rand" "testing" "time" @@ -30,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" + "github.com/holiman/uint256" ) // randomHash generates a random blob of data and returns it as a hash. @@ -44,7 +44,7 @@ func randomHash() common.Hash { // randomAccount generates a random account and returns it RLP encoded. func randomAccount() []byte { a := &types.StateAccount{ - Balance: big.NewInt(rand.Int63()), + Balance: uint256.NewInt(rand.Uint64()), Nonce: rand.Uint64(), Root: randomHash(), CodeHash: types.EmptyCodeHash[:], diff --git a/core/state/state_object.go b/core/state/state_object.go index 9383b98e44..fc26af68db 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -20,7 +20,6 @@ import ( "bytes" "fmt" "io" - "math/big" "time" "github.com/ethereum/go-ethereum/common" @@ -29,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/holiman/uint256" ) type Code []byte @@ -93,7 +93,7 @@ type stateObject struct { // empty returns whether the account is considered empty. func (s *stateObject) empty() bool { - return s.data.Nonce == 0 && s.data.Balance.Sign() == 0 && bytes.Equal(s.data.CodeHash, types.EmptyCodeHash.Bytes()) + return s.data.Nonce == 0 && s.data.Balance.IsZero() && bytes.Equal(s.data.CodeHash, types.EmptyCodeHash.Bytes()) } // newObject creates a state object. @@ -405,36 +405,36 @@ func (s *stateObject) commit() (*trienode.NodeSet, error) { // AddBalance adds amount to s's balance. // It is used to add funds to the destination account of a transfer. -func (s *stateObject) AddBalance(amount *big.Int) { +func (s *stateObject) AddBalance(amount *uint256.Int) { // EIP161: We must check emptiness for the objects such that the account // clearing (0,0,0 objects) can take effect. - if amount.Sign() == 0 { + if amount.IsZero() { if s.empty() { s.touch() } return } - s.SetBalance(new(big.Int).Add(s.Balance(), amount)) + s.SetBalance(new(uint256.Int).Add(s.Balance(), amount)) } // SubBalance removes amount from s's balance. // It is used to remove funds from the origin account of a transfer. -func (s *stateObject) SubBalance(amount *big.Int) { - if amount.Sign() == 0 { +func (s *stateObject) SubBalance(amount *uint256.Int) { + if amount.IsZero() { return } - s.SetBalance(new(big.Int).Sub(s.Balance(), amount)) + s.SetBalance(new(uint256.Int).Sub(s.Balance(), amount)) } -func (s *stateObject) SetBalance(amount *big.Int) { +func (s *stateObject) SetBalance(amount *uint256.Int) { s.db.journal.append(balanceChange{ account: &s.address, - prev: new(big.Int).Set(s.data.Balance), + prev: new(uint256.Int).Set(s.data.Balance), }) s.setBalance(amount) } -func (s *stateObject) setBalance(amount *big.Int) { +func (s *stateObject) setBalance(amount *uint256.Int) { s.data.Balance = amount } @@ -533,7 +533,7 @@ func (s *stateObject) CodeHash() []byte { return s.data.CodeHash } -func (s *stateObject) Balance() *big.Int { +func (s *stateObject) Balance() *uint256.Int { return s.data.Balance } diff --git a/core/state/state_test.go b/core/state/state_test.go index 029d03c22b..df7ebd2456 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -19,7 +19,6 @@ package state import ( "bytes" "encoding/json" - "math/big" "testing" "github.com/ethereum/go-ethereum/common" @@ -28,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) type stateEnv struct { @@ -49,11 +49,11 @@ func TestDump(t *testing.T) { // generate a few entries obj1 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01})) - obj1.AddBalance(big.NewInt(22)) + obj1.AddBalance(uint256.NewInt(22)) obj2 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01, 0x02})) obj2.SetCode(crypto.Keccak256Hash([]byte{3, 3, 3, 3, 3, 3, 3}), []byte{3, 3, 3, 3, 3, 3, 3}) obj3 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x02})) - obj3.SetBalance(big.NewInt(44)) + obj3.SetBalance(uint256.NewInt(44)) // write some of them to the trie s.state.updateStateObject(obj1) @@ -106,13 +106,13 @@ func TestIterativeDump(t *testing.T) { // generate a few entries obj1 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01})) - obj1.AddBalance(big.NewInt(22)) + obj1.AddBalance(uint256.NewInt(22)) obj2 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01, 0x02})) obj2.SetCode(crypto.Keccak256Hash([]byte{3, 3, 3, 3, 3, 3, 3}), []byte{3, 3, 3, 3, 3, 3, 3}) obj3 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x02})) - obj3.SetBalance(big.NewInt(44)) + obj3.SetBalance(uint256.NewInt(44)) obj4 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x00})) - obj4.AddBalance(big.NewInt(1337)) + obj4.AddBalance(uint256.NewInt(1337)) // write some of them to the trie s.state.updateStateObject(obj1) @@ -208,7 +208,7 @@ func TestSnapshot2(t *testing.T) { // db, trie are already non-empty values so0 := state.getStateObject(stateobjaddr0) - so0.SetBalance(big.NewInt(42)) + so0.SetBalance(uint256.NewInt(42)) so0.SetNonce(43) so0.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e'}), []byte{'c', 'a', 'f', 'e'}) so0.selfDestructed = false @@ -220,7 +220,7 @@ func TestSnapshot2(t *testing.T) { // and one with deleted == true so1 := state.getStateObject(stateobjaddr1) - so1.SetBalance(big.NewInt(52)) + so1.SetBalance(uint256.NewInt(52)) so1.SetNonce(53) so1.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e', '2'}), []byte{'c', 'a', 'f', 'e', '2'}) so1.selfDestructed = true diff --git a/core/state/statedb.go b/core/state/statedb.go index 3804c6603b..a4b8cf93e2 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -19,7 +19,6 @@ package state import ( "fmt" - "math/big" "sort" "time" @@ -34,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/holiman/uint256" ) const ( @@ -280,12 +280,12 @@ func (s *StateDB) Empty(addr common.Address) bool { } // GetBalance retrieves the balance from the given address or 0 if object not found -func (s *StateDB) GetBalance(addr common.Address) *big.Int { +func (s *StateDB) GetBalance(addr common.Address) *uint256.Int { stateObject := s.getStateObject(addr) if stateObject != nil { return stateObject.Balance() } - return common.Big0 + return common.U2560 } // GetNonce retrieves the nonce from the given address or 0 if object not found @@ -373,7 +373,7 @@ func (s *StateDB) HasSelfDestructed(addr common.Address) bool { */ // AddBalance adds amount to the account associated with addr. -func (s *StateDB) AddBalance(addr common.Address, amount *big.Int) { +func (s *StateDB) AddBalance(addr common.Address, amount *uint256.Int) { stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.AddBalance(amount) @@ -381,14 +381,14 @@ func (s *StateDB) AddBalance(addr common.Address, amount *big.Int) { } // SubBalance subtracts amount from the account associated with addr. -func (s *StateDB) SubBalance(addr common.Address, amount *big.Int) { +func (s *StateDB) SubBalance(addr common.Address, amount *uint256.Int) { stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.SubBalance(amount) } } -func (s *StateDB) SetBalance(addr common.Address, amount *big.Int) { +func (s *StateDB) SetBalance(addr common.Address, amount *uint256.Int) { stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.SetBalance(amount) @@ -450,10 +450,10 @@ func (s *StateDB) SelfDestruct(addr common.Address) { s.journal.append(selfDestructChange{ account: &addr, prev: stateObject.selfDestructed, - prevbalance: new(big.Int).Set(stateObject.Balance()), + prevbalance: new(uint256.Int).Set(stateObject.Balance()), }) stateObject.markSelfdestructed() - stateObject.data.Balance = new(big.Int) + stateObject.data.Balance = new(uint256.Int) } func (s *StateDB) Selfdestruct6780(addr common.Address) { diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index c4704257c7..620dee16d9 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "math" - "math/big" "math/rand" "reflect" "strings" @@ -38,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/holiman/uint256" ) // A stateTest checks that the state changes are correctly captured. Instances @@ -60,7 +60,7 @@ func newStateTestAction(addr common.Address, r *rand.Rand, index int) testAction { name: "SetBalance", fn: func(a testAction, s *StateDB) { - s.SetBalance(addr, big.NewInt(a.args[0])) + s.SetBalance(addr, uint256.NewInt(uint64(a.args[0]))) }, args: make([]int64, 1), }, diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 322299a468..889fbf9973 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "math" - "math/big" "math/rand" "reflect" "strings" @@ -56,7 +55,7 @@ func TestUpdateLeaks(t *testing.T) { // Update it with some accounts for i := byte(0); i < 255; i++ { addr := common.BytesToAddress([]byte{i}) - state.AddBalance(addr, big.NewInt(int64(11*i))) + state.AddBalance(addr, uint256.NewInt(uint64(11*i))) state.SetNonce(addr, uint64(42*i)) if i%2 == 0 { state.SetState(addr, common.BytesToHash([]byte{i, i, i}), common.BytesToHash([]byte{i, i, i, i})) @@ -91,7 +90,7 @@ func TestIntermediateLeaks(t *testing.T) { finalState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(finalDb, finalNdb), nil) modify := func(state *StateDB, addr common.Address, i, tweak byte) { - state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak))) + state.SetBalance(addr, uint256.NewInt(uint64(11*i)+uint64(tweak))) state.SetNonce(addr, uint64(42*i+tweak)) if i%2 == 0 { state.SetState(addr, common.Hash{i, i, i, 0}, common.Hash{}) @@ -167,7 +166,7 @@ func TestCopy(t *testing.T) { for i := byte(0); i < 255; i++ { obj := orig.getOrNewStateObject(common.BytesToAddress([]byte{i})) - obj.AddBalance(big.NewInt(int64(i))) + obj.AddBalance(uint256.NewInt(uint64(i))) orig.updateStateObject(obj) } orig.Finalise(false) @@ -184,9 +183,9 @@ func TestCopy(t *testing.T) { copyObj := copy.getOrNewStateObject(common.BytesToAddress([]byte{i})) ccopyObj := ccopy.getOrNewStateObject(common.BytesToAddress([]byte{i})) - origObj.AddBalance(big.NewInt(2 * int64(i))) - copyObj.AddBalance(big.NewInt(3 * int64(i))) - ccopyObj.AddBalance(big.NewInt(4 * int64(i))) + origObj.AddBalance(uint256.NewInt(2 * uint64(i))) + copyObj.AddBalance(uint256.NewInt(3 * uint64(i))) + ccopyObj.AddBalance(uint256.NewInt(4 * uint64(i))) orig.updateStateObject(origObj) copy.updateStateObject(copyObj) @@ -212,13 +211,13 @@ func TestCopy(t *testing.T) { copyObj := copy.getOrNewStateObject(common.BytesToAddress([]byte{i})) ccopyObj := ccopy.getOrNewStateObject(common.BytesToAddress([]byte{i})) - if want := big.NewInt(3 * int64(i)); origObj.Balance().Cmp(want) != 0 { + if want := uint256.NewInt(3 * uint64(i)); origObj.Balance().Cmp(want) != 0 { t.Errorf("orig obj %d: balance mismatch: have %v, want %v", i, origObj.Balance(), want) } - if want := big.NewInt(4 * int64(i)); copyObj.Balance().Cmp(want) != 0 { + if want := uint256.NewInt(4 * uint64(i)); copyObj.Balance().Cmp(want) != 0 { t.Errorf("copy obj %d: balance mismatch: have %v, want %v", i, copyObj.Balance(), want) } - if want := big.NewInt(5 * int64(i)); ccopyObj.Balance().Cmp(want) != 0 { + if want := uint256.NewInt(5 * uint64(i)); ccopyObj.Balance().Cmp(want) != 0 { t.Errorf("copy obj %d: balance mismatch: have %v, want %v", i, ccopyObj.Balance(), want) } } @@ -266,14 +265,14 @@ func newTestAction(addr common.Address, r *rand.Rand) testAction { { name: "SetBalance", fn: func(a testAction, s *StateDB) { - s.SetBalance(addr, big.NewInt(a.args[0])) + s.SetBalance(addr, uint256.NewInt(uint64(a.args[0]))) }, args: make([]int64, 1), }, { name: "AddBalance", fn: func(a testAction, s *StateDB) { - s.AddBalance(addr, big.NewInt(a.args[0])) + s.AddBalance(addr, uint256.NewInt(uint64(a.args[0]))) }, args: make([]int64, 1), }, @@ -536,7 +535,7 @@ func TestTouchDelete(t *testing.T) { s.state, _ = New(root, s.state.db, s.state.snaps) snapshot := s.state.Snapshot() - s.state.AddBalance(common.Address{}, new(big.Int)) + s.state.AddBalance(common.Address{}, new(uint256.Int)) if len(s.state.journal.dirties) != 1 { t.Fatal("expected one dirty state object") @@ -552,7 +551,7 @@ func TestTouchDelete(t *testing.T) { func TestCopyOfCopy(t *testing.T) { state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) addr := common.HexToAddress("aaaa") - state.SetBalance(addr, big.NewInt(42)) + state.SetBalance(addr, uint256.NewInt(42)) if got := state.Copy().GetBalance(addr).Uint64(); got != 42 { t.Fatalf("1st copy fail, expected 42, got %v", got) @@ -575,11 +574,11 @@ func TestCopyCommitCopy(t *testing.T) { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, big.NewInt(42)) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie - if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) } if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -593,7 +592,7 @@ func TestCopyCommitCopy(t *testing.T) { } // Copy the non-committed state database and check pre/post commit balance copyOne := state.Copy() - if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := copyOne.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("first copy pre-commit balance mismatch: have %v, want %v", balance, 42) } if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -607,7 +606,7 @@ func TestCopyCommitCopy(t *testing.T) { } // Copy the copy and check the balance once more copyTwo := copyOne.Copy() - if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := copyTwo.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("second copy balance mismatch: have %v, want %v", balance, 42) } if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -622,7 +621,7 @@ func TestCopyCommitCopy(t *testing.T) { // Commit state, ensure states can be loaded from disk root, _ := state.Commit(0, false) state, _ = New(root, tdb, nil) - if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("state post-commit balance mismatch: have %v, want %v", balance, 42) } if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -648,11 +647,11 @@ func TestCopyCopyCommitCopy(t *testing.T) { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, big.NewInt(42)) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie - if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) } if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -666,7 +665,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { } // Copy the non-committed state database and check pre/post commit balance copyOne := state.Copy() - if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := copyOne.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("first copy balance mismatch: have %v, want %v", balance, 42) } if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -680,7 +679,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { } // Copy the copy and check the balance once more copyTwo := copyOne.Copy() - if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := copyTwo.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("second copy pre-commit balance mismatch: have %v, want %v", balance, 42) } if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -694,7 +693,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { } // Copy the copy-copy and check the balance once more copyThree := copyTwo.Copy() - if balance := copyThree.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := copyThree.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("third copy balance mismatch: have %v, want %v", balance, 42) } if code := copyThree.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -717,11 +716,11 @@ func TestCommitCopy(t *testing.T) { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, big.NewInt(42)) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie - if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) } if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -736,7 +735,7 @@ func TestCommitCopy(t *testing.T) { // Copy the committed state database, the copied one is not functional. state.Commit(0, true) copied := state.Copy() - if balance := copied.GetBalance(addr); balance.Cmp(big.NewInt(0)) != 0 { + if balance := copied.GetBalance(addr); balance.Cmp(uint256.NewInt(0)) != 0 { t.Fatalf("unexpected balance: have %v", balance) } if code := copied.GetCode(addr); code != nil { @@ -766,7 +765,7 @@ func TestDeleteCreateRevert(t *testing.T) { state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) addr := common.BytesToAddress([]byte("so")) - state.SetBalance(addr, big.NewInt(1)) + state.SetBalance(addr, uint256.NewInt(1)) root, _ := state.Commit(0, false) state, _ = New(root, state.db, state.snaps) @@ -776,7 +775,7 @@ func TestDeleteCreateRevert(t *testing.T) { state.Finalise(true) id := state.Snapshot() - state.SetBalance(addr, big.NewInt(2)) + state.SetBalance(addr, uint256.NewInt(2)) state.RevertToSnapshot(id) // Commit the entire state and make sure we don't crash and have the correct state @@ -818,10 +817,10 @@ func testMissingTrieNodes(t *testing.T, scheme string) { state, _ := New(types.EmptyRootHash, db, nil) addr := common.BytesToAddress([]byte("so")) { - state.SetBalance(addr, big.NewInt(1)) + state.SetBalance(addr, uint256.NewInt(1)) state.SetCode(addr, []byte{1, 2, 3}) a2 := common.BytesToAddress([]byte("another")) - state.SetBalance(a2, big.NewInt(100)) + state.SetBalance(a2, uint256.NewInt(100)) state.SetCode(a2, []byte{1, 2, 4}) root, _ = state.Commit(0, false) t.Logf("root: %x", root) @@ -846,7 +845,7 @@ func testMissingTrieNodes(t *testing.T, scheme string) { t.Errorf("expected %d, got %d", exp, got) } // Modify the state - state.SetBalance(addr, big.NewInt(2)) + state.SetBalance(addr, uint256.NewInt(2)) root, err := state.Commit(0, false) if err == nil { t.Fatalf("expected error, got root :%x", root) @@ -1114,13 +1113,13 @@ func TestResetObject(t *testing.T) { slotB = common.HexToHash("0x2") ) // Initialize account with balance and storage in first transaction. - state.SetBalance(addr, big.NewInt(1)) + state.SetBalance(addr, uint256.NewInt(1)) state.SetState(addr, slotA, common.BytesToHash([]byte{0x1})) state.IntermediateRoot(true) // Reset account and mutate balance and storages state.CreateAccount(addr) - state.SetBalance(addr, big.NewInt(2)) + state.SetBalance(addr, uint256.NewInt(2)) state.SetState(addr, slotB, common.BytesToHash([]byte{0x2})) root, _ := state.Commit(0, true) @@ -1146,7 +1145,7 @@ func TestDeleteStorage(t *testing.T) { addr = common.HexToAddress("0x1") ) // Initialize account and populate storage - state.SetBalance(addr, big.NewInt(1)) + state.SetBalance(addr, uint256.NewInt(1)) state.CreateAccount(addr) for i := 0; i < 1000; i++ { slot := common.Hash(uint256.NewInt(uint64(i)).Bytes32()) diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 21c65b9104..c0a397c3af 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -18,7 +18,6 @@ package state import ( "bytes" - "math/big" "testing" "github.com/ethereum/go-ethereum/common" @@ -30,12 +29,13 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/holiman/uint256" ) // testAccount is the data associated with an account used by the state tests. type testAccount struct { address common.Address - balance *big.Int + balance *uint256.Int nonce uint64 code []byte } @@ -60,8 +60,8 @@ func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, com obj := state.getOrNewStateObject(common.BytesToAddress([]byte{i})) acc := &testAccount{address: common.BytesToAddress([]byte{i})} - obj.AddBalance(big.NewInt(int64(11 * i))) - acc.balance = big.NewInt(int64(11 * i)) + obj.AddBalance(uint256.NewInt(uint64(11 * i))) + acc.balance = uint256.NewInt(uint64(11 * i)) obj.SetNonce(uint64(42 * i)) acc.nonce = uint64(42 * i) @@ -237,7 +237,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, s id := trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root) stTrie, err := trie.New(id, ndb) if err != nil { - t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err) + t.Fatalf("failed to retrieve storage trie for path %x: %v", node.syncPath[1], err) } data, _, err := stTrie.GetNode(node.syncPath[1]) if err != nil { diff --git a/core/state/trie_prefetcher_test.go b/core/state/trie_prefetcher_test.go index b190567e92..711ec83250 100644 --- a/core/state/trie_prefetcher_test.go +++ b/core/state/trie_prefetcher_test.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/holiman/uint256" ) func filledStateDB() *StateDB { @@ -34,9 +35,9 @@ func filledStateDB() *StateDB { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, big.NewInt(42)) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie for i := 0; i < 100; i++ { sk := common.BigToHash(big.NewInt(int64(i))) state.SetState(addr, sk, sk) // Change the storage trie diff --git a/core/state_processor.go b/core/state_processor.go index 9a4333f723..9e32ab4e56 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -186,6 +186,6 @@ func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb *stat } vmenv.Reset(NewEVMTxContext(msg), statedb) statedb.AddAddressToAccessList(params.BeaconRootsStorageAddress) - _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.Big0) + _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) statedb.Finalise(true) } diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 5ff9353bd9..2f5f0dc02b 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -232,7 +232,7 @@ func TestStateProcessorErrors(t *testing.T) { txs: []*types.Transaction{ mkDynamicTx(0, common.Address{}, params.TxGas, bigNumber, bigNumber), }, - want: "could not apply tx 0 [0xd82a0c2519acfeac9a948258c47e784acd20651d9d80f9a1c67b4137651c3a24]: insufficient funds for gas * price + value: address 0x71562b71999873DB5b286dF957af199Ec94617F7 have 1000000000000000000 want 2431633873983640103894990685182446064918669677978451844828609264166175722438635000", + want: "could not apply tx 0 [0xd82a0c2519acfeac9a948258c47e784acd20651d9d80f9a1c67b4137651c3a24]: insufficient funds for gas * price + value: address 0x71562b71999873DB5b286dF957af199Ec94617F7 required balance exceeds 256 bits", }, { // ErrMaxInitCodeSizeExceeded txs: []*types.Transaction{ diff --git a/core/state_transition.go b/core/state_transition.go index df2faa19a9..2be54480f3 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) // ExecutionResult includes all output after executing given evm @@ -252,7 +253,11 @@ func (st *StateTransition) buyGas() error { mgval.Add(mgval, blobFee) } } - if have, want := st.state.GetBalance(st.msg.From), balanceCheck; have.Cmp(want) < 0 { + balanceCheckU256, overflow := uint256.FromBig(balanceCheck) + if overflow { + return fmt.Errorf("%w: address %v required balance exceeds 256 bits", ErrInsufficientFunds, st.msg.From.Hex()) + } + if have, want := st.state.GetBalance(st.msg.From), balanceCheckU256; have.Cmp(want) < 0 { return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) } if err := st.gp.SubGas(st.msg.GasLimit); err != nil { @@ -261,7 +266,8 @@ func (st *StateTransition) buyGas() error { st.gasRemaining += st.msg.GasLimit st.initialGas = st.msg.GasLimit - st.state.SubBalance(st.msg.From, mgval) + mgvalU256, _ := uint256.FromBig(mgval) + st.state.SubBalance(st.msg.From, mgvalU256) return nil } @@ -399,7 +405,11 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { st.gasRemaining -= gas // Check clause 6 - if msg.Value.Sign() > 0 && !st.evm.Context.CanTransfer(st.state, msg.From, msg.Value) { + value, overflow := uint256.FromBig(msg.Value) + if overflow { + return nil, fmt.Errorf("%w: address %v", ErrInsufficientFundsForTransfer, msg.From.Hex()) + } + if !value.IsZero() && !st.evm.Context.CanTransfer(st.state, msg.From, value) { return nil, fmt.Errorf("%w: address %v", ErrInsufficientFundsForTransfer, msg.From.Hex()) } @@ -418,11 +428,11 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { vmerr error // vm errors do not effect consensus and are therefore not assigned to err ) if contractCreation { - ret, _, st.gasRemaining, vmerr = st.evm.Create(sender, msg.Data, st.gasRemaining, msg.Value) + ret, _, st.gasRemaining, vmerr = st.evm.Create(sender, msg.Data, st.gasRemaining, value) } else { // Increment the nonce for the next transaction st.state.SetNonce(msg.From, st.state.GetNonce(sender.Address())+1) - ret, st.gasRemaining, vmerr = st.evm.Call(sender, st.to(), msg.Data, st.gasRemaining, msg.Value) + ret, st.gasRemaining, vmerr = st.evm.Call(sender, st.to(), msg.Data, st.gasRemaining, value) } var gasRefund uint64 @@ -437,14 +447,15 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { if rules.IsLondon { effectiveTip = cmath.BigMin(msg.GasTipCap, new(big.Int).Sub(msg.GasFeeCap, st.evm.Context.BaseFee)) } + effectiveTipU256, _ := uint256.FromBig(effectiveTip) if st.evm.Config.NoBaseFee && msg.GasFeeCap.Sign() == 0 && msg.GasTipCap.Sign() == 0 { // Skip fee payment when NoBaseFee is set and the fee fields // are 0. This avoids a negative effectiveTip being applied to // the coinbase when simulating calls. } else { - fee := new(big.Int).SetUint64(st.gasUsed()) - fee.Mul(fee, effectiveTip) + fee := new(uint256.Int).SetUint64(st.gasUsed()) + fee.Mul(fee, effectiveTipU256) st.state.AddBalance(st.evm.Context.Coinbase, fee) } @@ -465,7 +476,8 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { st.gasRemaining += refund // Return ETH for remaining gas, exchanged at the original rate. - remaining := new(big.Int).Mul(new(big.Int).SetUint64(st.gasRemaining), st.msg.GasPrice) + remaining := uint256.NewInt(st.gasRemaining) + remaining = remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice)) st.state.AddBalance(st.msg.From, remaining) // Also return remaining gas to the block gas counter so it is diff --git a/core/txindexer.go b/core/txindexer.go new file mode 100644 index 0000000000..70fe5f3322 --- /dev/null +++ b/core/txindexer.go @@ -0,0 +1,219 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package core + +import ( + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +// TxIndexProgress is the struct describing the progress for transaction indexing. +type TxIndexProgress struct { + Indexed uint64 // number of blocks whose transactions are indexed + Remaining uint64 // number of blocks whose transactions are not indexed yet +} + +// Done returns an indicator if the transaction indexing is finished. +func (progress TxIndexProgress) Done() bool { + return progress.Remaining == 0 +} + +// txIndexer is the module responsible for maintaining transaction indexes +// according to the configured indexing range by users. +type txIndexer struct { + // limit is the maximum number of blocks from head whose tx indexes + // are reserved: + // * 0: means the entire chain should be indexed + // * N: means the latest N blocks [HEAD-N+1, HEAD] should be indexed + // and all others shouldn't. + limit uint64 + db ethdb.Database + progress chan chan TxIndexProgress + term chan chan struct{} + closed chan struct{} +} + +// newTxIndexer initializes the transaction indexer. +func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer { + indexer := &txIndexer{ + limit: limit, + db: chain.db, + progress: make(chan chan TxIndexProgress), + term: make(chan chan struct{}), + closed: make(chan struct{}), + } + go indexer.loop(chain) + + var msg string + if limit == 0 { + msg = "entire chain" + } else { + msg = fmt.Sprintf("last %d blocks", limit) + } + log.Info("Initialized transaction indexer", "range", msg) + + return indexer +} + +// run executes the scheduled indexing/unindexing task in a separate thread. +// If the stop channel is closed, the task should be terminated as soon as +// possible, the done channel will be closed once the task is finished. +func (indexer *txIndexer) run(tail *uint64, head uint64, stop chan struct{}, done chan struct{}) { + defer func() { close(done) }() + + // Short circuit if chain is empty and nothing to index. + if head == 0 { + return + } + // The tail flag is not existent, it means the node is just initialized + // and all blocks in the chain (part of them may from ancient store) are + // not indexed yet, index the chain according to the configured limit. + if tail == nil { + from := uint64(0) + if indexer.limit != 0 && head >= indexer.limit { + from = head - indexer.limit + 1 + } + rawdb.IndexTransactions(indexer.db, from, head+1, stop, true) + return + } + // The tail flag is existent (which means indexes in [tail, head] should be + // present), while the whole chain are requested for indexing. + if indexer.limit == 0 || head < indexer.limit { + if *tail > 0 { + // It can happen when chain is rewound to a historical point which + // is even lower than the indexes tail, recap the indexing target + // to new head to avoid reading non-existent block bodies. + end := *tail + if end > head+1 { + end = head + 1 + } + rawdb.IndexTransactions(indexer.db, 0, end, stop, true) + } + return + } + // The tail flag is existent, adjust the index range according to configured + // limit and the latest chain head. + if head-indexer.limit+1 < *tail { + // Reindex a part of missing indices and rewind index tail to HEAD-limit + rawdb.IndexTransactions(indexer.db, head-indexer.limit+1, *tail, stop, true) + } else { + // Unindex a part of stale indices and forward index tail to HEAD-limit + rawdb.UnindexTransactions(indexer.db, *tail, head-indexer.limit+1, stop, false) + } +} + +// loop is the scheduler of the indexer, assigning indexing/unindexing tasks depending +// on the received chain event. +func (indexer *txIndexer) loop(chain *BlockChain) { + defer close(indexer.closed) + + // Listening to chain events and manipulate the transaction indexes. + var ( + stop chan struct{} // Non-nil if background routine is active. + done chan struct{} // Non-nil if background routine is active. + lastHead uint64 // The latest announced chain head (whose tx indexes are assumed created) + lastTail = rawdb.ReadTxIndexTail(indexer.db) // The oldest indexed block, nil means nothing indexed + + headCh = make(chan ChainHeadEvent) + sub = chain.SubscribeChainHeadEvent(headCh) + ) + defer sub.Unsubscribe() + + // Launch the initial processing if chain is not empty (head != genesis). + // This step is useful in these scenarios that chain has no progress. + if head := rawdb.ReadHeadBlock(indexer.db); head != nil && head.Number().Uint64() != 0 { + stop = make(chan struct{}) + done = make(chan struct{}) + lastHead = head.Number().Uint64() + go indexer.run(rawdb.ReadTxIndexTail(indexer.db), head.NumberU64(), stop, done) + } + for { + select { + case head := <-headCh: + if done == nil { + stop = make(chan struct{}) + done = make(chan struct{}) + go indexer.run(rawdb.ReadTxIndexTail(indexer.db), head.Block.NumberU64(), stop, done) + } + lastHead = head.Block.NumberU64() + case <-done: + stop = nil + done = nil + lastTail = rawdb.ReadTxIndexTail(indexer.db) + case ch := <-indexer.progress: + ch <- indexer.report(lastHead, lastTail) + case ch := <-indexer.term: + if stop != nil { + close(stop) + } + if done != nil { + log.Info("Waiting background transaction indexer to exit") + <-done + } + close(ch) + return + } + } +} + +// report returns the tx indexing progress. +func (indexer *txIndexer) report(head uint64, tail *uint64) TxIndexProgress { + total := indexer.limit + if indexer.limit == 0 || total > head { + total = head + 1 // genesis included + } + var indexed uint64 + if tail != nil { + indexed = head - *tail + 1 + } + // The value of indexed might be larger than total if some blocks need + // to be unindexed, avoiding a negative remaining. + var remaining uint64 + if indexed < total { + remaining = total - indexed + } + return TxIndexProgress{ + Indexed: indexed, + Remaining: remaining, + } +} + +// txIndexProgress retrieves the tx indexing progress, or an error if the +// background tx indexer is already stopped. +func (indexer *txIndexer) txIndexProgress() (TxIndexProgress, error) { + ch := make(chan TxIndexProgress, 1) + select { + case indexer.progress <- ch: + return <-ch, nil + case <-indexer.closed: + return TxIndexProgress{}, errors.New("indexer is closed") + } +} + +// close shutdown the indexer. Safe to be called for multiple times. +func (indexer *txIndexer) close() { + ch := make(chan struct{}) + select { + case indexer.term <- ch: + <-ch + case <-indexer.closed: + } +} diff --git a/core/txindexer_test.go b/core/txindexer_test.go new file mode 100644 index 0000000000..b2c2dcec2b --- /dev/null +++ b/core/txindexer_test.go @@ -0,0 +1,243 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package core + +import ( + "math/big" + "os" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" +) + +// TestTxIndexer tests the functionalities for managing transaction indexes. +func TestTxIndexer(t *testing.T) { + var ( + testBankKey, _ = crypto.GenerateKey() + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) + testBankFunds = big.NewInt(1000000000000000000) + + gspec = &Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + engine = ethash.NewFaker() + nonce = uint64(0) + chainHead = uint64(128) + ) + _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, int(chainHead), func(i int, gen *BlockGen) { + tx, _ := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("0xdeadbeef"), big.NewInt(1000), params.TxGas, big.NewInt(10*params.InitialBaseFee), nil), types.HomesteadSigner{}, testBankKey) + gen.AddTx(tx) + nonce += 1 + }) + + // verifyIndexes checks if the transaction indexes are present or not + // of the specified block. + verifyIndexes := func(db ethdb.Database, number uint64, exist bool) { + if number == 0 { + return + } + block := blocks[number-1] + for _, tx := range block.Transactions() { + lookup := rawdb.ReadTxLookupEntry(db, tx.Hash()) + if exist && lookup == nil { + t.Fatalf("missing %d %x", number, tx.Hash().Hex()) + } + if !exist && lookup != nil { + t.Fatalf("unexpected %d %x", number, tx.Hash().Hex()) + } + } + } + verify := func(db ethdb.Database, expTail uint64, indexer *txIndexer) { + tail := rawdb.ReadTxIndexTail(db) + if tail == nil { + t.Fatal("Failed to write tx index tail") + } + if *tail != expTail { + t.Fatalf("Unexpected tx index tail, want %v, got %d", expTail, *tail) + } + if *tail != 0 { + for number := uint64(0); number < *tail; number += 1 { + verifyIndexes(db, number, false) + } + } + for number := *tail; number <= chainHead; number += 1 { + verifyIndexes(db, number, true) + } + progress := indexer.report(chainHead, tail) + if !progress.Done() { + t.Fatalf("Expect fully indexed") + } + } + + var cases = []struct { + limitA uint64 + tailA uint64 + limitB uint64 + tailB uint64 + limitC uint64 + tailC uint64 + }{ + { + // LimitA: 0 + // TailA: 0 + // + // all blocks are indexed + limitA: 0, + tailA: 0, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 64 + // TailA: 65 + // + // block [65, 128] are indexed + limitA: 64, + tailA: 65, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 127 + // TailA: 2 + // + // block [2, 128] are indexed + limitA: 127, + tailA: 2, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 128 + // TailA: 1 + // + // block [2, 128] are indexed + limitA: 128, + tailA: 1, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 129 + // TailA: 0 + // + // block [0, 128] are indexed + limitA: 129, + tailA: 0, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + } + for _, c := range cases { + frdir := t.TempDir() + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) + + // Index the initial blocks from ancient store + indexer := &txIndexer{ + limit: c.limitA, + db: db, + progress: make(chan chan TxIndexProgress), + } + indexer.run(nil, 128, make(chan struct{}), make(chan struct{})) + verify(db, c.tailA, indexer) + + indexer.limit = c.limitB + indexer.run(rawdb.ReadTxIndexTail(db), 128, make(chan struct{}), make(chan struct{})) + verify(db, c.tailB, indexer) + + indexer.limit = c.limitC + indexer.run(rawdb.ReadTxIndexTail(db), 128, make(chan struct{}), make(chan struct{})) + verify(db, c.tailC, indexer) + + // Recover all indexes + indexer.limit = 0 + indexer.run(rawdb.ReadTxIndexTail(db), 128, make(chan struct{}), make(chan struct{})) + verify(db, 0, indexer) + + db.Close() + os.RemoveAll(frdir) + } +} diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 92be8cef43..41ec930d50 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -386,6 +386,8 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr if len(fails) > 0 { log.Warn("Dropping invalidated blob transactions", "ids", fails) + dropInvalidMeter.Mark(int64(len(fails))) + for _, id := range fails { if err := p.store.Delete(id); err != nil { p.Close() @@ -456,7 +458,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { tx := new(types.Transaction) if err := rlp.DecodeBytes(blob, tx); err != nil { // This path is impossible unless the disk data representation changes - // across restarts. For that ever unprobable case, recover gracefully + // across restarts. For that ever improbable case, recover gracefully // by ignoring this data entry. log.Error("Failed to decode blob pool entry", "id", id, "err", err) return err @@ -467,11 +469,17 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { } meta := newBlobTxMeta(id, size, tx) - + if _, exists := p.lookup[meta.hash]; exists { + // This path is only possible after a crash, where deleted items are not + // removed via the normal shutdown-startup procedure and thus may get + // partially resurrected. + log.Error("Rejecting duplicate blob pool entry", "id", id, "hash", tx.Hash()) + return errors.New("duplicate blob entry") + } sender, err := p.signer.Sender(tx) if err != nil { // This path is impossible unless the signature validity changes across - // restarts. For that ever unprobable case, recover gracefully by ignoring + // restarts. For that ever improbable case, recover gracefully by ignoring // this data entry. log.Error("Failed to recover blob tx sender", "id", id, "hash", tx.Hash(), "err", err) return err @@ -537,8 +545,10 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 if gapped { log.Warn("Dropping dangling blob transactions", "from", addr, "missing", next, "drop", nonces, "ids", ids) + dropDanglingMeter.Mark(int64(len(ids))) } else { log.Trace("Dropping filled blob transactions", "from", addr, "filled", nonces, "ids", ids) + dropFilledMeter.Mark(int64(len(ids))) } for _, id := range ids { if err := p.store.Delete(id); err != nil { @@ -569,6 +579,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 txs = txs[1:] } log.Trace("Dropping overlapped blob transactions", "from", addr, "overlapped", nonces, "ids", ids, "left", len(txs)) + dropOverlappedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -600,10 +612,30 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 } continue } - // Sanity check that there's no double nonce. This case would be a coding - // error, but better know about it + // Sanity check that there's no double nonce. This case would generally + // be a coding error, so better know about it. + // + // Also, Billy behind the blobpool does not journal deletes. A process + // crash would result in previously deleted entities being resurrected. + // That could potentially cause a duplicate nonce to appear. if txs[i].nonce == txs[i-1].nonce { - log.Error("Duplicate nonce blob transaction", "from", addr, "nonce", txs[i].nonce) + id := p.lookup[txs[i].hash] + + log.Error("Dropping repeat nonce blob transaction", "from", addr, "nonce", txs[i].nonce, "id", id) + dropRepeatedMeter.Mark(1) + + p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[i].costCap) + p.stored -= uint64(txs[i].size) + delete(p.lookup, txs[i].hash) + + if err := p.store.Delete(id); err != nil { + log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) + } + txs = append(txs[:i], txs[i+1:]...) + p.index[addr] = txs + + i-- + continue } // Otherwise if there's a nonce gap evict all later transactions var ( @@ -621,6 +653,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 txs = txs[:i] log.Error("Dropping gapped blob transactions", "from", addr, "missing", txs[i-1].nonce+1, "drop", nonces, "ids", ids) + dropGappedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -632,7 +666,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 // Ensure that there's no over-draft, this is expected to happen when some // transactions get included without publishing on the network var ( - balance = uint256.MustFromBig(p.state.GetBalance(addr)) + balance = p.state.GetBalance(addr) spent = p.spent[addr] ) if spent.Cmp(balance) > 0 { @@ -665,6 +699,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 p.index[addr] = txs } log.Warn("Dropping overdrafted blob transactions", "from", addr, "balance", balance, "spent", spent, "drop", nonces, "ids", ids) + dropOverdraftedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -695,6 +731,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 p.index[addr] = txs log.Warn("Dropping overcapped blob transactions", "from", addr, "kept", len(txs), "drop", nonces, "ids", ids) + dropOvercappedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -711,7 +749,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 // offload removes a tracked blob transaction from the pool and moves it into the // limbo for tracking until finality. // -// The method may log errors for various unexpcted scenarios but will not return +// The method may log errors for various unexpected scenarios but will not return // any of it since there's no clear error case. Some errors may be due to coding // issues, others caused by signers mining MEV stuff or swapping transactions. In // all cases, the pool needs to continue operating. @@ -952,7 +990,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error { return err } - // Update the indixes and metrics + // Update the indices and metrics meta := newBlobTxMeta(id, p.store.Size(id), tx) if _, ok := p.index[addr]; !ok { if err := p.reserve(addr, true); err != nil { @@ -1019,6 +1057,8 @@ func (p *BlobPool) SetGasTip(tip *big.Int) { } // Clear out the transactions from the data store log.Warn("Dropping underpriced blob transaction", "from", addr, "rejected", tx.nonce, "tip", tx.execTipCap, "want", tip, "drop", nonces, "ids", ids) + dropUnderpricedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete dropped transaction", "id", id, "err", err) @@ -1161,7 +1201,7 @@ func (p *BlobPool) Get(hash common.Hash) *types.Transaction { } // Add inserts a set of blob transactions into the pool if they pass validation (both -// consensus validity and pool restictions). +// consensus validity and pool restrictions). func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error { var ( adds = make([]*types.Transaction, 0, len(txs)) @@ -1181,7 +1221,7 @@ func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error } // Add inserts a new blob transaction into the pool if it passes validation (both -// consensus validity and pool restictions). +// consensus validity and pool restrictions). func (p *BlobPool) add(tx *types.Transaction) (err error) { // The blob pool blocks on adding a transaction. This is because blob txs are // only even pulled form the network, so this method will act as the overload @@ -1198,6 +1238,22 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { // Ensure the transaction is valid from all perspectives if err := p.validateTx(tx); err != nil { log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err) + switch { + case errors.Is(err, txpool.ErrUnderpriced): + addUnderpricedMeter.Mark(1) + case errors.Is(err, core.ErrNonceTooLow): + addStaleMeter.Mark(1) + case errors.Is(err, core.ErrNonceTooHigh): + addGappedMeter.Mark(1) + case errors.Is(err, core.ErrInsufficientFunds): + addOverdraftedMeter.Mark(1) + case errors.Is(err, txpool.ErrAccountLimitExceeded): + addOvercappedMeter.Mark(1) + case errors.Is(err, txpool.ErrReplaceUnderpriced): + addNoreplaceMeter.Mark(1) + default: + addInvalidMeter.Mark(1) + } return err } // If the address is not yet known, request exclusivity to track the account @@ -1205,6 +1261,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { from, _ := types.Sender(p.signer, tx) // already validated above if _, ok := p.index[from]; !ok { if err := p.reserve(from, true); err != nil { + addNonExclusiveMeter.Mark(1) return err } defer func() { @@ -1244,6 +1301,8 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { } if len(p.index[from]) > offset { // Transaction replaces a previously queued one + dropReplacedMeter.Mark(1) + prev := p.index[from][offset] if err := p.store.Delete(prev.id); err != nil { // Shitty situation, but try to recover gracefully instead of going boom @@ -1322,6 +1381,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { } p.updateStorageMetrics() + addValidMeter.Mark(1) return nil } @@ -1371,7 +1431,9 @@ func (p *BlobPool) drop() { } } // Remove the transaction from the data store - log.Warn("Evicting overflown blob transaction", "from", from, "evicted", drop.nonce, "id", drop.id) + log.Debug("Evicting overflown blob transaction", "from", from, "evicted", drop.nonce, "id", drop.id) + dropOverflownMeter.Mark(1) + if err := p.store.Delete(drop.id); err != nil { log.Error("Failed to drop evicted transaction", "id", drop.id, "err", err) } diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 09c78cfd80..a71c452b79 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -305,7 +305,16 @@ func verifyPoolInternals(t *testing.T, pool *BlobPool) { // - 1. A transaction that cannot be decoded must be dropped // - 2. A transaction that cannot be recovered (bad signature) must be dropped // - 3. All transactions after a nonce gap must be dropped -// - 4. All transactions after an underpriced one (including it) must be dropped +// - 4. All transactions after an already included nonce must be dropped +// - 5. All transactions after an underpriced one (including it) must be dropped +// - 6. All transactions after an overdrafting sequence must be dropped +// - 7. All transactions exceeding the per-account limit must be dropped +// +// Furthermore, some strange corner-cases can also occur after a crash, as Billy's +// simplicity also allows it to resurrect past deleted entities: +// +// - 8. Fully duplicate transactions (matching hash) must be dropped +// - 9. Duplicate nonces from the same account must be dropped func TestOpenDrops(t *testing.T) { log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) @@ -338,7 +347,7 @@ func TestOpenDrops(t *testing.T) { badsig, _ := store.Put(blob) // Insert a sequence of transactions with a nonce gap in between to verify - // that anything gapped will get evicted (case 3) + // that anything gapped will get evicted (case 3). var ( gapper, _ = crypto.GenerateKey() @@ -357,7 +366,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions with a gapped starting nonce to verify - // that the entire set will get dropped. + // that the entire set will get dropped (case 3). var ( dangler, _ = crypto.GenerateKey() dangling = make(map[uint64]struct{}) @@ -370,7 +379,7 @@ func TestOpenDrops(t *testing.T) { dangling[id] = struct{}{} } // Insert a sequence of transactions with already passed nonces to veirfy - // that the entire set will get dropped. + // that the entire set will get dropped (case 4). var ( filler, _ = crypto.GenerateKey() filled = make(map[uint64]struct{}) @@ -383,7 +392,7 @@ func TestOpenDrops(t *testing.T) { filled[id] = struct{}{} } // Insert a sequence of transactions with partially passed nonces to veirfy - // that the included part of the set will get dropped + // that the included part of the set will get dropped (case 4). var ( overlapper, _ = crypto.GenerateKey() overlapped = make(map[uint64]struct{}) @@ -400,7 +409,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions with an underpriced first to verify that - // the entire set will get dropped (case 4). + // the entire set will get dropped (case 5). var ( underpayer, _ = crypto.GenerateKey() underpaid = make(map[uint64]struct{}) @@ -419,7 +428,7 @@ func TestOpenDrops(t *testing.T) { } // Insert a sequence of transactions with an underpriced in between to verify - // that it and anything newly gapped will get evicted (case 4). + // that it and anything newly gapped will get evicted (case 5). var ( outpricer, _ = crypto.GenerateKey() outpriced = make(map[uint64]struct{}) @@ -441,7 +450,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions fully overdrafted to verify that the - // entire set will get invalidated. + // entire set will get invalidated (case 6). var ( exceeder, _ = crypto.GenerateKey() exceeded = make(map[uint64]struct{}) @@ -459,7 +468,7 @@ func TestOpenDrops(t *testing.T) { exceeded[id] = struct{}{} } // Insert a sequence of transactions partially overdrafted to verify that part - // of the set will get invalidated. + // of the set will get invalidated (case 6). var ( overdrafter, _ = crypto.GenerateKey() overdrafted = make(map[uint64]struct{}) @@ -481,7 +490,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions overflowing the account cap to verify - // that part of the set will get invalidated. + // that part of the set will get invalidated (case 7). var ( overcapper, _ = crypto.GenerateKey() overcapped = make(map[uint64]struct{}) @@ -496,21 +505,59 @@ func TestOpenDrops(t *testing.T) { overcapped[id] = struct{}{} } } + // Insert a batch of duplicated transactions to verify that only one of each + // version will remain (case 8). + var ( + duplicater, _ = crypto.GenerateKey() + duplicated = make(map[uint64]struct{}) + ) + for _, nonce := range []uint64{0, 1, 2} { + blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, 1, 1, duplicater)) + + for i := 0; i < int(nonce)+1; i++ { + id, _ := store.Put(blob) + if i == 0 { + valids[id] = struct{}{} + } else { + duplicated[id] = struct{}{} + } + } + } + // Insert a batch of duplicated nonces to verify that only one of each will + // remain (case 9). + var ( + repeater, _ = crypto.GenerateKey() + repeated = make(map[uint64]struct{}) + ) + for _, nonce := range []uint64{0, 1, 2} { + for i := 0; i < int(nonce)+1; i++ { + blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater)) + + id, _ := store.Put(blob) + if i == 0 { + valids[id] = struct{}{} + } else { + repeated[id] = struct{}{} + } + } + } store.Close() // Create a blob pool out of the pre-seeded data statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(crypto.PubkeyToAddress(gapper.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(dangler.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(filler.PublicKey), big.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(gapper.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(dangler.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(filler.PublicKey), uint256.NewInt(1000000)) statedb.SetNonce(crypto.PubkeyToAddress(filler.PublicKey), 3) - statedb.AddBalance(crypto.PubkeyToAddress(overlapper.PublicKey), big.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(overlapper.PublicKey), uint256.NewInt(1000000)) statedb.SetNonce(crypto.PubkeyToAddress(overlapper.PublicKey), 2) - statedb.AddBalance(crypto.PubkeyToAddress(underpayer.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(outpricer.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(exceeder.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(overdrafter.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(overcapper.PublicKey), big.NewInt(10000000)) + statedb.AddBalance(crypto.PubkeyToAddress(underpayer.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(outpricer.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(exceeder.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(overdrafter.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(overcapper.PublicKey), uint256.NewInt(10000000)) + statedb.AddBalance(crypto.PubkeyToAddress(duplicater.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(repeater.PublicKey), uint256.NewInt(1000000)) statedb.Commit(0, true) chain := &testBlockChain{ @@ -554,6 +601,10 @@ func TestOpenDrops(t *testing.T) { t.Errorf("partially overdrafted transaction remained in storage: %d", tx.id) } else if _, ok := overcapped[tx.id]; ok { t.Errorf("overcapped transaction remained in storage: %d", tx.id) + } else if _, ok := duplicated[tx.id]; ok { + t.Errorf("duplicated transaction remained in storage: %d", tx.id) + } else if _, ok := repeated[tx.id]; ok { + t.Errorf("repeated nonce transaction remained in storage: %d", tx.id) } else { alive[tx.id] = struct{}{} } @@ -584,7 +635,7 @@ func TestOpenDrops(t *testing.T) { // Tests that transactions loaded from disk are indexed correctly. // -// - 1. Transactions must be groupped by sender, sorted by nonce +// - 1. Transactions must be grouped by sender, sorted by nonce // - 2. Eviction thresholds are calculated correctly for the sequences // - 3. Balance usage of an account is totals across all transactions func TestOpenIndex(t *testing.T) { @@ -598,7 +649,7 @@ func TestOpenIndex(t *testing.T) { store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(), nil) // Insert a sequence of transactions with varying price points to check that - // the cumulative minimumw will be maintained. + // the cumulative minimum will be maintained. var ( key, _ = crypto.GenerateKey() addr = crypto.PubkeyToAddress(key.PublicKey) @@ -625,7 +676,7 @@ func TestOpenIndex(t *testing.T) { // Create a blob pool out of the pre-seeded data statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(addr, big.NewInt(1_000_000_000)) + statedb.AddBalance(addr, uint256.NewInt(1_000_000_000)) statedb.Commit(0, true) chain := &testBlockChain{ @@ -725,9 +776,9 @@ func TestOpenHeap(t *testing.T) { // Create a blob pool out of the pre-seeded data statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(addr1, big.NewInt(1_000_000_000)) - statedb.AddBalance(addr2, big.NewInt(1_000_000_000)) - statedb.AddBalance(addr3, big.NewInt(1_000_000_000)) + statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000)) + statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000)) + statedb.AddBalance(addr3, uint256.NewInt(1_000_000_000)) statedb.Commit(0, true) chain := &testBlockChain{ @@ -805,9 +856,9 @@ func TestOpenCap(t *testing.T) { for _, datacap := range []uint64{2 * (txAvgSize + blobSize), 100 * (txAvgSize + blobSize)} { // Create a blob pool out of the pre-seeded data, but cap it to 2 blob transaction statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(addr1, big.NewInt(1_000_000_000)) - statedb.AddBalance(addr2, big.NewInt(1_000_000_000)) - statedb.AddBalance(addr3, big.NewInt(1_000_000_000)) + statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000)) + statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000)) + statedb.AddBalance(addr3, uint256.NewInt(1_000_000_000)) statedb.Commit(0, true) chain := &testBlockChain{ @@ -1197,8 +1248,8 @@ func TestAdd(t *testing.T) { keys[acc], _ = crypto.GenerateKey() addrs[acc] = crypto.PubkeyToAddress(keys[acc].PublicKey) - // Seed the state database with this acocunt - statedb.AddBalance(addrs[acc], new(big.Int).SetUint64(seed.balance)) + // Seed the state database with this account + statedb.AddBalance(addrs[acc], new(uint256.Int).SetUint64(seed.balance)) statedb.SetNonce(addrs[acc], seed.nonce) // Sign the seed transactions and store them in the data store diff --git a/core/txpool/blobpool/limbo.go b/core/txpool/blobpool/limbo.go index d1fe9c7394..ec754f6894 100644 --- a/core/txpool/blobpool/limbo.go +++ b/core/txpool/blobpool/limbo.go @@ -53,7 +53,7 @@ func newLimbo(datadir string) (*limbo, error) { index: make(map[common.Hash]uint64), groups: make(map[uint64]map[uint64]common.Hash), } - // Index all limboed blobs on disk and delete anything inprocessable + // Index all limboed blobs on disk and delete anything unprocessable var fails []uint64 index := func(id uint64, size uint32, data []byte) { if l.parseBlob(id, data) != nil { @@ -89,7 +89,7 @@ func (l *limbo) parseBlob(id uint64, data []byte) error { item := new(limboBlob) if err := rlp.DecodeBytes(data, item); err != nil { // This path is impossible unless the disk data representation changes - // across restarts. For that ever unprobable case, recover gracefully + // across restarts. For that ever improbable case, recover gracefully // by ignoring this data entry. log.Error("Failed to decode blob limbo entry", "id", id, "err", err) return err @@ -172,7 +172,7 @@ func (l *limbo) pull(tx common.Hash) (*types.Transaction, error) { // update changes the block number under which a blob transaction is tracked. This // method should be used when a reorg changes a transaction's inclusion block. // -// The method may log errors for various unexpcted scenarios but will not return +// The method may log errors for various unexpected scenarios but will not return // any of it since there's no clear error case. Some errors may be due to coding // issues, others caused by signers mining MEV stuff or swapping transactions. In // all cases, the pool needs to continue operating. diff --git a/core/txpool/blobpool/metrics.go b/core/txpool/blobpool/metrics.go index 587804cc61..52419ade09 100644 --- a/core/txpool/blobpool/metrics.go +++ b/core/txpool/blobpool/metrics.go @@ -65,8 +65,8 @@ var ( pooltipGauge = metrics.NewRegisteredGauge("blobpool/pooltip", nil) // addwait/time, resetwait/time and getwait/time track the rough health of - // the pool and whether or not it's capable of keeping up with the load from - // the network. + // the pool and whether it's capable of keeping up with the load from the + // network. addwaitHist = metrics.NewRegisteredHistogram("blobpool/addwait", nil, metrics.NewExpDecaySample(1028, 0.015)) addtimeHist = metrics.NewRegisteredHistogram("blobpool/addtime", nil, metrics.NewExpDecaySample(1028, 0.015)) getwaitHist = metrics.NewRegisteredHistogram("blobpool/getwait", nil, metrics.NewExpDecaySample(1028, 0.015)) @@ -75,4 +75,31 @@ var ( pendtimeHist = metrics.NewRegisteredHistogram("blobpool/pendtime", nil, metrics.NewExpDecaySample(1028, 0.015)) resetwaitHist = metrics.NewRegisteredHistogram("blobpool/resetwait", nil, metrics.NewExpDecaySample(1028, 0.015)) resettimeHist = metrics.NewRegisteredHistogram("blobpool/resettime", nil, metrics.NewExpDecaySample(1028, 0.015)) + + // The below metrics track various cases where transactions are dropped out + // of the pool. Most are exceptional, some are chain progression and some + // threshold cappings. + dropInvalidMeter = metrics.NewRegisteredMeter("blobpool/drop/invalid", nil) // Invalid transaction, consensus change or bugfix, neutral-ish + dropDanglingMeter = metrics.NewRegisteredMeter("blobpool/drop/dangling", nil) // First nonce gapped, bad + dropFilledMeter = metrics.NewRegisteredMeter("blobpool/drop/filled", nil) // State full-overlap, chain progress, ok + dropOverlappedMeter = metrics.NewRegisteredMeter("blobpool/drop/overlapped", nil) // State partial-overlap, chain progress, ok + dropRepeatedMeter = metrics.NewRegisteredMeter("blobpool/drop/repeated", nil) // Repeated nonce, bad + dropGappedMeter = metrics.NewRegisteredMeter("blobpool/drop/gapped", nil) // Non-first nonce gapped, bad + dropOverdraftedMeter = metrics.NewRegisteredMeter("blobpool/drop/overdrafted", nil) // Balance exceeded, bad + dropOvercappedMeter = metrics.NewRegisteredMeter("blobpool/drop/overcapped", nil) // Per-account cap exceeded, bad + dropOverflownMeter = metrics.NewRegisteredMeter("blobpool/drop/overflown", nil) // Global disk cap exceeded, neutral-ish + dropUnderpricedMeter = metrics.NewRegisteredMeter("blobpool/drop/underpriced", nil) // Gas tip changed, neutral + dropReplacedMeter = metrics.NewRegisteredMeter("blobpool/drop/replaced", nil) // Transaction replaced, neutral + + // The below metrics track various outcomes of transactions being added to + // the pool. + addInvalidMeter = metrics.NewRegisteredMeter("blobpool/add/invalid", nil) // Invalid transaction, reject, neutral + addUnderpricedMeter = metrics.NewRegisteredMeter("blobpool/add/underpriced", nil) // Gas tip too low, neutral + addStaleMeter = metrics.NewRegisteredMeter("blobpool/add/stale", nil) // Nonce already filled, reject, bad-ish + addGappedMeter = metrics.NewRegisteredMeter("blobpool/add/gapped", nil) // Nonce gapped, reject, bad-ish + addOverdraftedMeter = metrics.NewRegisteredMeter("blobpool/add/overdrafted", nil) // Balance exceeded, reject, neutral + addOvercappedMeter = metrics.NewRegisteredMeter("blobpool/add/overcapped", nil) // Per-account cap exceeded, reject, neutral + addNoreplaceMeter = metrics.NewRegisteredMeter("blobpool/add/noreplace", nil) // Replacement fees or tips too low, neutral + addNonExclusiveMeter = metrics.NewRegisteredMeter("blobpool/add/nonexclusive", nil) // Plain transaction from same account exists, reject, neutral + addValidMeter = metrics.NewRegisteredMeter("blobpool/add/valid", nil) // Valid transaction, add, neutral ) diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 959e328b9c..624dafc60d 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1441,7 +1441,7 @@ func (pool *LegacyPool) promoteExecutables(accounts []common.Address) []*types.T } log.Trace("Removed old queued transactions", "count", len(forwards)) // Drop all transactions that are too costly (low balance or out of gas) - drops, _ := list.Filter(pool.currentState.GetBalance(addr), gasLimit) + drops, _ := list.Filter(pool.currentState.GetBalance(addr).ToBig(), gasLimit) for _, tx := range drops { hash := tx.Hash() pool.all.Remove(hash) @@ -1642,7 +1642,7 @@ func (pool *LegacyPool) demoteUnexecutables() { log.Trace("Removed old pending transaction", "hash", hash) } // Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later - drops, invalids := list.Filter(pool.currentState.GetBalance(addr), gasLimit) + drops, invalids := list.Filter(pool.currentState.GetBalance(addr).ToBig(), gasLimit) for _, tx := range drops { hash := tx.Hash() log.Trace("Removed unpayable pending transaction", "hash", hash) diff --git a/core/txpool/legacypool/legacypool2_test.go b/core/txpool/legacypool/legacypool2_test.go index a73c1bb8a7..0f53000b3d 100644 --- a/core/txpool/legacypool/legacypool2_test.go +++ b/core/txpool/legacypool/legacypool2_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/event" + "github.com/holiman/uint256" ) func pricedValuedTransaction(nonce uint64, value int64, gaslimit uint64, gasprice *big.Int, key *ecdsa.PrivateKey) *types.Transaction { @@ -49,7 +50,7 @@ func fillPool(t testing.TB, pool *LegacyPool) { nonExecutableTxs := types.Transactions{} for i := 0; i < 384; i++ { key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(10000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(10000000000)) // Add executable ones for j := 0; j < int(pool.config.AccountSlots); j++ { executableTxs = append(executableTxs, pricedTransaction(uint64(j), 100000, big.NewInt(300), key)) @@ -91,7 +92,7 @@ func TestTransactionFutureAttack(t *testing.T) { // Now, future transaction attack starts, let's add a bunch of expensive non-executables, and see if the pending-count drops { key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(100000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(100000000000)) futureTxs := types.Transactions{} for j := 0; j < int(pool.config.GlobalSlots+pool.config.GlobalQueue); j++ { futureTxs = append(futureTxs, pricedTransaction(1000+uint64(j), 100000, big.NewInt(500), key)) @@ -128,7 +129,7 @@ func TestTransactionFuture1559(t *testing.T) { // Now, future transaction attack starts, let's add a bunch of expensive non-executables, and see if the pending-count drops { key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(100000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(100000000000)) futureTxs := types.Transactions{} for j := 0; j < int(pool.config.GlobalSlots+pool.config.GlobalQueue); j++ { futureTxs = append(futureTxs, dynamicFeeTx(1000+uint64(j), 100000, big.NewInt(200), big.NewInt(101), key)) @@ -161,7 +162,7 @@ func TestTransactionZAttack(t *testing.T) { var ivpendingNum int pendingtxs, _ := pool.Content() for account, txs := range pendingtxs { - cur_balance := new(big.Int).Set(pool.currentState.GetBalance(account)) + cur_balance := new(big.Int).Set(pool.currentState.GetBalance(account).ToBig()) for _, tx := range txs { if cur_balance.Cmp(tx.Value()) <= 0 { ivpendingNum++ @@ -182,7 +183,7 @@ func TestTransactionZAttack(t *testing.T) { for j := 0; j < int(pool.config.GlobalQueue); j++ { futureTxs := types.Transactions{} key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(100000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(100000000000)) futureTxs = append(futureTxs, pricedTransaction(1000+uint64(j), 21000, big.NewInt(500), key)) pool.addRemotesSync(futureTxs) } @@ -190,7 +191,7 @@ func TestTransactionZAttack(t *testing.T) { overDraftTxs := types.Transactions{} { key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(100000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(100000000000)) for j := 0; j < int(pool.config.GlobalSlots); j++ { overDraftTxs = append(overDraftTxs, pricedValuedTransaction(uint64(j), 600000000000, 21000, big.NewInt(500), key)) } @@ -227,7 +228,7 @@ func BenchmarkFutureAttack(b *testing.B) { fillPool(b, pool) key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(100000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(100000000000)) futureTxs := types.Transactions{} for n := 0; n < b.N; n++ { diff --git a/core/txpool/legacypool/legacypool_test.go b/core/txpool/legacypool/legacypool_test.go index 0366a58d61..cd2cfb92e4 100644 --- a/core/txpool/legacypool/legacypool_test.go +++ b/core/txpool/legacypool/legacypool_test.go @@ -39,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) var ( @@ -255,7 +256,7 @@ func (c *testChain) State() (*state.StateDB, error) { c.statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) // simulate that the new head block included tx0 and tx1 c.statedb.SetNonce(c.address, 2) - c.statedb.SetBalance(c.address, new(big.Int).SetUint64(params.Ether)) + c.statedb.SetBalance(c.address, new(uint256.Int).SetUint64(params.Ether)) *c.trigger = false } return stdb, nil @@ -275,7 +276,7 @@ func TestStateChangeDuringReset(t *testing.T) { ) // setup pool with 2 transaction in it - statedb.SetBalance(address, new(big.Int).SetUint64(params.Ether)) + statedb.SetBalance(address, new(uint256.Int).SetUint64(params.Ether)) blockchain := &testChain{newTestBlockChain(params.TestChainConfig, 1000000000, statedb, new(event.Feed)), address, &trigger} tx0 := transaction(0, 100000, key) @@ -309,7 +310,7 @@ func TestStateChangeDuringReset(t *testing.T) { func testAddBalance(pool *LegacyPool, addr common.Address, amount *big.Int) { pool.mu.Lock() - pool.currentState.AddBalance(addr, amount) + pool.currentState.AddBalance(addr, uint256.MustFromBig(amount)) pool.mu.Unlock() } @@ -470,7 +471,7 @@ func TestChainFork(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - statedb.AddBalance(addr, big.NewInt(100000000000000)) + statedb.AddBalance(addr, uint256.NewInt(100000000000000)) pool.chain = newTestBlockChain(pool.chainconfig, 1000000, statedb, new(event.Feed)) <-pool.requestReset(nil, nil) @@ -499,7 +500,7 @@ func TestDoubleNonce(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - statedb.AddBalance(addr, big.NewInt(100000000000000)) + statedb.AddBalance(addr, uint256.NewInt(100000000000000)) pool.chain = newTestBlockChain(pool.chainconfig, 1000000, statedb, new(event.Feed)) <-pool.requestReset(nil, nil) @@ -2662,7 +2663,7 @@ func BenchmarkMultiAccountBatchInsert(b *testing.B) { for i := 0; i < b.N; i++ { key, _ := crypto.GenerateKey() account := crypto.PubkeyToAddress(key.PublicKey) - pool.currentState.AddBalance(account, big.NewInt(1000000)) + pool.currentState.AddBalance(account, uint256.NewInt(1000000)) tx := transaction(uint64(0), 100000, key) batches[i] = tx } diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index de05b38d43..2722174d79 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -44,11 +44,17 @@ type LazyTransaction struct { // Resolve retrieves the full transaction belonging to a lazy handle if it is still // maintained by the transaction pool. +// +// Note, the method will *not* cache the retrieved transaction if the original +// pool has not cached it. The idea being, that if the tx was too big to insert +// originally, silently saving it will cause more trouble down the line (and +// indeed seems to have caused a memory bloat in the original implementation +// which did just that). func (ltx *LazyTransaction) Resolve() *types.Transaction { - if ltx.Tx == nil { - ltx.Tx = ltx.Pool.Get(ltx.Hash) + if ltx.Tx != nil { + return ltx.Tx } - return ltx.Tx + return ltx.Pool.Get(ltx.Hash) } // LazyResolver is a minimal interface needed for a transaction pool to satisfy @@ -69,7 +75,7 @@ type AddressReserver func(addr common.Address, reserve bool) error // production, this interface defines the common methods that allow the primary // transaction pool to manage the subpools. type SubPool interface { - // Filter is a selector used to decide whether a transaction whould be added + // Filter is a selector used to decide whether a transaction would be added // to this particular subpool. Filter(tx *types.Transaction) bool diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 0d4e05da4c..d03e025a9e 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -72,6 +72,9 @@ type TxPool struct { subs event.SubscriptionScope // Subscription scope to unsubscribe all on shutdown quit chan chan error // Quit channel to tear down the head updater + term chan struct{} // Termination channel to detect a closed pool + + sync chan chan error // Testing / simulator channel to block until internal reset is done } // New creates a new transaction pool to gather, sort and filter inbound @@ -86,6 +89,8 @@ func New(gasTip *big.Int, chain BlockChain, subpools []SubPool) (*TxPool, error) subpools: subpools, reservations: make(map[common.Address]SubPool), quit: make(chan chan error), + term: make(chan struct{}), + sync: make(chan chan error), } for i, subpool := range subpools { if err := subpool.Init(gasTip, head, pool.reserver(i, subpool)); err != nil { @@ -174,6 +179,9 @@ func (p *TxPool) Close() error { // outside blockchain events as well as for various reporting and transaction // eviction events. func (p *TxPool) loop(head *types.Header, chain BlockChain) { + // Close the termination marker when the pool stops + defer close(p.term) + // Subscribe to chain head events to trigger subpool resets var ( newHeadCh = make(chan core.ChainHeadEvent) @@ -190,13 +198,23 @@ func (p *TxPool) loop(head *types.Header, chain BlockChain) { var ( resetBusy = make(chan struct{}, 1) // Allow 1 reset to run concurrently resetDone = make(chan *types.Header) + + resetForced bool // Whether a forced reset was requested, only used in simulator mode + resetWaiter chan error // Channel waiting on a forced reset, only used in simulator mode ) + // Notify the live reset waiter to not block if the txpool is closed. + defer func() { + if resetWaiter != nil { + resetWaiter <- errors.New("pool already terminated") + resetWaiter = nil + } + }() var errc chan error for errc == nil { // Something interesting might have happened, run a reset if there is // one needed but none is running. The resetter will run on its own // goroutine to allow chain head events to be consumed contiguously. - if newHead != oldHead { + if newHead != oldHead || resetForced { // Try to inject a busy marker and start a reset if successful select { case resetBusy <- struct{}{}: @@ -208,8 +226,17 @@ func (p *TxPool) loop(head *types.Header, chain BlockChain) { resetDone <- newHead }(oldHead, newHead) + // If the reset operation was explicitly requested, consider it + // being fulfilled and drop the request marker. If it was not, + // this is a noop. + resetForced = false + default: - // Reset already running, wait until it finishes + // Reset already running, wait until it finishes. + // + // Note, this will not drop any forced reset request. If a forced + // reset was requested, but we were busy, then when the currently + // running reset finishes, a new one will be spun up. } } // Wait for the next chain head event or a previous reset finish @@ -223,8 +250,26 @@ func (p *TxPool) loop(head *types.Header, chain BlockChain) { oldHead = head <-resetBusy + // If someone is waiting for a reset to finish, notify them, unless + // the forced op is still pending. In that case, wait another round + // of resets. + if resetWaiter != nil && !resetForced { + resetWaiter <- nil + resetWaiter = nil + } + case errc = <-p.quit: // Termination requested, break out on the next loop round + + case syncc := <-p.sync: + // Transaction pool is running inside a simulator, and we are about + // to create a new block. Request a forced sync operation to ensure + // that any running reset operation finishes to make block imports + // deterministic. On top of that, run a new reset operation to make + // transaction insertions deterministic instead of being stuck in a + // queue waiting for a reset. + resetForced = true + resetWaiter = syncc } } // Notify the closer of termination (no error possible for now) @@ -415,3 +460,20 @@ func (p *TxPool) Status(hash common.Hash) TxStatus { } return TxStatusUnknown } + +// Sync is a helper method for unit tests or simulator runs where the chain events +// are arriving in quick succession, without any time in between them to run the +// internal background reset operations. This method will run an explicit reset +// operation to ensure the pool stabilises, thus avoiding flakey behavior. +// +// Note, do not use this in production / live code. In live code, the pool is +// meant to reset on a separate thread to avoid DoS vectors. +func (p *TxPool) Sync() error { + sync := make(chan error) + select { + case p.sync <- sync: + return <-sync + case <-p.term: + return errors.New("pool already terminated") + } +} diff --git a/core/txpool/validation.go b/core/txpool/validation.go index cac2f334ac..a9bd14020b 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -209,7 +209,7 @@ func ValidateTransactionWithState(tx *types.Transaction, signer types.Signer, op } // Ensure the transactor has enough funds to cover the transaction costs var ( - balance = opts.State.GetBalance(from) + balance = opts.State.GetBalance(from).ToBig() cost = tx.Cost() ) if balance.Cmp(cost) < 0 { diff --git a/core/types/gen_account_rlp.go b/core/types/gen_account_rlp.go index 3fb36f4038..8b424493af 100644 --- a/core/types/gen_account_rlp.go +++ b/core/types/gen_account_rlp.go @@ -12,10 +12,7 @@ func (obj *StateAccount) EncodeRLP(_w io.Writer) error { if obj.Balance == nil { w.Write(rlp.EmptyString) } else { - if obj.Balance.Sign() == -1 { - return rlp.ErrNegativeBigInt - } - w.WriteBigInt(obj.Balance) + w.WriteUint256(obj.Balance) } w.WriteBytes(obj.Root[:]) w.WriteBytes(obj.CodeHash) diff --git a/core/types/state_account.go b/core/types/state_account.go index ad07ca3f3a..52ef843b35 100644 --- a/core/types/state_account.go +++ b/core/types/state_account.go @@ -18,10 +18,10 @@ package types import ( "bytes" - "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" + "github.com/holiman/uint256" ) //go:generate go run ../../rlp/rlpgen -type StateAccount -out gen_account_rlp.go @@ -30,7 +30,7 @@ import ( // These objects are stored in the main account trie. type StateAccount struct { Nonce uint64 - Balance *big.Int + Balance *uint256.Int Root common.Hash // merkle root of the storage trie CodeHash []byte } @@ -38,7 +38,7 @@ type StateAccount struct { // NewEmptyStateAccount constructs an empty state account. func NewEmptyStateAccount() *StateAccount { return &StateAccount{ - Balance: new(big.Int), + Balance: new(uint256.Int), Root: EmptyRootHash, CodeHash: EmptyCodeHash.Bytes(), } @@ -46,9 +46,9 @@ func NewEmptyStateAccount() *StateAccount { // Copy returns a deep-copied state account object. func (acct *StateAccount) Copy() *StateAccount { - var balance *big.Int + var balance *uint256.Int if acct.Balance != nil { - balance = new(big.Int).Set(acct.Balance) + balance = new(uint256.Int).Set(acct.Balance) } return &StateAccount{ Nonce: acct.Nonce, @@ -63,7 +63,7 @@ func (acct *StateAccount) Copy() *StateAccount { // or slim format which replaces the empty root and code hash as nil byte slice. type SlimAccount struct { Nonce uint64 - Balance *big.Int + Balance *uint256.Int Root []byte // Nil if root equals to types.EmptyRootHash CodeHash []byte // Nil if hash equals to types.EmptyCodeHash } diff --git a/core/types/transaction.go b/core/types/transaction.go index 9ec0199a03..7d2e9d5325 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -19,6 +19,7 @@ package types import ( "bytes" "errors" + "fmt" "io" "math/big" "sync/atomic" @@ -320,6 +321,7 @@ func (tx *Transaction) Cost() *big.Int { // RawSignatureValues returns the V, R, S signature values of the transaction. // The return values should not be modified by the caller. +// The return values may be nil or zero, if the transaction is unsigned. func (tx *Transaction) RawSignatureValues() (v, r, s *big.Int) { return tx.inner.rawSignatureValues() } @@ -508,6 +510,9 @@ func (tx *Transaction) WithSignature(signer Signer, sig []byte) (*Transaction, e if err != nil { return nil, err } + if r == nil || s == nil || v == nil { + return nil, fmt.Errorf("%w: r: %s, s: %s, v: %s", ErrInvalidSig, r, s, v) + } cpy := tx.inner.copy() cpy.setSignatureValues(signer.ChainID(), v, r, s) return &Transaction{inner: cpy, time: tx.time}, nil diff --git a/core/types/transaction_marshalling.go b/core/types/transaction_marshalling.go index 08ce80b07c..4d5b2bcdd4 100644 --- a/core/types/transaction_marshalling.go +++ b/core/types/transaction_marshalling.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/holiman/uint256" ) @@ -47,6 +48,11 @@ type txJSON struct { S *hexutil.Big `json:"s"` YParity *hexutil.Uint64 `json:"yParity,omitempty"` + // Blob transaction sidecar encoding: + Blobs []kzg4844.Blob `json:"blobs,omitempty"` + Commitments []kzg4844.Commitment `json:"commitments,omitempty"` + Proofs []kzg4844.Proof `json:"proofs,omitempty"` + // Only used for encoding: Hash common.Hash `json:"hash"` } @@ -142,6 +148,11 @@ func (tx *Transaction) MarshalJSON() ([]byte, error) { enc.S = (*hexutil.Big)(itx.S.ToBig()) yparity := itx.V.Uint64() enc.YParity = (*hexutil.Uint64)(&yparity) + if sidecar := itx.Sidecar; sidecar != nil { + enc.Blobs = itx.Sidecar.Blobs + enc.Commitments = itx.Sidecar.Commitments + enc.Proofs = itx.Sidecar.Proofs + } } return json.Marshal(&enc) } diff --git a/core/types/transaction_signing_test.go b/core/types/transaction_signing_test.go index 2a9ceb0952..b66577f7ed 100644 --- a/core/types/transaction_signing_test.go +++ b/core/types/transaction_signing_test.go @@ -18,11 +18,13 @@ package types import ( "errors" + "fmt" "math/big" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -41,7 +43,7 @@ func TestEIP155Signing(t *testing.T) { t.Fatal(err) } if from != addr { - t.Errorf("exected from and address to be equal. Got %x want %x", from, addr) + t.Errorf("expected from and address to be equal. Got %x want %x", from, addr) } } @@ -136,3 +138,53 @@ func TestChainId(t *testing.T) { t.Error("expected no error") } } + +type nilSigner struct { + v, r, s *big.Int + Signer +} + +func (ns *nilSigner) SignatureValues(tx *Transaction, sig []byte) (r, s, v *big.Int, err error) { + return ns.v, ns.r, ns.s, nil +} + +// TestNilSigner ensures a faulty Signer implementation does not result in nil signature values or panics. +func TestNilSigner(t *testing.T) { + key, _ := crypto.GenerateKey() + innerSigner := LatestSignerForChainID(big.NewInt(1)) + for i, signer := range []Signer{ + &nilSigner{v: nil, r: nil, s: nil, Signer: innerSigner}, + &nilSigner{v: big.NewInt(1), r: big.NewInt(1), s: nil, Signer: innerSigner}, + &nilSigner{v: big.NewInt(1), r: nil, s: big.NewInt(1), Signer: innerSigner}, + &nilSigner{v: nil, r: big.NewInt(1), s: big.NewInt(1), Signer: innerSigner}, + } { + t.Run(fmt.Sprintf("signer_%d", i), func(t *testing.T) { + t.Run("legacy", func(t *testing.T) { + legacyTx := createTestLegacyTxInner() + _, err := SignNewTx(key, signer, legacyTx) + if !errors.Is(err, ErrInvalidSig) { + t.Fatal("expected signature values error, no nil result or panic") + } + }) + // test Blob tx specifically, since the signature value types changed + t.Run("blobtx", func(t *testing.T) { + blobtx := createEmptyBlobTxInner(false) + _, err := SignNewTx(key, signer, blobtx) + if !errors.Is(err, ErrInvalidSig) { + t.Fatal("expected signature values error, no nil result or panic") + } + }) + }) + } +} + +func createTestLegacyTxInner() *LegacyTx { + return &LegacyTx{ + Nonce: uint64(0), + To: nil, + Value: big.NewInt(0), + Gas: params.TxGas, + GasPrice: big.NewInt(params.GWei), + Data: nil, + } +} diff --git a/core/types/tx_blob.go b/core/types/tx_blob.go index caede7cc53..25a85695ef 100644 --- a/core/types/tx_blob.go +++ b/core/types/tx_blob.go @@ -43,7 +43,7 @@ type BlobTx struct { BlobHashes []common.Hash // A blob transaction can optionally contain blobs. This field must be set when BlobTx - // is used to create a transaction for sigining. + // is used to create a transaction for signing. Sidecar *BlobTxSidecar `rlp:"-"` // Signature values diff --git a/core/types/tx_blob_test.go b/core/types/tx_blob_test.go index 44ac48cc6f..25d09e31ce 100644 --- a/core/types/tx_blob_test.go +++ b/core/types/tx_blob_test.go @@ -65,6 +65,12 @@ var ( ) func createEmptyBlobTx(key *ecdsa.PrivateKey, withSidecar bool) *Transaction { + blobtx := createEmptyBlobTxInner(withSidecar) + signer := NewCancunSigner(blobtx.ChainID.ToBig()) + return MustSignNewTx(key, signer, blobtx) +} + +func createEmptyBlobTxInner(withSidecar bool) *BlobTx { sidecar := &BlobTxSidecar{ Blobs: []kzg4844.Blob{emptyBlob}, Commitments: []kzg4844.Commitment{emptyBlobCommit}, @@ -85,6 +91,5 @@ func createEmptyBlobTx(key *ecdsa.PrivateKey, withSidecar bool) *Transaction { if withSidecar { blobtx.Sidecar = sidecar } - signer := NewCancunSigner(blobtx.ChainID.ToBig()) - return MustSignNewTx(key, signer, blobtx) + return blobtx } diff --git a/core/vm/contract.go b/core/vm/contract.go index e4b03bd74f..16b669ebca 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -17,8 +17,6 @@ package vm import ( - "math/big" - "github.com/ethereum/go-ethereum/common" "github.com/holiman/uint256" ) @@ -59,11 +57,11 @@ type Contract struct { Input []byte Gas uint64 - value *big.Int + value *uint256.Int } // NewContract returns a new contract environment for the execution of EVM. -func NewContract(caller ContractRef, object ContractRef, value *big.Int, gas uint64) *Contract { +func NewContract(caller ContractRef, object ContractRef, value *uint256.Int, gas uint64) *Contract { c := &Contract{CallerAddress: caller.Address(), caller: caller, self: object} if parent, ok := caller.(*Contract); ok { @@ -173,7 +171,7 @@ func (c *Contract) Address() common.Address { } // Value returns the contract's value (sent to it from it's caller) -func (c *Contract) Value() *big.Int { +func (c *Contract) Value() *uint256.Int { return c.value } diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 574bb9bef6..33a867654e 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -267,7 +267,6 @@ type bigModExp struct { } var ( - big0 = big.NewInt(0) big1 = big.NewInt(1) big3 = big.NewInt(3) big4 = big.NewInt(4) diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index f40e2c8f9e..fc30541d45 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -223,7 +223,7 @@ func BenchmarkPrecompiledRipeMD(bench *testing.B) { benchmarkPrecompiled("03", t, bench) } -// Benchmarks the sample inputs from the identiy precompile. +// Benchmarks the sample inputs from the identity precompile. func BenchmarkPrecompiledIdentity(bench *testing.B) { t := precompiledTest{ Input: "38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e000000000000000000000000000000000000000000000000000000000000001b38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e789d1dd423d25f0772d2748d60f7e4b81bb14d086eba8e8e8efb6dcff8a4ae02", diff --git a/core/vm/eips.go b/core/vm/eips.go index 35f0a3f7c2..9f06b2818f 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -85,7 +85,7 @@ func enable1884(jt *JumpTable) { } func opSelfBalance(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { - balance, _ := uint256.FromBig(interpreter.evm.StateDB.GetBalance(scope.Contract.Address())) + balance := interpreter.evm.StateDB.GetBalance(scope.Contract.Address()) scope.Stack.push(balance) return nil, nil } diff --git a/core/vm/evm.go b/core/vm/evm.go index 088b18aaa4..16cc854908 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -29,9 +29,9 @@ import ( type ( // CanTransferFunc is the signature of a transfer guard function - CanTransferFunc func(StateDB, common.Address, *big.Int) bool + CanTransferFunc func(StateDB, common.Address, *uint256.Int) bool // TransferFunc is the signature of a transfer function - TransferFunc func(StateDB, common.Address, common.Address, *big.Int) + TransferFunc func(StateDB, common.Address, common.Address, *uint256.Int) // GetHashFunc returns the n'th block hash in the blockchain // and is used by the BLOCKHASH EVM op code. GetHashFunc func(uint64) common.Hash @@ -176,13 +176,13 @@ func (evm *EVM) Interpreter() *EVMInterpreter { // parameters. It also handles any necessary value transfer required and takes // the necessary steps to create accounts and reverses the state in case of an // execution error or failed value transfer. -func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas uint64, value *big.Int) (ret []byte, leftOverGas uint64, err error) { +func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas uint64, value *uint256.Int) (ret []byte, leftOverGas uint64, err error) { // Fail if we're trying to execute above the call depth limit if evm.depth > int(params.CallCreateDepth) { return nil, gas, ErrDepth } // Fail if we're trying to transfer more than the available balance - if value.Sign() != 0 && !evm.Context.CanTransfer(evm.StateDB, caller.Address(), value) { + if !value.IsZero() && !evm.Context.CanTransfer(evm.StateDB, caller.Address(), value) { return nil, gas, ErrInsufficientBalance } snapshot := evm.StateDB.Snapshot() @@ -190,14 +190,14 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas debug := evm.Config.Tracer != nil if !evm.StateDB.Exist(addr) { - if !isPrecompile && evm.chainRules.IsEIP158 && value.Sign() == 0 { + if !isPrecompile && evm.chainRules.IsEIP158 && value.IsZero() { // Calling a non existing account, don't do anything, but ping the tracer if debug { if evm.depth == 0 { - evm.Config.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value) + evm.Config.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value.ToBig()) evm.Config.Tracer.CaptureEnd(ret, 0, nil) } else { - evm.Config.Tracer.CaptureEnter(CALL, caller.Address(), addr, input, gas, value) + evm.Config.Tracer.CaptureEnter(CALL, caller.Address(), addr, input, gas, value.ToBig()) evm.Config.Tracer.CaptureExit(ret, 0, nil) } } @@ -210,13 +210,13 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas // Capture the tracer start/end events in debug mode if debug { if evm.depth == 0 { - evm.Config.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value) + evm.Config.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value.ToBig()) defer func(startGas uint64) { // Lazy evaluation of the parameters evm.Config.Tracer.CaptureEnd(ret, startGas-gas, err) }(gas) } else { // Handle tracer events for entering and exiting a call frame - evm.Config.Tracer.CaptureEnter(CALL, caller.Address(), addr, input, gas, value) + evm.Config.Tracer.CaptureEnter(CALL, caller.Address(), addr, input, gas, value.ToBig()) defer func(startGas uint64) { evm.Config.Tracer.CaptureExit(ret, startGas-gas, err) }(gas) @@ -263,7 +263,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas // // CallCode differs from Call in the sense that it executes the given address' // code with the caller as context. -func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, gas uint64, value *big.Int) (ret []byte, leftOverGas uint64, err error) { +func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, gas uint64, value *uint256.Int) (ret []byte, leftOverGas uint64, err error) { // Fail if we're trying to execute above the call depth limit if evm.depth > int(params.CallCreateDepth) { return nil, gas, ErrDepth @@ -279,7 +279,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, // Invoke tracer hooks that signal entering/exiting a call frame if evm.Config.Tracer != nil { - evm.Config.Tracer.CaptureEnter(CALLCODE, caller.Address(), addr, input, gas, value) + evm.Config.Tracer.CaptureEnter(CALLCODE, caller.Address(), addr, input, gas, value.ToBig()) defer func(startGas uint64) { evm.Config.Tracer.CaptureExit(ret, startGas-gas, err) }(gas) @@ -324,7 +324,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by // that caller is something other than a Contract. parent := caller.(*Contract) // DELEGATECALL inherits value from parent call - evm.Config.Tracer.CaptureEnter(DELEGATECALL, caller.Address(), addr, input, gas, parent.value) + evm.Config.Tracer.CaptureEnter(DELEGATECALL, caller.Address(), addr, input, gas, parent.value.ToBig()) defer func(startGas uint64) { evm.Config.Tracer.CaptureExit(ret, startGas-gas, err) }(gas) @@ -370,7 +370,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // This doesn't matter on Mainnet, where all empties are gone at the time of Byzantium, // but is the correct thing to do and matters on other networks, in tests, and potential // future scenarios - evm.StateDB.AddBalance(addr, big0) + evm.StateDB.AddBalance(addr, new(uint256.Int)) // Invoke tracer hooks that signal entering/exiting a call frame if evm.Config.Tracer != nil { @@ -389,7 +389,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte addrCopy := addr // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. - contract := NewContract(caller, AccountRef(addrCopy), new(big.Int), gas) + contract := NewContract(caller, AccountRef(addrCopy), new(uint256.Int), gas) contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy)) // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally @@ -419,7 +419,7 @@ func (c *codeAndHash) Hash() common.Hash { } // create creates a new contract using code as deployment code. -func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *big.Int, address common.Address, typ OpCode) ([]byte, common.Address, uint64, error) { +func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *uint256.Int, address common.Address, typ OpCode) ([]byte, common.Address, uint64, error) { // Depth check execution. Fail if we're trying to execute above the // limit. if evm.depth > int(params.CallCreateDepth) { @@ -458,9 +458,9 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, if evm.Config.Tracer != nil { if evm.depth == 0 { - evm.Config.Tracer.CaptureStart(evm, caller.Address(), address, true, codeAndHash.code, gas, value) + evm.Config.Tracer.CaptureStart(evm, caller.Address(), address, true, codeAndHash.code, gas, value.ToBig()) } else { - evm.Config.Tracer.CaptureEnter(typ, caller.Address(), address, codeAndHash.code, gas, value) + evm.Config.Tracer.CaptureEnter(typ, caller.Address(), address, codeAndHash.code, gas, value.ToBig()) } } @@ -510,7 +510,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, } // Create creates a new contract using code as deployment code. -func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { +func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address())) return evm.create(caller, &codeAndHash{code: code}, gas, value, contractAddr, CREATE) } @@ -519,7 +519,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I // // The different between Create2 with Create is Create2 uses keccak256(0xff ++ msg.sender ++ salt ++ keccak256(init_code))[12:] // instead of the usual sender-and-nonce-hash as the address where the contract is initialized at. -func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment *big.Int, salt *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { +func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment *uint256.Int, salt *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { codeAndHash := &codeAndHash{code: code} contractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes()) return evm.create(caller, codeAndHash, gas, endowment, contractAddr, CREATE2) diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 4a5259a262..4a2545b6ed 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) func TestMemoryGasCost(t *testing.T) { @@ -91,12 +92,12 @@ func TestEIP2200(t *testing.T) { statedb.Finalise(true) // Push the state into the "original" slot vmctx := BlockContext{ - CanTransfer: func(StateDB, common.Address, *big.Int) bool { return true }, - Transfer: func(StateDB, common.Address, common.Address, *big.Int) {}, + CanTransfer: func(StateDB, common.Address, *uint256.Int) bool { return true }, + Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, } vmenv := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, Config{ExtraEips: []int{2200}}) - _, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, tt.gaspool, new(big.Int)) + _, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, tt.gaspool, new(uint256.Int)) if err != tt.failure { t.Errorf("test %d: failure mismatch: have %v, want %v", i, err, tt.failure) } @@ -141,8 +142,8 @@ func TestCreateGas(t *testing.T) { statedb.SetCode(address, hexutil.MustDecode(tt.code)) statedb.Finalise(true) vmctx := BlockContext{ - CanTransfer: func(StateDB, common.Address, *big.Int) bool { return true }, - Transfer: func(StateDB, common.Address, common.Address, *big.Int) {}, + CanTransfer: func(StateDB, common.Address, *uint256.Int) bool { return true }, + Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, BlockNumber: big.NewInt(0), } config := Config{} @@ -152,7 +153,7 @@ func TestCreateGas(t *testing.T) { vmenv := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, config) var startGas = uint64(testGas) - ret, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, startGas, new(big.Int)) + ret, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, startGas, new(uint256.Int)) if err != nil { return false } diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 56ff350201..023aa0af00 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -260,7 +260,7 @@ func opAddress(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([] func opBalance(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { slot := scope.Stack.peek() address := common.Address(slot.Bytes20()) - slot.SetFromBig(interpreter.evm.StateDB.GetBalance(address)) + slot.Set(interpreter.evm.StateDB.GetBalance(address)) return nil, nil } @@ -275,8 +275,7 @@ func opCaller(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]b } func opCallValue(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { - v, _ := uint256.FromBig(scope.Contract.value) - scope.Stack.push(v) + scope.Stack.push(scope.Contract.value) return nil, nil } @@ -348,9 +347,7 @@ func opExtCodeSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) } func opCodeSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { - l := new(uint256.Int) - l.SetUint64(uint64(len(scope.Contract.Code))) - scope.Stack.push(l) + scope.Stack.push(new(uint256.Int).SetUint64(uint64(len(scope.Contract.Code)))) return nil, nil } @@ -592,13 +589,8 @@ func opCreate(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]b stackvalue := size scope.Contract.UseGas(gas) - //TODO: use uint256.Int instead of converting with toBig() - var bigVal = big0 - if !value.IsZero() { - bigVal = value.ToBig() - } - res, addr, returnGas, suberr := interpreter.evm.Create(scope.Contract, input, gas, bigVal) + res, addr, returnGas, suberr := interpreter.evm.Create(scope.Contract, input, gas, &value) // Push item on the stack based on the returned error. If the ruleset is // homestead we must check for CodeStoreOutOfGasError (homestead only // rule) and treat as an error, if the ruleset is frontier we must @@ -637,13 +629,8 @@ func opCreate2(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([] scope.Contract.UseGas(gas) // reuse size int for stackvalue stackvalue := size - //TODO: use uint256.Int instead of converting with toBig() - bigEndowment := big0 - if !endowment.IsZero() { - bigEndowment = endowment.ToBig() - } res, addr, returnGas, suberr := interpreter.evm.Create2(scope.Contract, input, gas, - bigEndowment, &salt) + &endowment, &salt) // Push item on the stack based on the returned error. if suberr != nil { stackvalue.Clear() @@ -676,16 +663,10 @@ func opCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byt if interpreter.readOnly && !value.IsZero() { return nil, ErrWriteProtection } - var bigVal = big0 - //TODO: use uint256.Int instead of converting with toBig() - // By using big0 here, we save an alloc for the most common case (non-ether-transferring contract calls), - // but it would make more sense to extend the usage of uint256.Int if !value.IsZero() { gas += params.CallStipend - bigVal = value.ToBig() } - - ret, returnGas, err := interpreter.evm.Call(scope.Contract, toAddr, args, gas, bigVal) + ret, returnGas, err := interpreter.evm.Call(scope.Contract, toAddr, args, gas, &value) if err != nil { temp.Clear() @@ -714,14 +695,11 @@ func opCallCode(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ // Get arguments from the memory. args := scope.Memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) - //TODO: use uint256.Int instead of converting with toBig() - var bigVal = big0 if !value.IsZero() { gas += params.CallStipend - bigVal = value.ToBig() } - ret, returnGas, err := interpreter.evm.CallCode(scope.Contract, toAddr, args, gas, bigVal) + ret, returnGas, err := interpreter.evm.CallCode(scope.Contract, toAddr, args, gas, &value) if err != nil { temp.Clear() } else { @@ -825,7 +803,7 @@ func opSelfdestruct(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance) interpreter.evm.StateDB.SelfDestruct(scope.Contract.Address()) if tracer := interpreter.evm.Config.Tracer; tracer != nil { - tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance) + tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) tracer.CaptureExit([]byte{}, 0, nil) } return nil, errStopToken @@ -841,7 +819,7 @@ func opSelfdestruct6780(pc *uint64, interpreter *EVMInterpreter, scope *ScopeCon interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance) interpreter.evm.StateDB.Selfdestruct6780(scope.Contract.Address()) if tracer := interpreter.evm.Config.Tracer; tracer != nil { - tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance) + tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) tracer.CaptureExit([]byte{}, 0, nil) } return nil, errStopToken diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 807073336d..8653864d11 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -590,7 +590,7 @@ func TestOpTstore(t *testing.T) { caller = common.Address{} to = common.Address{1} contractRef = contractRef{caller} - contract = NewContract(contractRef, AccountRef(to), new(big.Int), 0) + contract = NewContract(contractRef, AccountRef(to), new(uint256.Int), 0) scopeContext = ScopeContext{mem, stack, contract} value = common.Hex2Bytes("abcdef00000000000000abba000000000deaf000000c0de00100000000133700") ) diff --git a/core/vm/interface.go b/core/vm/interface.go index 26814d3d2f..25bfa06720 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -22,15 +22,16 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) // StateDB is an EVM database for full state querying. type StateDB interface { CreateAccount(common.Address) - SubBalance(common.Address, *big.Int) - AddBalance(common.Address, *big.Int) - GetBalance(common.Address) *big.Int + SubBalance(common.Address, *uint256.Int) + AddBalance(common.Address, *uint256.Int) + GetBalance(common.Address) *uint256.Int GetNonce(common.Address) uint64 SetNonce(common.Address, uint64) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 28da2e80e6..1968289f4e 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -147,7 +147,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( debug = in.evm.Config.Tracer != nil ) // Don't move this deferred function, it's placed before the capturestate-deferred method, - // so that it get's executed _after_: the capturestate needs the stacks before + // so that it gets executed _after_: the capturestate needs the stacks before // they are returned to the pools defer func() { returnStack(stack) diff --git a/core/vm/interpreter_test.go b/core/vm/interpreter_test.go index 96e681fccd..ff4977d728 100644 --- a/core/vm/interpreter_test.go +++ b/core/vm/interpreter_test.go @@ -17,7 +17,6 @@ package vm import ( - "math/big" "testing" "time" @@ -27,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) var loopInterruptTests = []string{ @@ -39,7 +39,7 @@ var loopInterruptTests = []string{ func TestLoopInterrupt(t *testing.T) { address := common.BytesToAddress([]byte("contract")) vmctx := BlockContext{ - Transfer: func(StateDB, common.Address, common.Address, *big.Int) {}, + Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, } for i, tt := range loopInterruptTests { @@ -54,7 +54,7 @@ func TestLoopInterrupt(t *testing.T) { timeout := make(chan bool) go func(evm *EVM) { - _, _, err := evm.Call(AccountRef(common.Address{}), address, nil, math.MaxUint64, new(big.Int)) + _, _, err := evm.Call(AccountRef(common.Address{}), address, nil, math.MaxUint64, new(uint256.Int)) errChannel <- err }(evm) diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index fb87258326..65716f9442 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -122,7 +122,7 @@ func newLondonInstructionSet() JumpTable { // constantinople, istanbul, petersburg and berlin instructions. func newBerlinInstructionSet() JumpTable { instructionSet := newIstanbulInstructionSet() - enable2929(&instructionSet) // Access lists for trie accesses https://eips.ethereum.org/EIPS/eip-2929 + enable2929(&instructionSet) // Gas cost increases for state access opcodes https://eips.ethereum.org/EIPS/eip-2929 return validate(instructionSet) } diff --git a/core/vm/jump_table_test.go b/core/vm/jump_table_test.go index f67915fff3..02558035c0 100644 --- a/core/vm/jump_table_test.go +++ b/core/vm/jump_table_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" ) -// TestJumpTableCopy tests that deep copy is necessery to prevent modify shared jump table +// TestJumpTableCopy tests that deep copy is necessary to prevent modify shared jump table func TestJumpTableCopy(t *testing.T) { tbl := newMergeInstructionSet() require.Equal(t, uint64(0), tbl[SLOAD].constantGas) diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index abb0a20e24..46f2bb5d5f 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) // Config is a basic type specifying certain configuration flags for running @@ -135,7 +136,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { common.BytesToAddress([]byte("contract")), input, cfg.GasLimit, - cfg.Value, + uint256.MustFromBig(cfg.Value), ) return ret, cfg.State, err } @@ -164,7 +165,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { sender, input, cfg.GasLimit, - cfg.Value, + uint256.MustFromBig(cfg.Value), ) return code, address, leftOverGas, err } @@ -194,7 +195,7 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, er address, input, cfg.GasLimit, - cfg.Value, + uint256.MustFromBig(cfg.Value), ) return ret, leftOverGas, err } diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index e71760bb23..b9e3c8ed66 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -38,6 +38,7 @@ import ( // force-load js tracers to trigger registration _ "github.com/ethereum/go-ethereum/eth/tracers/js" + "github.com/holiman/uint256" ) func TestDefaults(t *testing.T) { @@ -362,12 +363,12 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode //cfg.State.CreateAccount(cfg.Origin) // set the receiver's (the executing contract) code for execution. cfg.State.SetCode(destination, code) - vmenv.Call(sender, destination, nil, gas, cfg.Value) + vmenv.Call(sender, destination, nil, gas, uint256.MustFromBig(cfg.Value)) b.Run(name, func(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - vmenv.Call(sender, destination, nil, gas, cfg.Value) + vmenv.Call(sender, destination, nil, gas, uint256.MustFromBig(cfg.Value)) } }) } diff --git a/crypto/bls12381/g2.go b/crypto/bls12381/g2.go index e5fe75af20..b942bf94fd 100644 --- a/crypto/bls12381/g2.go +++ b/crypto/bls12381/g2.go @@ -27,7 +27,7 @@ import ( // If z is equal to one the point is considered as in affine form. type PointG2 [3]fe2 -// Set copies valeus of one point to another. +// Set copies values of one point to another. func (p *PointG2) Set(p2 *PointG2) *PointG2 { p[0].set(&p2[0]) p[1].set(&p2[1]) diff --git a/crypto/bn256/google/bn256.go b/crypto/bn256/google/bn256.go index 0a9d5cd35d..93953e23a9 100644 --- a/crypto/bn256/google/bn256.go +++ b/crypto/bn256/google/bn256.go @@ -166,7 +166,7 @@ type G2 struct { p *twistPoint } -// RandomG1 returns x and gâ‚‚Ë£ where x is a random, non-zero number read from r. +// RandomG2 returns x and gâ‚‚Ë£ where x is a random, non-zero number read from r. func RandomG2(r io.Reader) (*big.Int, *G2, error) { var k *big.Int var err error diff --git a/crypto/kzg4844/kzg4844.go b/crypto/kzg4844/kzg4844.go index 4561ef9de9..52124df674 100644 --- a/crypto/kzg4844/kzg4844.go +++ b/crypto/kzg4844/kzg4844.go @@ -21,21 +21,60 @@ import ( "embed" "errors" "hash" + "reflect" "sync/atomic" + + "github.com/ethereum/go-ethereum/common/hexutil" ) //go:embed trusted_setup.json var content embed.FS +var ( + blobT = reflect.TypeOf(Blob{}) + commitmentT = reflect.TypeOf(Commitment{}) + proofT = reflect.TypeOf(Proof{}) +) + // Blob represents a 4844 data blob. type Blob [131072]byte +// UnmarshalJSON parses a blob in hex syntax. +func (b *Blob) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(blobT, input, b[:]) +} + +// MarshalText returns the hex representation of b. +func (b Blob) MarshalText() ([]byte, error) { + return hexutil.Bytes(b[:]).MarshalText() +} + // Commitment is a serialized commitment to a polynomial. type Commitment [48]byte +// UnmarshalJSON parses a commitment in hex syntax. +func (c *Commitment) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(commitmentT, input, c[:]) +} + +// MarshalText returns the hex representation of c. +func (c Commitment) MarshalText() ([]byte, error) { + return hexutil.Bytes(c[:]).MarshalText() +} + // Proof is a serialized commitment to the quotient polynomial. type Proof [48]byte +// UnmarshalJSON parses a proof in hex syntax. +func (p *Proof) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(proofT, input, p[:]) +} + +// MarshalText returns the hex representation of p. +func (p Proof) MarshalText() ([]byte, error) { + return hexutil.Bytes(p[:]).MarshalText() +} + // Point is a BLS field element. type Point [32]byte diff --git a/docs/postmortems/2021-08-22-split-postmortem.md b/docs/postmortems/2021-08-22-split-postmortem.md index 962aa51f64..0986f00b65 100644 --- a/docs/postmortems/2021-08-22-split-postmortem.md +++ b/docs/postmortems/2021-08-22-split-postmortem.md @@ -87,7 +87,7 @@ The blocks on the 'bad' chain were investigated, and Tim Beiko reached out to th ### Disclosure decision -The geth-team have an official policy regarding [vulnerability disclosure](https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities). +The geth-team have an official policy regarding [vulnerability disclosure](https://geth.ethereum.org/docs/developers/geth-developer/disclosures). > The primary goal for the Geth team is the health of the Ethereum network as a whole, and the decision whether or not to publish details about a serious vulnerability boils down to minimizing the risk and/or impact of discovery and exploitation. diff --git a/eth/api_backend.go b/eth/api_backend.go index bc8398d217..0edcce5c87 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -308,9 +308,25 @@ func (b *EthAPIBackend) GetPoolTransaction(hash common.Hash) *types.Transaction return b.eth.txPool.Get(hash) } -func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { - tx, blockHash, blockNumber, index := rawdb.ReadTransaction(b.eth.ChainDb(), txHash) - return tx, blockHash, blockNumber, index, nil +// GetTransaction retrieves the lookup along with the transaction itself associate +// with the given transaction hash. +// +// An error will be returned if the transaction is not found, and background +// indexing for transactions is still in progress. The error is used to indicate the +// scenario explicitly that the transaction might be reachable shortly. +// +// A null will be returned in the transaction is not found and background transaction +// indexing is already finished. The transaction is not existent from the perspective +// of node. +func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) { + lookup, tx, err := b.eth.blockchain.GetTransactionLookup(txHash) + if err != nil { + return false, nil, common.Hash{}, 0, 0, err + } + if lookup == nil || tx == nil { + return false, nil, common.Hash{}, 0, 0, nil + } + return true, tx, lookup.BlockHash, lookup.BlockIndex, lookup.Index, nil } func (b *EthAPIBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { @@ -338,7 +354,12 @@ func (b *EthAPIBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.S } func (b *EthAPIBackend) SyncProgress() ethereum.SyncProgress { - return b.eth.Downloader().Progress() + prog := b.eth.Downloader().Progress() + if txProg, err := b.eth.blockchain.TxIndexProgress(); err == nil { + prog.TxIndexFinishedBlocks = txProg.Indexed + prog.TxIndexRemainingBlocks = txProg.Remaining + } + return prog } func (b *EthAPIBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { diff --git a/eth/api_debug_test.go b/eth/api_debug_test.go index 184b90dd09..4641735cce 100644 --- a/eth/api_debug_test.go +++ b/eth/api_debug_test.go @@ -19,7 +19,6 @@ package eth import ( "bytes" "fmt" - "math/big" "reflect" "strings" "testing" @@ -31,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" "golang.org/x/exp/slices" ) @@ -73,7 +73,7 @@ func TestAccountRange(t *testing.T) { hash := common.HexToHash(fmt.Sprintf("%x", i)) addr := common.BytesToAddress(crypto.Keccak256Hash(hash.Bytes()).Bytes()) addrs[i] = addr - sdb.SetBalance(addrs[i], big.NewInt(1)) + sdb.SetBalance(addrs[i], uint256.NewInt(1)) if _, ok := m[addr]; ok { t.Fatalf("bad") } else { diff --git a/eth/api_miner.go b/eth/api_miner.go index 477531d494..2fe296548a 100644 --- a/eth/api_miner.go +++ b/eth/api_miner.go @@ -64,6 +64,7 @@ func (api *MinerAPI) SetGasPrice(gasPrice hexutil.Big) bool { api.e.lock.Unlock() api.e.txPool.SetGasTip((*big.Int)(&gasPrice)) + api.e.Miner().SetGasTip((*big.Int)(&gasPrice)) return true } diff --git a/eth/backend.go b/eth/backend.go index 774ffaf248..aff23a910b 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -322,7 +322,7 @@ func (s *Ethereum) APIs() []rpc.API { Service: NewMinerAPI(s), }, { Namespace: "eth", - Service: downloader.NewDownloaderAPI(s.handler.downloader, s.eventMux), + Service: downloader.NewDownloaderAPI(s.handler.downloader, s.blockchain, s.eventMux), }, { Namespace: "admin", Service: NewAdminAPI(s), diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 37b0248f28..c48a7d0e49 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -20,7 +20,6 @@ package catalyst import ( "errors" "fmt" - "math/big" "sync" "time" @@ -34,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params/forks" "github.com/ethereum/go-ethereum/rpc" ) @@ -173,61 +173,65 @@ func newConsensusAPIWithoutHeartbeat(eth *eth.Ethereum) *ConsensusAPI { // and return its payloadID. func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if payloadAttributes != nil { - if payloadAttributes.Withdrawals != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals not supported in V1")) + if payloadAttributes.Withdrawals != nil || payloadAttributes.BeaconRoot != nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals and beacon root not supported in V1")) } if api.eth.BlockChain().Config().IsShanghai(api.eth.BlockChain().Config().LondonBlock, payloadAttributes.Timestamp) { return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("forkChoiceUpdateV1 called post-shanghai")) } } - return api.forkchoiceUpdated(update, payloadAttributes) + return api.forkchoiceUpdated(update, payloadAttributes, engine.PayloadV1, false) } -// ForkchoiceUpdatedV2 is equivalent to V1 with the addition of withdrawals in the payload attributes. -func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { - if payloadAttributes != nil { - if err := api.verifyPayloadAttributes(payloadAttributes); err != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(err) +// ForkchoiceUpdatedV2 is equivalent to V1 with the addition of withdrawals in the payload +// attributes. It supports both PayloadAttributesV1 and PayloadAttributesV2. +func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { + if params != nil { + switch api.eth.BlockChain().Config().LatestFork(params.Timestamp) { + case forks.Paris: + if params.Withdrawals != nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals before shanghai")) + } + case forks.Shanghai: + if params.Withdrawals == nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals")) + } + default: + return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV2 must only be called with paris and shanghai payloads")) + } + if params.BeaconRoot != nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("unexpected beacon root")) } } - return api.forkchoiceUpdated(update, payloadAttributes) + return api.forkchoiceUpdated(update, params, engine.PayloadV2, false) } -// ForkchoiceUpdatedV3 is equivalent to V2 with the addition of parent beacon block root in the payload attributes. -func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { - if payloadAttributes != nil { - if err := api.verifyPayloadAttributes(payloadAttributes); err != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(err) +// ForkchoiceUpdatedV3 is equivalent to V2 with the addition of parent beacon block root +// in the payload attributes. It supports only PayloadAttributesV3. +func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { + if params != nil { + // TODO(matt): according to https://github.com/ethereum/execution-apis/pull/498, + // payload attributes that are invalid should return error + // engine.InvalidPayloadAttributes. Once hive updates this, we should update + // on our end. + if params.Withdrawals == nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals")) + } + if params.BeaconRoot == nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing beacon root")) + } + if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun { + return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV3 must only be called for cancun payloads")) } } - return api.forkchoiceUpdated(update, payloadAttributes) + // TODO(matt): the spec requires that fcu is applied when called on a valid + // hash, even if params are wrong. To do this we need to split up + // forkchoiceUpdate into a function that only updates the head and then a + // function that kicks off block construction. + return api.forkchoiceUpdated(update, params, engine.PayloadV3, false) } -func (api *ConsensusAPI) verifyPayloadAttributes(attr *engine.PayloadAttributes) error { - c := api.eth.BlockChain().Config() - - // Verify withdrawals attribute for Shanghai. - if err := checkAttribute(c.IsShanghai, attr.Withdrawals != nil, c.LondonBlock, attr.Timestamp); err != nil { - return fmt.Errorf("invalid withdrawals: %w", err) - } - // Verify beacon root attribute for Cancun. - if err := checkAttribute(c.IsCancun, attr.BeaconRoot != nil, c.LondonBlock, attr.Timestamp); err != nil { - return fmt.Errorf("invalid parent beacon block root: %w", err) - } - return nil -} - -func checkAttribute(active func(*big.Int, uint64) bool, exists bool, block *big.Int, time uint64) error { - if active(block, time) && !exists { - return errors.New("fork active, missing expected attribute") - } - if !active(block, time) && exists { - return errors.New("fork inactive, unexpected attribute set") - } - return nil -} - -func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes, payloadVersion engine.PayloadVersion, simulatorMode bool) (engine.ForkChoiceResponse, error) { api.forkchoiceLock.Lock() defer api.forkchoiceLock.Unlock() @@ -334,7 +338,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl if merger := api.eth.Merger(); !merger.PoSFinalized() { merger.FinalizePoS() } - // If the finalized block is not in our canonical tree, somethings wrong + // If the finalized block is not in our canonical tree, something is wrong finalBlock := api.eth.BlockChain().GetBlockByHash(update.FinalizedBlockHash) if finalBlock == nil { log.Warn("Final block not available in database", "hash", update.FinalizedBlockHash) @@ -346,7 +350,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl // Set the finalized block api.eth.BlockChain().SetFinalized(finalBlock.Header()) } - // Check if the safe block hash is in our canonical tree, if not somethings wrong + // Check if the safe block hash is in our canonical tree, if not something is wrong if update.SafeBlockHash != (common.Hash{}) { safeBlock := api.eth.BlockChain().GetBlockByHash(update.SafeBlockHash) if safeBlock == nil { @@ -371,6 +375,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl Random: payloadAttributes.Random, Withdrawals: payloadAttributes.Withdrawals, BeaconRoot: payloadAttributes.BeaconRoot, + Version: payloadVersion, } id := args.Id() // If we already are busy generating this work, then we do not need @@ -378,6 +383,19 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl if api.localBlocks.has(id) { return valid(&id), nil } + // If the beacon chain is ran by a simulator, then transaction insertion, + // block insertion and block production will happen without any timing + // delay between them. This will cause flaky simulator executions due to + // the transaction pool running its internal reset operation on a back- + // ground thread. To avoid the racey behavior - in simulator mode - the + // pool will be explicitly blocked on its reset before continuing to the + // block production below. + if simulatorMode { + if err := api.eth.TxPool().Sync(); err != nil { + log.Error("Failed to sync transaction pool", "err", err) + return valid(nil), engine.InvalidPayloadAttributes.With(err) + } + } payload, err := api.eth.Miner().BuildPayload(args) if err != nil { log.Error("Failed to build payload", "err", err) @@ -421,6 +439,9 @@ func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config engine.Transit // GetPayloadV1 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV1(payloadID engine.PayloadID) (*engine.ExecutableData, error) { + if !payloadID.Is(engine.PayloadV1) { + return nil, engine.UnsupportedFork + } data, err := api.getPayload(payloadID, false) if err != nil { return nil, err @@ -430,11 +451,17 @@ func (api *ConsensusAPI) GetPayloadV1(payloadID engine.PayloadID) (*engine.Execu // GetPayloadV2 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV2(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { + if !payloadID.Is(engine.PayloadV1, engine.PayloadV2) { + return nil, engine.UnsupportedFork + } return api.getPayload(payloadID, false) } // GetPayloadV3 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV3(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { + if !payloadID.Is(engine.PayloadV3) { + return nil, engine.UnsupportedFork + } return api.getPayload(payloadID, false) } @@ -457,27 +484,39 @@ func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.Payl // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) { - if api.eth.BlockChain().Config().IsShanghai(new(big.Int).SetUint64(params.Number), params.Timestamp) { + if api.eth.BlockChain().Config().IsCancun(api.eth.BlockChain().Config().LondonBlock, params.Timestamp) { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use new payload v2 post-shanghai")) + } + if api.eth.BlockChain().Config().LatestFork(params.Timestamp) == forks.Shanghai { if params.Withdrawals == nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil withdrawals post-shanghai")) } - } else if params.Withdrawals != nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil withdrawals pre-shanghai")) + } else { + if params.Withdrawals != nil { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil withdrawals pre-shanghai")) + } } - if api.eth.BlockChain().Config().IsCancun(new(big.Int).SetUint64(params.Number), params.Timestamp) { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("newPayloadV2 called post-cancun")) + if params.ExcessBlobGas != nil { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil excessBlobGas pre-cancun")) + } + if params.BlobGasUsed != nil { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil params.BlobGasUsed pre-cancun")) } return api.newPayload(params, nil, nil) } // NewPayloadV3 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { + if params.Withdrawals == nil { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil withdrawals post-shanghai")) + } if params.ExcessBlobGas == nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil excessBlobGas post-cancun")) } if params.BlobGasUsed == nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil params.BlobGasUsed post-cancun")) } + if versionedHashes == nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil versionedHashes post-cancun")) } @@ -485,10 +524,9 @@ func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHas return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil parentBeaconBlockRoot post-cancun")) } - if !api.eth.BlockChain().Config().IsCancun(new(big.Int).SetUint64(params.Number), params.Timestamp) { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.UnsupportedFork.With(errors.New("newPayloadV3 called pre-cancun")) + if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.UnsupportedFork.With(errors.New("newPayloadV3 must only be called for cancun payloads")) } - return api.newPayload(params, versionedHashes, beaconRoot) } diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index c875c485dd..f1d48d0dea 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -210,6 +210,7 @@ func TestEth2PrepareAndGetPayload(t *testing.T) { FeeRecipient: blockParams.SuggestedFeeRecipient, Random: blockParams.Random, BeaconRoot: blockParams.BeaconRoot, + Version: engine.PayloadV1, }).Id() execData, err := api.GetPayloadV1(payloadID) if err != nil { @@ -1076,6 +1077,7 @@ func TestWithdrawals(t *testing.T) { Random: blockParams.Random, Withdrawals: blockParams.Withdrawals, BeaconRoot: blockParams.BeaconRoot, + Version: engine.PayloadV2, }).Id() execData, err := api.GetPayloadV2(payloadID) if err != nil { @@ -1124,6 +1126,7 @@ func TestWithdrawals(t *testing.T) { Random: blockParams.Random, Withdrawals: blockParams.Withdrawals, BeaconRoot: blockParams.BeaconRoot, + Version: engine.PayloadV2, }).Id() execData, err = api.GetPayloadV2(payloadID) if err != nil { @@ -1237,7 +1240,18 @@ func TestNilWithdrawals(t *testing.T) { } for _, test := range tests { - _, err := api.ForkchoiceUpdatedV2(fcState, &test.blockParams) + var ( + err error + payloadVersion engine.PayloadVersion + shanghai = genesis.Config.IsShanghai(genesis.Config.LondonBlock, test.blockParams.Timestamp) + ) + if !shanghai { + payloadVersion = engine.PayloadV1 + _, err = api.ForkchoiceUpdatedV1(fcState, &test.blockParams) + } else { + payloadVersion = engine.PayloadV2 + _, err = api.ForkchoiceUpdatedV2(fcState, &test.blockParams) + } if test.wantErr { if err == nil { t.Fatal("wanted error on fcuv2 with invalid withdrawals") @@ -1254,14 +1268,20 @@ func TestNilWithdrawals(t *testing.T) { Timestamp: test.blockParams.Timestamp, FeeRecipient: test.blockParams.SuggestedFeeRecipient, Random: test.blockParams.Random, - BeaconRoot: test.blockParams.BeaconRoot, + Version: payloadVersion, }).Id() execData, err := api.GetPayloadV2(payloadID) if err != nil { t.Fatalf("error getting payload, err=%v", err) } - if status, err := api.NewPayloadV2(*execData.ExecutionPayload); err != nil { - t.Fatalf("error validating payload: %v", err) + var status engine.PayloadStatusV1 + if !shanghai { + status, err = api.NewPayloadV1(*execData.ExecutionPayload) + } else { + status, err = api.NewPayloadV2(*execData.ExecutionPayload) + } + if err != nil { + t.Fatalf("error validating payload: %v", err.(*engine.EngineAPIError).ErrorData()) } else if status.Status != engine.VALID { t.Fatalf("invalid payload") } @@ -1587,7 +1607,7 @@ func TestParentBeaconBlockRoot(t *testing.T) { fcState := engine.ForkchoiceStateV1{ HeadBlockHash: parent.Hash(), } - resp, err := api.ForkchoiceUpdatedV2(fcState, &blockParams) + resp, err := api.ForkchoiceUpdatedV3(fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err.(*engine.EngineAPIError).ErrorData()) } @@ -1603,6 +1623,7 @@ func TestParentBeaconBlockRoot(t *testing.T) { Random: blockParams.Random, Withdrawals: blockParams.Withdrawals, BeaconRoot: blockParams.BeaconRoot, + Version: engine.PayloadV3, }).Id() execData, err := api.GetPayloadV3(payloadID) if err != nil { diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index 3c081074cc..5ad50f14c1 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -155,12 +155,12 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u var random [32]byte rand.Read(random[:]) - fcResponse, err := c.engineAPI.ForkchoiceUpdatedV2(c.curForkchoiceState, &engine.PayloadAttributes{ + fcResponse, err := c.engineAPI.forkchoiceUpdated(c.curForkchoiceState, &engine.PayloadAttributes{ Timestamp: timestamp, SuggestedFeeRecipient: feeRecipient, Withdrawals: withdrawals, Random: random, - }) + }, engine.PayloadV2, true) if err != nil { return err } diff --git a/eth/downloader/api.go b/eth/downloader/api.go index 606c6d4e7e..f09122904c 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -19,16 +19,20 @@ package downloader import ( "context" "sync" + "time" "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" ) -// DownloaderAPI provides an API which gives information about the current synchronisation status. -// It offers only methods that operates on data that can be available to anyone without security risks. +// DownloaderAPI provides an API which gives information about the current +// synchronisation status. It offers only methods that operates on data that +// can be available to anyone without security risks. type DownloaderAPI struct { d *Downloader + chain *core.BlockChain mux *event.TypeMux installSyncSubscription chan chan interface{} uninstallSyncSubscription chan *uninstallSyncSubscriptionRequest @@ -38,31 +42,57 @@ type DownloaderAPI struct { // listens for events from the downloader through the global event mux. In case it receives one of // these events it broadcasts it to all syncing subscriptions that are installed through the // installSyncSubscription channel. -func NewDownloaderAPI(d *Downloader, m *event.TypeMux) *DownloaderAPI { +func NewDownloaderAPI(d *Downloader, chain *core.BlockChain, m *event.TypeMux) *DownloaderAPI { api := &DownloaderAPI{ d: d, + chain: chain, mux: m, installSyncSubscription: make(chan chan interface{}), uninstallSyncSubscription: make(chan *uninstallSyncSubscriptionRequest), } - go api.eventLoop() - return api } -// eventLoop runs a loop until the event mux closes. It will install and uninstall new -// sync subscriptions and broadcasts sync status updates to the installed sync subscriptions. +// eventLoop runs a loop until the event mux closes. It will install and uninstall +// new sync subscriptions and broadcasts sync status updates to the installed sync +// subscriptions. +// +// The sync status pushed to subscriptions can be a stream like: +// >>> {Syncing: true, Progress: {...}} +// >>> {false} +// +// If the node is already synced up, then only a single event subscribers will +// receive is {false}. func (api *DownloaderAPI) eventLoop() { var ( - sub = api.mux.Subscribe(StartEvent{}, DoneEvent{}, FailedEvent{}) + sub = api.mux.Subscribe(StartEvent{}) syncSubscriptions = make(map[chan interface{}]struct{}) + checkInterval = time.Second * 60 + checkTimer = time.NewTimer(checkInterval) + + // status flags + started bool + done bool + + getProgress = func() ethereum.SyncProgress { + prog := api.d.Progress() + if txProg, err := api.chain.TxIndexProgress(); err == nil { + prog.TxIndexFinishedBlocks = txProg.Indexed + prog.TxIndexRemainingBlocks = txProg.Remaining + } + return prog + } ) + defer checkTimer.Stop() for { select { case i := <-api.installSyncSubscription: syncSubscriptions[i] = struct{}{} + if done { + i <- false + } case u := <-api.uninstallSyncSubscription: delete(syncSubscriptions, u.c) close(u.uninstalled) @@ -70,21 +100,31 @@ func (api *DownloaderAPI) eventLoop() { if event == nil { return } - - var notification interface{} switch event.Data.(type) { case StartEvent: - notification = &SyncingResult{ + started = true + } + case <-checkTimer.C: + if !started { + checkTimer.Reset(checkInterval) + continue + } + prog := getProgress() + if !prog.Done() { + notification := &SyncingResult{ Syncing: true, - Status: api.d.Progress(), + Status: prog, } - case DoneEvent, FailedEvent: - notification = false + for c := range syncSubscriptions { + c <- notification + } + checkTimer.Reset(checkInterval) + continue } - // broadcast for c := range syncSubscriptions { - c <- notification + c <- false } + done = true } } } diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index df8af68bc7..d3f75c8527 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -50,7 +50,8 @@ func newBeaconBackfiller(dl *Downloader, success func()) backfiller { } // suspend cancels any background downloader threads and returns the last header -// that has been successfully backfilled. +// that has been successfully backfilled (potentially in a previous run), or the +// genesis. func (b *beaconBackfiller) suspend() *types.Header { // If no filling is running, don't waste cycles b.lock.Lock() diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index f1cfa92d5d..8d449246a6 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -611,6 +611,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td, ttd * if err := d.lightchain.SetHead(origin); err != nil { return err } + log.Info("Truncated excess ancient chain segment", "oldhead", frozen-1, "newhead", origin) } } // Initiate the sync using a concurrent header and content retrieval algorithm diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index e4875b959a..99a003e59f 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -440,9 +440,6 @@ func assertOwnChain(t *testing.T, tester *downloadTester, length int) { func TestCanonicalSynchronisation68Full(t *testing.T) { testCanonSync(t, eth.ETH68, FullSync) } func TestCanonicalSynchronisation68Snap(t *testing.T) { testCanonSync(t, eth.ETH68, SnapSync) } func TestCanonicalSynchronisation68Light(t *testing.T) { testCanonSync(t, eth.ETH68, LightSync) } -func TestCanonicalSynchronisation67Full(t *testing.T) { testCanonSync(t, eth.ETH67, FullSync) } -func TestCanonicalSynchronisation67Snap(t *testing.T) { testCanonSync(t, eth.ETH67, SnapSync) } -func TestCanonicalSynchronisation67Light(t *testing.T) { testCanonSync(t, eth.ETH67, LightSync) } func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -463,8 +460,6 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // until the cached blocks are retrieved. func TestThrottling68Full(t *testing.T) { testThrottling(t, eth.ETH68, FullSync) } func TestThrottling68Snap(t *testing.T) { testThrottling(t, eth.ETH68, SnapSync) } -func TestThrottling67Full(t *testing.T) { testThrottling(t, eth.ETH67, FullSync) } -func TestThrottling67Snap(t *testing.T) { testThrottling(t, eth.ETH67, SnapSync) } func testThrottling(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -546,9 +541,6 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { func TestForkedSync68Full(t *testing.T) { testForkedSync(t, eth.ETH68, FullSync) } func TestForkedSync68Snap(t *testing.T) { testForkedSync(t, eth.ETH68, SnapSync) } func TestForkedSync68Light(t *testing.T) { testForkedSync(t, eth.ETH68, LightSync) } -func TestForkedSync67Full(t *testing.T) { testForkedSync(t, eth.ETH67, FullSync) } -func TestForkedSync67Snap(t *testing.T) { testForkedSync(t, eth.ETH67, SnapSync) } -func TestForkedSync67Light(t *testing.T) { testForkedSync(t, eth.ETH67, LightSync) } func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -576,9 +568,6 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestHeavyForkedSync68Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH68, FullSync) } func TestHeavyForkedSync68Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH68, SnapSync) } func TestHeavyForkedSync68Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH68, LightSync) } -func TestHeavyForkedSync67Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, FullSync) } -func TestHeavyForkedSync67Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, SnapSync) } -func TestHeavyForkedSync67Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, LightSync) } func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -608,9 +597,6 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestBoundedForkedSync68Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH68, FullSync) } func TestBoundedForkedSync68Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH68, SnapSync) } func TestBoundedForkedSync68Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH68, LightSync) } -func TestBoundedForkedSync67Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, FullSync) } -func TestBoundedForkedSync67Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, SnapSync) } -func TestBoundedForkedSync67Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, LightSync) } func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -645,15 +631,6 @@ func TestBoundedHeavyForkedSync68Snap(t *testing.T) { func TestBoundedHeavyForkedSync68Light(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH68, LightSync) } -func TestBoundedHeavyForkedSync67Full(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH67, FullSync) -} -func TestBoundedHeavyForkedSync67Snap(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH67, SnapSync) -} -func TestBoundedHeavyForkedSync67Light(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH67, LightSync) -} func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -681,9 +658,6 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestCancel68Full(t *testing.T) { testCancel(t, eth.ETH68, FullSync) } func TestCancel68Snap(t *testing.T) { testCancel(t, eth.ETH68, SnapSync) } func TestCancel68Light(t *testing.T) { testCancel(t, eth.ETH68, LightSync) } -func TestCancel67Full(t *testing.T) { testCancel(t, eth.ETH67, FullSync) } -func TestCancel67Snap(t *testing.T) { testCancel(t, eth.ETH67, SnapSync) } -func TestCancel67Light(t *testing.T) { testCancel(t, eth.ETH67, LightSync) } func testCancel(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -711,9 +685,6 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { func TestMultiSynchronisation68Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH68, FullSync) } func TestMultiSynchronisation68Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH68, SnapSync) } func TestMultiSynchronisation68Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH68, LightSync) } -func TestMultiSynchronisation67Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, FullSync) } -func TestMultiSynchronisation67Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, SnapSync) } -func TestMultiSynchronisation67Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, LightSync) } func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -738,9 +709,6 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { func TestMultiProtoSynchronisation68Full(t *testing.T) { testMultiProtoSync(t, eth.ETH68, FullSync) } func TestMultiProtoSynchronisation68Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH68, SnapSync) } func TestMultiProtoSynchronisation68Light(t *testing.T) { testMultiProtoSync(t, eth.ETH68, LightSync) } -func TestMultiProtoSynchronisation67Full(t *testing.T) { testMultiProtoSync(t, eth.ETH67, FullSync) } -func TestMultiProtoSynchronisation67Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH67, SnapSync) } -func TestMultiProtoSynchronisation67Light(t *testing.T) { testMultiProtoSync(t, eth.ETH67, LightSync) } func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -751,7 +719,6 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Create peers of every type tester.newPeer("peer 68", eth.ETH68, chain.blocks[1:]) - tester.newPeer("peer 67", eth.ETH67, chain.blocks[1:]) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -760,7 +727,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, len(chain.blocks)) // Check that no peers have been dropped off - for _, version := range []int{68, 67} { + for _, version := range []int{68} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -773,9 +740,6 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { func TestEmptyShortCircuit68Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, FullSync) } func TestEmptyShortCircuit68Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, SnapSync) } func TestEmptyShortCircuit68Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, LightSync) } -func TestEmptyShortCircuit67Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, FullSync) } -func TestEmptyShortCircuit67Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, SnapSync) } -func TestEmptyShortCircuit67Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, LightSync) } func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -824,9 +788,6 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { func TestMissingHeaderAttack68Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH68, FullSync) } func TestMissingHeaderAttack68Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH68, SnapSync) } func TestMissingHeaderAttack68Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH68, LightSync) } -func TestMissingHeaderAttack67Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, FullSync) } -func TestMissingHeaderAttack67Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, SnapSync) } -func TestMissingHeaderAttack67Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, LightSync) } func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -853,9 +814,6 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { func TestShiftedHeaderAttack68Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH68, FullSync) } func TestShiftedHeaderAttack68Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH68, SnapSync) } func TestShiftedHeaderAttack68Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH68, LightSync) } -func TestShiftedHeaderAttack67Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, FullSync) } -func TestShiftedHeaderAttack67Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, SnapSync) } -func TestShiftedHeaderAttack67Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, LightSync) } func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -889,15 +847,6 @@ func TestHighTDStarvationAttack68Snap(t *testing.T) { func TestHighTDStarvationAttack68Light(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH68, LightSync) } -func TestHighTDStarvationAttack67Full(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH67, FullSync) -} -func TestHighTDStarvationAttack67Snap(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH67, SnapSync) -} -func TestHighTDStarvationAttack67Light(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH67, LightSync) -} func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -912,7 +861,6 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that misbehaving peers are disconnected, whilst behaving ones are not. func TestBlockHeaderAttackerDropping68(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH68) } -func TestBlockHeaderAttackerDropping67(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH67) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Define the disconnection requirement for individual hash fetch errors @@ -963,9 +911,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { func TestSyncProgress68Full(t *testing.T) { testSyncProgress(t, eth.ETH68, FullSync) } func TestSyncProgress68Snap(t *testing.T) { testSyncProgress(t, eth.ETH68, SnapSync) } func TestSyncProgress68Light(t *testing.T) { testSyncProgress(t, eth.ETH68, LightSync) } -func TestSyncProgress67Full(t *testing.T) { testSyncProgress(t, eth.ETH67, FullSync) } -func TestSyncProgress67Snap(t *testing.T) { testSyncProgress(t, eth.ETH67, SnapSync) } -func TestSyncProgress67Light(t *testing.T) { testSyncProgress(t, eth.ETH67, LightSync) } func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1043,9 +988,6 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync func TestForkedSyncProgress68Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH68, FullSync) } func TestForkedSyncProgress68Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH68, SnapSync) } func TestForkedSyncProgress68Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH68, LightSync) } -func TestForkedSyncProgress67Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, FullSync) } -func TestForkedSyncProgress67Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, SnapSync) } -func TestForkedSyncProgress67Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, LightSync) } func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1117,9 +1059,6 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFailedSyncProgress68Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH68, FullSync) } func TestFailedSyncProgress68Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH68, SnapSync) } func TestFailedSyncProgress68Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH68, LightSync) } -func TestFailedSyncProgress67Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, FullSync) } -func TestFailedSyncProgress67Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, SnapSync) } -func TestFailedSyncProgress67Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, LightSync) } func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1186,9 +1125,6 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFakedSyncProgress68Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH68, FullSync) } func TestFakedSyncProgress68Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH68, SnapSync) } func TestFakedSyncProgress68Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH68, LightSync) } -func TestFakedSyncProgress67Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, FullSync) } -func TestFakedSyncProgress67Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, SnapSync) } -func TestFakedSyncProgress67Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, LightSync) } func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1332,8 +1268,6 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // being fast-synced from, avoiding potential cheap eclipse attacks. func TestBeaconSync68Full(t *testing.T) { testBeaconSync(t, eth.ETH68, FullSync) } func TestBeaconSync68Snap(t *testing.T) { testBeaconSync(t, eth.ETH68, SnapSync) } -func TestBeaconSync67Full(t *testing.T) { testBeaconSync(t, eth.ETH67, FullSync) } -func TestBeaconSync67Snap(t *testing.T) { testBeaconSync(t, eth.ETH67, SnapSync) } func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) { //log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index f40ca24d99..873ee950b6 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -161,7 +161,7 @@ type backfiller interface { // on initial startup. // // The method should return the last block header that has been successfully - // backfilled, or nil if the backfiller was not resumed. + // backfilled (in the current or a previous run), falling back to the genesis. suspend() *types.Header // resume requests the backfiller to start running fill or snap sync based on @@ -382,14 +382,17 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { done := make(chan struct{}) go func() { defer close(done) - if filled := s.filler.suspend(); filled != nil { - // If something was filled, try to delete stale sync helpers. If - // unsuccessful, warn the user, but not much else we can do (it's - // a programming error, just let users report an issue and don't - // choke in the meantime). - if err := s.cleanStales(filled); err != nil { - log.Error("Failed to clean stale beacon headers", "err", err) - } + filled := s.filler.suspend() + if filled == nil { + log.Error("Latest filled block is not available") + return + } + // If something was filled, try to delete stale sync helpers. If + // unsuccessful, warn the user, but not much else we can do (it's + // a programming error, just let users report an issue and don't + // choke in the meantime). + if err := s.cleanStales(filled); err != nil { + log.Error("Failed to clean stale beacon headers", "err", err) } }() // Wait for the suspend to finish, consuming head events in the meantime @@ -1120,33 +1123,46 @@ func (s *skeleton) cleanStales(filled *types.Header) error { number := filled.Number.Uint64() log.Trace("Cleaning stale beacon headers", "filled", number, "hash", filled.Hash()) - // If the filled header is below the linked subchain, something's - // corrupted internally. Report and error and refuse to do anything. - if number < s.progress.Subchains[0].Tail { + // If the filled header is below the linked subchain, something's corrupted + // internally. Report and error and refuse to do anything. + if number+1 < s.progress.Subchains[0].Tail { return fmt.Errorf("filled header below beacon header tail: %d < %d", number, s.progress.Subchains[0].Tail) } - // Subchain seems trimmable, push the tail forward up to the last - // filled header and delete everything before it - if available. In - // case we filled past the head, recreate the subchain with a new - // head to keep it consistent with the data on disk. + // If nothing in subchain is filled, don't bother to do cleanup. + if number+1 == s.progress.Subchains[0].Tail { + return nil + } var ( - start = s.progress.Subchains[0].Tail // start deleting from the first known header - end = number // delete until the requested threshold + start uint64 + end uint64 batch = s.db.NewBatch() ) - s.progress.Subchains[0].Tail = number - s.progress.Subchains[0].Next = filled.ParentHash + if number < s.progress.Subchains[0].Head { + // The skeleton chain is partially consumed, set the new tail as filled+1. + tail := rawdb.ReadSkeletonHeader(s.db, number+1) + if tail.ParentHash != filled.Hash() { + return fmt.Errorf("filled header is discontinuous with subchain: %d %s, please file an issue", number, filled.Hash()) + } + start, end = s.progress.Subchains[0].Tail, number+1 // remove headers in [tail, filled] + s.progress.Subchains[0].Tail = tail.Number.Uint64() + s.progress.Subchains[0].Next = tail.ParentHash + } else { + // The skeleton chain is fully consumed, set both head and tail as filled. + start, end = s.progress.Subchains[0].Tail, filled.Number.Uint64() // remove headers in [tail, filled) + s.progress.Subchains[0].Tail = filled.Number.Uint64() + s.progress.Subchains[0].Next = filled.ParentHash - if s.progress.Subchains[0].Head < number { - // If more headers were filled than available, push the entire - // subchain forward to keep tracking the node's block imports - end = s.progress.Subchains[0].Head + 1 // delete the entire original range, including the head - s.progress.Subchains[0].Head = number // assign a new head (tail is already assigned to this) + // If more headers were filled than available, push the entire subchain + // forward to keep tracking the node's block imports. + if number > s.progress.Subchains[0].Head { + end = s.progress.Subchains[0].Head + 1 // delete the entire original range, including the head + s.progress.Subchains[0].Head = number // assign a new head (tail is already assigned to this) - // The entire original skeleton chain was deleted and a new one - // defined. Make sure the new single-header chain gets pushed to - // disk to keep internal state consistent. - rawdb.WriteSkeletonHeader(batch, filled) + // The entire original skeleton chain was deleted and a new one + // defined. Make sure the new single-header chain gets pushed to + // disk to keep internal state consistent. + rawdb.WriteSkeletonHeader(batch, filled) + } } // Execute the trimming and the potential rewiring of the progress s.saveSyncStatus(batch) diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index aceadd00d3..2b108dfe93 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -811,7 +811,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { // Create a peer set to feed headers through peerset := newPeerSet() for _, peer := range tt.peers { - peerset.Register(newPeerConnection(peer.id, eth.ETH67, peer, log.New("id", peer.id))) + peerset.Register(newPeerConnection(peer.id, eth.ETH68, peer, log.New("id", peer.id))) } // Create a peer dropper to track malicious peers dropped := make(map[string]int) @@ -913,7 +913,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { skeleton.Sync(tt.newHead, nil, true) } if tt.newPeer != nil { - if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH67, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil { + if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH68, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil { t.Errorf("test %d: failed to register new peer: %v", i, err) } } diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go index a36c670747..f07f98956e 100644 --- a/eth/gasestimator/gasestimator.go +++ b/eth/gasestimator/gasestimator.go @@ -71,9 +71,9 @@ func Estimate(ctx context.Context, call *core.Message, opts *Options, gasCap uin } // Recap the highest gas limit with account's available balance. if feeCap.BitLen() != 0 { - balance := opts.State.GetBalance(call.From) + balance := opts.State.GetBalance(call.From).ToBig() - available := new(big.Int).Set(balance) + available := balance if call.Value != nil { if call.Value.Cmp(available) >= 0 { return 0, nil, core.ErrInsufficientFundsForTransfer diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 226991b24b..d657eb6d99 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -227,8 +227,8 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks uint64, unresolvedL if p < 0 || p > 100 { return common.Big0, nil, nil, nil, fmt.Errorf("%w: %f", errInvalidPercentile, p) } - if i > 0 && p < rewardPercentiles[i-1] { - return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f > #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p) + if i > 0 && p <= rewardPercentiles[i-1] { + return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f >= #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p) } } var ( diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 2a839f615f..f1284c10e6 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -67,10 +67,7 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { case *eth.NewBlockPacket: return h.handleBlockBroadcast(peer, packet.Block, packet.TD) - case *eth.NewPooledTransactionHashesPacket67: - return h.txFetcher.Notify(peer.ID(), nil, nil, *packet) - - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes) case *eth.TransactionsPacket: diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index bb342acc18..579ca3c097 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -58,11 +58,7 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { h.blockBroadcasts.Send(packet.Block) return nil - case *eth.NewPooledTransactionHashesPacket67: - h.txAnnounces.Send(([]common.Hash)(*packet)) - return nil - - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: h.txAnnounces.Send(packet.Hashes) return nil @@ -81,7 +77,6 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. -func TestForkIDSplit67(t *testing.T) { testForkIDSplit(t, eth.ETH67) } func TestForkIDSplit68(t *testing.T) { testForkIDSplit(t, eth.ETH68) } func testForkIDSplit(t *testing.T, protocol uint) { @@ -236,7 +231,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. -func TestRecvTransactions67(t *testing.T) { testRecvTransactions(t, eth.ETH67) } func TestRecvTransactions68(t *testing.T) { testRecvTransactions(t, eth.ETH68) } func testRecvTransactions(t *testing.T, protocol uint) { @@ -294,7 +288,6 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. -func TestSendTransactions67(t *testing.T) { testSendTransactions(t, eth.ETH67) } func TestSendTransactions68(t *testing.T) { testSendTransactions(t, eth.ETH68) } func testSendTransactions(t *testing.T, protocol uint) { @@ -353,7 +346,7 @@ func testSendTransactions(t *testing.T, protocol uint) { seen := make(map[common.Hash]struct{}) for len(seen) < len(insert) { switch protocol { - case 67, 68: + case 68: select { case hashes := <-anns: for _, hash := range hashes { @@ -379,7 +372,6 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. -func TestTransactionPropagation67(t *testing.T) { testTransactionPropagation(t, eth.ETH67) } func TestTransactionPropagation68(t *testing.T) { testTransactionPropagation(t, eth.ETH68) } func testTransactionPropagation(t *testing.T, protocol uint) { @@ -486,8 +478,8 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { defer sourcePipe.Close() defer sinkPipe.Close() - sourcePeer := eth.NewPeer(eth.ETH67, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH67, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) + sourcePeer := eth.NewPeer(eth.ETH68, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) + sinkPeer := eth.NewPeer(eth.ETH68, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) defer sourcePeer.Close() defer sinkPeer.Close() @@ -539,7 +531,6 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { // Tests that a propagated malformed block (uncles or transactions don't match // with the hashes in the header) gets discarded and not broadcast forward. -func TestBroadcastMalformedBlock67(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH67) } func TestBroadcastMalformedBlock68(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH68) } func testBroadcastMalformedBlock(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index 3045303f22..ad5395cb8d 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -163,16 +163,9 @@ func (p *Peer) announceTransactions() { if len(pending) > 0 { done = make(chan struct{}) go func() { - if p.version >= ETH68 { - if err := p.sendPooledTransactionHashes68(pending, pendingTypes, pendingSizes); err != nil { - fail <- err - return - } - } else { - if err := p.sendPooledTransactionHashes66(pending); err != nil { - fail <- err - return - } + if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes); err != nil { + fail <- err + return } close(done) p.Log().Trace("Sent transaction announcements", "count", len(pending)) diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 42d0412a12..2d69ecdc83 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -93,10 +93,6 @@ type TxPool interface { func MakeProtocols(backend Backend, network uint64, dnsdisc enode.Iterator) []p2p.Protocol { protocols := make([]p2p.Protocol, 0, len(ProtocolVersions)) for _, version := range ProtocolVersions { - // Blob transactions require eth/68 announcements, disable everything else - if version <= ETH67 && backend.Chain().Config().CancunTime != nil { - continue - } version := version // Closure protocols = append(protocols, p2p.Protocol{ @@ -166,26 +162,11 @@ type Decoder interface { Time() time.Time } -var eth67 = map[uint64]msgHandler{ - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes67, - GetBlockHeadersMsg: handleGetBlockHeaders, - BlockHeadersMsg: handleBlockHeaders, - GetBlockBodiesMsg: handleGetBlockBodies, - BlockBodiesMsg: handleBlockBodies, - GetReceiptsMsg: handleGetReceipts, - ReceiptsMsg: handleReceipts, - GetPooledTransactionsMsg: handleGetPooledTransactions, - PooledTransactionsMsg: handlePooledTransactions, -} - var eth68 = map[uint64]msgHandler{ NewBlockHashesMsg: handleNewBlockhashes, NewBlockMsg: handleNewBlock, TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes68, + NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, GetBlockHeadersMsg: handleGetBlockHeaders, BlockHeadersMsg: handleBlockHeaders, GetBlockBodiesMsg: handleGetBlockBodies, @@ -209,10 +190,8 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() - var handlers = eth67 - if peer.Version() >= ETH68 { - handlers = eth68 - } + var handlers = eth68 + // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 41e18bfb3e..08882faa74 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -150,7 +150,6 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. -func TestGetBlockHeaders67(t *testing.T) { testGetBlockHeaders(t, ETH67) } func TestGetBlockHeaders68(t *testing.T) { testGetBlockHeaders(t, ETH68) } func testGetBlockHeaders(t *testing.T, protocol uint) { @@ -336,7 +335,6 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. -func TestGetBlockBodies67(t *testing.T) { testGetBlockBodies(t, ETH67) } func TestGetBlockBodies68(t *testing.T) { testGetBlockBodies(t, ETH68) } func testGetBlockBodies(t *testing.T, protocol uint) { @@ -431,7 +429,6 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } // Tests that the transaction receipts can be retrieved based on hashes. -func TestGetBlockReceipts67(t *testing.T) { testGetBlockReceipts(t, ETH67) } func TestGetBlockReceipts68(t *testing.T) { testGetBlockReceipts(t, ETH68) } func testGetBlockReceipts(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 069e92dadf..0275708a6c 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -383,30 +383,13 @@ func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { }, metadata) } -func handleNewPooledTransactionHashes67(backend Backend, msg Decoder, peer *Peer) error { +func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) error { // New transaction announcement arrived, make sure we have // a valid and fresh chain to handle them if !backend.AcceptTxs() { return nil } - ann := new(NewPooledTransactionHashesPacket67) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Schedule all the unknown hashes for retrieval - for _, hash := range *ann { - peer.markTransaction(hash) - } - return backend.Handle(peer, ann) -} - -func handleNewPooledTransactionHashes68(backend Backend, msg Decoder, peer *Peer) error { - // New transaction announcement arrived, make sure we have - // a valid and fresh chain to handle them - if !backend.AcceptTxs() { - return nil - } - ann := new(NewPooledTransactionHashesPacket68) + ann := new(NewPooledTransactionHashesPacket) if err := msg.Decode(ann); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index d96cfc8165..b9fd13d863 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -27,7 +27,6 @@ import ( ) // Tests that handshake failures are detected and reported correctly. -func TestHandshake67(t *testing.T) { testHandshake(t, ETH67) } func TestHandshake68(t *testing.T) { testHandshake(t, ETH68) } func testHandshake(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 98ad22a8cf..caa5239cf9 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -210,29 +210,17 @@ func (p *Peer) AsyncSendTransactions(hashes []common.Hash) { } } -// sendPooledTransactionHashes66 sends transaction hashes to the peer and includes -// them in its transaction hash set for future reference. -// -// This method is a helper used by the async transaction announcer. Don't call it -// directly as the queueing (memory) and transmission (bandwidth) costs should -// not be managed directly. -func (p *Peer) sendPooledTransactionHashes66(hashes []common.Hash) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - p.knownTxs.Add(hashes...) - return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket67(hashes)) -} - -// sendPooledTransactionHashes68 sends transaction hashes (tagged with their type +// sendPooledTransactionHashes sends transaction hashes (tagged with their type // and size) to the peer and includes them in its transaction hash set for future // reference. // // This method is a helper used by the async transaction announcer. Don't call it // directly as the queueing (memory) and transmission (bandwidth) costs should // not be managed directly. -func (p *Peer) sendPooledTransactionHashes68(hashes []common.Hash, types []byte, sizes []uint32) error { +func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32) error { // Mark all the transactions as known, but ensure we don't overflow our limits p.knownTxs.Add(hashes...) - return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket68{Types: types, Sizes: sizes, Hashes: hashes}) + return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket{Types: types, Sizes: sizes, Hashes: hashes}) } // AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 0f44f83de1..47e8d97244 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -30,7 +30,6 @@ import ( // Constants to match up protocol versions and messages const ( - ETH67 = 67 ETH68 = 68 ) @@ -40,11 +39,11 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH68, ETH67} +var ProtocolVersions = []uint{ETH68} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH68: 17, ETH67: 17} +var protocolLengths = map[uint]uint64{ETH68: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 @@ -283,11 +282,8 @@ type ReceiptsRLPPacket struct { ReceiptsRLPResponse } -// NewPooledTransactionHashesPacket67 represents a transaction announcement packet on eth/67. -type NewPooledTransactionHashesPacket67 []common.Hash - -// NewPooledTransactionHashesPacket68 represents a transaction announcement packet on eth/68 and newer. -type NewPooledTransactionHashesPacket68 struct { +// NewPooledTransactionHashesPacket represents a transaction announcement packet on eth/68 and newer. +type NewPooledTransactionHashesPacket struct { Types []byte Sizes []uint32 Hashes []common.Hash @@ -346,10 +342,8 @@ func (*BlockBodiesResponse) Kind() byte { return BlockBodiesMsg } func (*NewBlockPacket) Name() string { return "NewBlock" } func (*NewBlockPacket) Kind() byte { return NewBlockMsg } -func (*NewPooledTransactionHashesPacket67) Name() string { return "NewPooledTransactionHashes" } -func (*NewPooledTransactionHashesPacket67) Kind() byte { return NewPooledTransactionHashesMsg } -func (*NewPooledTransactionHashesPacket68) Name() string { return "NewPooledTransactionHashes" } -func (*NewPooledTransactionHashesPacket68) Kind() byte { return NewPooledTransactionHashesMsg } +func (*NewPooledTransactionHashesPacket) Name() string { return "NewPooledTransactionHashes" } +func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransactionHashesMsg } func (*GetPooledTransactionsRequest) Name() string { return "GetPooledTransactions" } func (*GetPooledTransactionsRequest) Kind() byte { return GetPooledTransactionsMsg } diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 5d4099a814..73d61c2ffd 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/trie/testutil" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" "golang.org/x/exp/slices" ) @@ -1510,7 +1511,7 @@ func makeAccountTrieNoStorage(n int, scheme string) (string, *trie.Trie, []*kv) for i := uint64(1); i <= uint64(n); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, - Balance: big.NewInt(int64(i)), + Balance: uint256.NewInt(i), Root: types.EmptyRootHash, CodeHash: getCodeHash(i), }) @@ -1561,7 +1562,7 @@ func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) { for i := 0; i < len(boundaries); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: uint64(0), - Balance: big.NewInt(int64(i)), + Balance: uint256.NewInt(uint64(i)), Root: types.EmptyRootHash, CodeHash: getCodeHash(uint64(i)), }) @@ -1573,7 +1574,7 @@ func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) { for i := uint64(1); i <= uint64(n); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, - Balance: big.NewInt(int64(i)), + Balance: uint256.NewInt(i), Root: types.EmptyRootHash, CodeHash: getCodeHash(i), }) @@ -1617,7 +1618,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, - Balance: big.NewInt(int64(i)), + Balance: uint256.NewInt(i), Root: stRoot, CodeHash: codehash, }) @@ -1683,7 +1684,7 @@ func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, bounda value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, - Balance: big.NewInt(int64(i)), + Balance: uint256.NewInt(i), Root: stRoot, CodeHash: codehash, }) diff --git a/eth/sync.go b/eth/sync.go index c7ba7c93d6..c2a0f453bf 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -228,24 +228,6 @@ func (cs *chainSyncer) startSync(op *chainSyncOp) { // doSync synchronizes the local blockchain with a remote peer. func (h *handler) doSync(op *chainSyncOp) error { - if op.mode == downloader.SnapSync { - // Before launch the snap sync, we have to ensure user uses the same - // txlookup limit. - // The main concern here is: during the snap sync Geth won't index the - // block(generate tx indices) before the HEAD-limit. But if user changes - // the limit in the next snap sync(e.g. user kill Geth manually and - // restart) then it will be hard for Geth to figure out the oldest block - // has been indexed. So here for the user-experience wise, it's non-optimal - // that user can't change limit during the snap sync. If changed, Geth - // will just blindly use the original one. - limit := h.chain.TxLookupLimit() - if stored := rawdb.ReadFastTxLookupLimit(h.database); stored == nil { - rawdb.WriteFastTxLookupLimit(h.database, limit) - } else if *stored != limit { - h.chain.SetTxLookupLimit(*stored) - log.Warn("Update txLookup limit", "provided", limit, "updated", *stored) - } - } // Run the sync cycle, and disable snap sync if we're past the pivot block err := h.downloader.LegacySync(op.peer.ID(), op.head, op.td, h.chain.Config().TerminalTotalDifficulty, op.mode) if err != nil { diff --git a/eth/sync_test.go b/eth/sync_test.go index d26cbb66ea..a31986730f 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,7 +28,6 @@ import ( ) // Tests that snap sync is disabled after a successful sync cycle. -func TestSnapSyncDisabling67(t *testing.T) { testSnapSyncDisabling(t, eth.ETH67, snap.SNAP1) } func TestSnapSyncDisabling68(t *testing.T) { testSnapSyncDisabling(t, eth.ETH68, snap.SNAP1) } // Tests that snap sync gets disabled as soon as a real block is successfully diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 7c0028601d..4d4428f6c6 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -80,7 +80,7 @@ type Backend interface { HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) - GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) + GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) RPCGasCap() uint64 ChainConfig() *params.ChainConfig Engine() consensus.Engine @@ -826,12 +826,12 @@ func containsTx(block *types.Block, hash common.Hash) bool { // TraceTransaction returns the structured logs created during the execution of EVM // and returns them as a JSON object. func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config *TraceConfig) (interface{}, error) { - tx, blockHash, blockNumber, index, err := api.backend.GetTransaction(ctx, hash) + found, _, blockHash, blockNumber, index, err := api.backend.GetTransaction(ctx, hash) if err != nil { - return nil, err + return nil, ethapi.NewTxIndexingError() } // Only mined txes are supported - if tx == nil { + if !found { return nil, errTxNotFound } // It shouldn't happen in practice. diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 49c3ebb67d..8aaa20fce5 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -113,9 +113,9 @@ func (b *testBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) return b.chain.GetBlockByNumber(uint64(number)), nil } -func (b *testBackend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { +func (b *testBackend) GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) { tx, hash, blockNumber, index := rawdb.ReadTransaction(b.chaindb, txHash) - return tx, hash, blockNumber, index, nil + return tx != nil, tx, hash, blockNumber, index, nil } func (b *testBackend) RPCGasCap() uint64 { diff --git a/eth/tracers/js/internal/tracers/call_tracer_legacy.js b/eth/tracers/js/internal/tracers/call_tracer_legacy.js index 451a644b91..0760bb1e3f 100644 --- a/eth/tracers/js/internal/tracers/call_tracer_legacy.js +++ b/eth/tracers/js/internal/tracers/call_tracer_legacy.js @@ -219,7 +219,7 @@ return this.finalize(result); }, - // finalize recreates a call object using the final desired field oder for json + // finalize recreates a call object using the final desired field order for json // serialization. This is a nicety feature to pass meaningfully ordered results // to users who don't interpret it, just display it. finalize: function(call) { diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index bf6427faf6..b7f2693770 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) type account struct{} @@ -37,9 +38,9 @@ func (account) SubBalance(amount *big.Int) {} func (account) AddBalance(amount *big.Int) {} func (account) SetAddress(common.Address) {} func (account) Value() *big.Int { return nil } -func (account) SetBalance(*big.Int) {} +func (account) SetBalance(*uint256.Int) {} func (account) SetNonce(uint64) {} -func (account) Balance() *big.Int { return nil } +func (account) Balance() *uint256.Int { return nil } func (account) Address() common.Address { return common.Address{} } func (account) SetCode(common.Hash, []byte) {} func (account) ForEachStorage(cb func(key, value common.Hash) bool) {} @@ -48,8 +49,8 @@ type dummyStatedb struct { state.StateDB } -func (*dummyStatedb) GetRefund() uint64 { return 1337 } -func (*dummyStatedb) GetBalance(addr common.Address) *big.Int { return new(big.Int) } +func (*dummyStatedb) GetRefund() uint64 { return 1337 } +func (*dummyStatedb) GetBalance(addr common.Address) *uint256.Int { return new(uint256.Int) } type vmContext struct { blockCtx vm.BlockContext @@ -65,7 +66,7 @@ func runTrace(tracer tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCon env = vm.NewEVM(vmctx.blockCtx, vmctx.txCtx, &dummyStatedb{}, chaincfg, vm.Config{Tracer: tracer}) gasLimit uint64 = 31000 startGas uint64 = 10000 - value = big.NewInt(0) + value = uint256.NewInt(0) contract = vm.NewContract(account{}, account{}, value, startGas) ) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} @@ -74,7 +75,7 @@ func runTrace(tracer tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCon } tracer.CaptureTxStart(gasLimit) - tracer.CaptureStart(env, contract.Caller(), contract.Address(), false, []byte{}, startGas, value) + tracer.CaptureStart(env, contract.Caller(), contract.Address(), false, []byte{}, startGas, value.ToBig()) ret, err := env.Interpreter().Run(contract, []byte{}, false) tracer.CaptureEnd(ret, startGas-contract.Gas, err) // Rest gas assumes no refund @@ -182,7 +183,7 @@ func TestHaltBetweenSteps(t *testing.T) { } env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{GasPrice: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: tracer}) scope := &vm.ScopeContext{ - Contract: vm.NewContract(&account{}, &account{}, big.NewInt(0), 0), + Contract: vm.NewContract(&account{}, &account{}, uint256.NewInt(0), 0), } tracer.CaptureStart(env, common.Address{}, common.Address{}, false, []byte{}, 0, big.NewInt(0)) tracer.CaptureState(0, 0, 0, 0, scope, nil, 0, nil) @@ -273,7 +274,7 @@ func TestEnterExit(t *testing.T) { t.Fatal(err) } scope := &vm.ScopeContext{ - Contract: vm.NewContract(&account{}, &account{}, big.NewInt(0), 0), + Contract: vm.NewContract(&account{}, &account{}, uint256.NewInt(0), 0), } tracer.CaptureEnter(vm.CALL, scope.Contract.Caller(), scope.Contract.Address(), []byte{}, 1000, new(big.Int)) tracer.CaptureExit([]byte{}, 400, nil) diff --git a/eth/tracers/logger/logger_test.go b/eth/tracers/logger/logger_test.go index 3192a15cba..1d8eb320f6 100644 --- a/eth/tracers/logger/logger_test.go +++ b/eth/tracers/logger/logger_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) type dummyContractRef struct { @@ -56,7 +57,7 @@ func TestStoreCapture(t *testing.T) { var ( logger = NewStructLogger(nil) env = vm.NewEVM(vm.BlockContext{}, vm.TxContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: logger}) - contract = vm.NewContract(&dummyContractRef{}, &dummyContractRef{}, new(big.Int), 100000) + contract = vm.NewContract(&dummyContractRef{}, &dummyContractRef{}, new(uint256.Int), 100000) ) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x0, byte(vm.SSTORE)} var index common.Hash diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index 82451c40a6..d7e10173cf 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -195,7 +195,7 @@ func (t *prestateTracer) CaptureTxEnd(restGas uint64) { } modified := false postAccount := &account{Storage: make(map[common.Hash]common.Hash)} - newBalance := t.env.StateDB.GetBalance(addr) + newBalance := t.env.StateDB.GetBalance(addr).ToBig() newNonce := t.env.StateDB.GetNonce(addr) newCode := t.env.StateDB.GetCode(addr) @@ -279,7 +279,7 @@ func (t *prestateTracer) lookupAccount(addr common.Address) { } t.pre[addr] = &account{ - Balance: t.env.StateDB.GetBalance(addr), + Balance: t.env.StateDB.GetBalance(addr).ToBig(), Nonce: t.env.StateDB.GetNonce(addr), Code: t.env.StateDB.GetCode(addr), Storage: make(map[common.Hash]common.Hash), diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 54d34ec5d1..b10f3503e0 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -124,9 +124,9 @@ func TestMemCopying(t *testing.T) { {0, 100, 0, "", 0}, // No need to pad (0 size) {100, 50, 100, "", 100}, // Should pad 100-150 {100, 50, 5, "", 5}, // Wanted range fully within memory - {100, -50, 0, "offset or size must not be negative", 0}, // Errror - {0, 1, 1024*1024 + 1, "reached limit for padding memory slice: 1048578", 0}, // Errror - {10, 0, 1024*1024 + 100, "reached limit for padding memory slice: 1048666", 0}, // Errror + {100, -50, 0, "offset or size must not be negative", 0}, // Error + {0, 1, 1024*1024 + 1, "reached limit for padding memory slice: 1048578", 0}, // Error + {10, 0, 1024*1024 + 100, "reached limit for padding memory slice: 1048666", 0}, // Error } { mem := vm.NewMemory() diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 5b4e906cbb..4c63b776ef 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -677,18 +677,20 @@ type rpcProgress struct { PulledStates hexutil.Uint64 KnownStates hexutil.Uint64 - SyncedAccounts hexutil.Uint64 - SyncedAccountBytes hexutil.Uint64 - SyncedBytecodes hexutil.Uint64 - SyncedBytecodeBytes hexutil.Uint64 - SyncedStorage hexutil.Uint64 - SyncedStorageBytes hexutil.Uint64 - HealedTrienodes hexutil.Uint64 - HealedTrienodeBytes hexutil.Uint64 - HealedBytecodes hexutil.Uint64 - HealedBytecodeBytes hexutil.Uint64 - HealingTrienodes hexutil.Uint64 - HealingBytecode hexutil.Uint64 + SyncedAccounts hexutil.Uint64 + SyncedAccountBytes hexutil.Uint64 + SyncedBytecodes hexutil.Uint64 + SyncedBytecodeBytes hexutil.Uint64 + SyncedStorage hexutil.Uint64 + SyncedStorageBytes hexutil.Uint64 + HealedTrienodes hexutil.Uint64 + HealedTrienodeBytes hexutil.Uint64 + HealedBytecodes hexutil.Uint64 + HealedBytecodeBytes hexutil.Uint64 + HealingTrienodes hexutil.Uint64 + HealingBytecode hexutil.Uint64 + TxIndexFinishedBlocks hexutil.Uint64 + TxIndexRemainingBlocks hexutil.Uint64 } func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress { @@ -696,22 +698,24 @@ func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress { return nil } return ðereum.SyncProgress{ - StartingBlock: uint64(p.StartingBlock), - CurrentBlock: uint64(p.CurrentBlock), - HighestBlock: uint64(p.HighestBlock), - PulledStates: uint64(p.PulledStates), - KnownStates: uint64(p.KnownStates), - SyncedAccounts: uint64(p.SyncedAccounts), - SyncedAccountBytes: uint64(p.SyncedAccountBytes), - SyncedBytecodes: uint64(p.SyncedBytecodes), - SyncedBytecodeBytes: uint64(p.SyncedBytecodeBytes), - SyncedStorage: uint64(p.SyncedStorage), - SyncedStorageBytes: uint64(p.SyncedStorageBytes), - HealedTrienodes: uint64(p.HealedTrienodes), - HealedTrienodeBytes: uint64(p.HealedTrienodeBytes), - HealedBytecodes: uint64(p.HealedBytecodes), - HealedBytecodeBytes: uint64(p.HealedBytecodeBytes), - HealingTrienodes: uint64(p.HealingTrienodes), - HealingBytecode: uint64(p.HealingBytecode), + StartingBlock: uint64(p.StartingBlock), + CurrentBlock: uint64(p.CurrentBlock), + HighestBlock: uint64(p.HighestBlock), + PulledStates: uint64(p.PulledStates), + KnownStates: uint64(p.KnownStates), + SyncedAccounts: uint64(p.SyncedAccounts), + SyncedAccountBytes: uint64(p.SyncedAccountBytes), + SyncedBytecodes: uint64(p.SyncedBytecodes), + SyncedBytecodeBytes: uint64(p.SyncedBytecodeBytes), + SyncedStorage: uint64(p.SyncedStorage), + SyncedStorageBytes: uint64(p.SyncedStorageBytes), + HealedTrienodes: uint64(p.HealedTrienodes), + HealedTrienodeBytes: uint64(p.HealedTrienodeBytes), + HealedBytecodes: uint64(p.HealedBytecodes), + HealedBytecodeBytes: uint64(p.HealedBytecodeBytes), + HealingTrienodes: uint64(p.HealingTrienodes), + HealingBytecode: uint64(p.HealingBytecode), + TxIndexFinishedBlocks: uint64(p.TxIndexFinishedBlocks), + TxIndexRemainingBlocks: uint64(p.TxIndexRemainingBlocks), } } diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 2ef68337c6..fd053c1d73 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -231,6 +231,13 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { if _, err := ethservice.BlockChain().InsertChain(blocks[1:]); err != nil { t.Fatalf("can't import test blocks: %v", err) } + // Ensure the tx indexing is fully generated + for ; ; time.Sleep(time.Millisecond * 100) { + progress, err := ethservice.BlockChain().TxIndexProgress() + if err == nil && progress.Done() { + break + } + } return n, blocks } diff --git a/ethclient/simulated/backend_test.go b/ethclient/simulated/backend_test.go index a9a8accfea..49b1065ec5 100644 --- a/ethclient/simulated/backend_test.go +++ b/ethclient/simulated/backend_test.go @@ -52,7 +52,7 @@ func newTx(sim *Backend, key *ecdsa.PrivateKey) (*types.Transaction, error) { // create a signed transaction to send head, _ := client.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(params.GWei)) addr := crypto.PubkeyToAddress(key.PublicKey) chainid, _ := client.ChainID(context.Background()) nonce, err := client.PendingNonceAt(context.Background(), addr) @@ -62,7 +62,7 @@ func newTx(sim *Backend, key *ecdsa.PrivateKey) (*types.Transaction, error) { tx := types.NewTx(&types.DynamicFeeTx{ ChainID: chainid, Nonce: nonce, - GasTipCap: big.NewInt(1), + GasTipCap: big.NewInt(params.GWei), GasFeeCap: gasPrice, Gas: 21000, To: &addr, diff --git a/ethclient/simulated/options.go b/ethclient/simulated/options.go index 1b2f4c090d..6db995c917 100644 --- a/ethclient/simulated/options.go +++ b/ethclient/simulated/options.go @@ -17,6 +17,8 @@ package simulated import ( + "math/big" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/node" ) @@ -37,3 +39,17 @@ func WithCallGasLimit(gaslimit uint64) func(nodeConf *node.Config, ethConf *ethc ethConf.RPCGasCap = gaslimit } } + +// WithMinerMinTip configures the simulated backend to require a specific minimum +// gas tip for a transaction to be included. +// +// 0 is not possible as a live Geth node would reject that due to DoS protection, +// so the simulated backend will replicate that behavior for consistency. +func WithMinerMinTip(tip *big.Int) func(nodeConf *node.Config, ethConf *ethconfig.Config) { + if tip == nil || tip.Cmp(new(big.Int)) <= 0 { + panic("invalid miner minimum tip") + } + return func(nodeConf *node.Config, ethConf *ethconfig.Config) { + ethConf.Miner.GasPrice = tip + } +} diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 75d0faac54..29559991be 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -792,7 +792,7 @@ func (s *Service) reportStats(conn *connWrapper) error { } sync := fullBackend.SyncProgress() - syncing = fullBackend.CurrentHeader().Number.Uint64() >= sync.HighestBlock + syncing = !sync.Done() price, _ := fullBackend.SuggestGasTipCap(context.Background()) gasprice = int(price.Uint64()) @@ -801,7 +801,7 @@ func (s *Service) reportStats(conn *connWrapper) error { } } else { sync := s.backend.SyncProgress() - syncing = s.backend.CurrentHeader().Number.Uint64() >= sync.HighestBlock + syncing = !sync.Done() } // Assemble the node stats and send it to the server log.Trace("Sending node details to ethstats") diff --git a/go.mod b/go.mod index b4d077fc47..7b276ebfc5 100644 --- a/go.mod +++ b/go.mod @@ -22,8 +22,9 @@ require ( github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 github.com/ethereum/c-kzg-4844 v0.4.0 github.com/fatih/color v1.13.0 + github.com/ferranbt/fastssz v0.1.2 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e - github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 + github.com/fjl/memsize v0.0.2 github.com/fsnotify/fsnotify v1.6.0 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 @@ -65,7 +66,7 @@ require ( golang.org/x/crypto v0.17.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/sync v0.5.0 - golang.org/x/sys v0.15.0 + golang.org/x/sys v0.16.0 golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.15.0 @@ -101,7 +102,7 @@ require ( github.com/deepmap/oapi-codegen v1.6.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect - github.com/go-ole/go-ole v1.2.5 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -114,10 +115,12 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect github.com/klauspost/compress v1.15.15 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect diff --git a/go.sum b/go.sum index bab51b1345..f0cdf72f0f 100644 --- a/go.sum +++ b/go.sum @@ -187,10 +187,12 @@ github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk= +github.com/ferranbt/fastssz v0.1.2/go.mod h1:X5UPrE2u1UJjxHA8X54u04SBwdAQjG2sFtWs39YxyWs= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= +github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -221,8 +223,9 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= -github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= @@ -398,6 +401,9 @@ github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -445,6 +451,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= @@ -522,6 +530,7 @@ github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7 h1:cZC+usqsYgHtlBaGulVnZ1hfKAi8iWtujBnRLQE698c= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7/go.mod h1:IToEjHuttnUzwZI5KBSM/LOOW3qLbbrHOEfp3SbECGY= +github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220714111606-acbb2962fb48 h1:cSo6/vk8YpvkLbk9v3FO97cakNmUoxwi2KMP8hd5WIw= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -771,11 +780,12 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/graphql/graphql.go b/graphql/graphql.go index 49be23af69..bac86476b1 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -100,7 +100,7 @@ func (a *Account) Balance(ctx context.Context) (hexutil.Big, error) { if err != nil { return hexutil.Big{}, err } - balance := state.GetBalance(a.address) + balance := state.GetBalance(a.address).ToBig() if balance == nil { return hexutil.Big{}, fmt.Errorf("failed to load balance %x", a.address) } @@ -230,8 +230,8 @@ func (t *Transaction) resolve(ctx context.Context) (*types.Transaction, *Block) return t.tx, t.block } // Try to return an already finalized transaction - tx, blockHash, _, index, err := t.r.backend.GetTransaction(ctx, t.hash) - if err == nil && tx != nil { + found, tx, blockHash, _, index, _ := t.r.backend.GetTransaction(ctx, t.hash) + if found { t.tx = tx blockNrOrHash := rpc.BlockNumberOrHashWithHash(blockHash, false) t.block = &Block{ @@ -1509,6 +1509,12 @@ func (s *SyncState) HealingTrienodes() hexutil.Uint64 { func (s *SyncState) HealingBytecode() hexutil.Uint64 { return hexutil.Uint64(s.progress.HealingBytecode) } +func (s *SyncState) TxIndexFinishedBlocks() hexutil.Uint64 { + return hexutil.Uint64(s.progress.TxIndexFinishedBlocks) +} +func (s *SyncState) TxIndexRemainingBlocks() hexutil.Uint64 { + return hexutil.Uint64(s.progress.TxIndexRemainingBlocks) +} // Syncing returns false in case the node is currently not syncing with the network. It can be up-to-date or has not // yet received the latest block headers from its pears. In case it is synchronizing: @@ -1527,11 +1533,13 @@ func (s *SyncState) HealingBytecode() hexutil.Uint64 { // - healedBytecodeBytes: number of bytecodes persisted to disk // - healingTrienodes: number of state trie nodes pending // - healingBytecode: number of bytecodes pending +// - txIndexFinishedBlocks: number of blocks whose transactions are indexed +// - txIndexRemainingBlocks: number of blocks whose transactions are not indexed yet func (r *Resolver) Syncing() (*SyncState, error) { progress := r.backend.SyncProgress() // Return not syncing if the synchronisation already completed - if progress.CurrentBlock >= progress.HighestBlock { + if progress.Done() { return nil, nil } // Otherwise gather the block sync stats diff --git a/interfaces.go b/interfaces.go index 1892309ed3..c6aee295ee 100644 --- a/interfaces.go +++ b/interfaces.go @@ -120,6 +120,18 @@ type SyncProgress struct { HealingTrienodes uint64 // Number of state trie nodes pending HealingBytecode uint64 // Number of bytecodes pending + + // "transaction indexing" fields + TxIndexFinishedBlocks uint64 // Number of blocks whose transactions are already indexed + TxIndexRemainingBlocks uint64 // Number of blocks whose transactions are not indexed yet +} + +// Done returns the indicator if the initial sync is finished or not. +func (prog SyncProgress) Done() bool { + if prog.CurrentBlock < prog.HighestBlock { + return false + } + return prog.TxIndexRemainingBlocks == 0 } // ChainSyncReader wraps access to the node's current sync status. If there's no diff --git a/internal/build/download.go b/internal/build/download.go index 903d0308df..fda573df83 100644 --- a/internal/build/download.go +++ b/internal/build/download.go @@ -40,7 +40,7 @@ func MustLoadChecksums(file string) *ChecksumDB { if err != nil { log.Fatal("can't load checksum file: " + err.Error()) } - return &ChecksumDB{strings.Split(string(content), "\n")} + return &ChecksumDB{strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n")} } // Verify checks whether the given file is valid according to the checksum database. diff --git a/internal/era/accumulator.go b/internal/era/accumulator.go new file mode 100644 index 0000000000..19e03973f1 --- /dev/null +++ b/internal/era/accumulator.go @@ -0,0 +1,90 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + ssz "github.com/ferranbt/fastssz" +) + +// ComputeAccumulator calculates the SSZ hash tree root of the Era1 +// accumulator of header records. +func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, error) { + if len(hashes) != len(tds) { + return common.Hash{}, fmt.Errorf("must have equal number hashes as td values") + } + if len(hashes) > MaxEra1Size { + return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEra1Size) + } + hh := ssz.NewHasher() + for i := range hashes { + rec := headerRecord{hashes[i], tds[i]} + root, err := rec.HashTreeRoot() + if err != nil { + return common.Hash{}, err + } + hh.Append(root[:]) + } + hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxEra1Size)) + return hh.HashRoot() +} + +// headerRecord is an individual record for a historical header. +// +// See https://github.com/ethereum/portal-network-specs/blob/master/history-network.md#the-header-accumulator +// for more information. +type headerRecord struct { + Hash common.Hash + TotalDifficulty *big.Int +} + +// GetTree completes the ssz.HashRoot interface, but is unused. +func (h *headerRecord) GetTree() (*ssz.Node, error) { + return nil, nil +} + +// HashTreeRoot ssz hashes the headerRecord object. +func (h *headerRecord) HashTreeRoot() ([32]byte, error) { + return ssz.HashWithDefaultHasher(h) +} + +// HashTreeRootWith ssz hashes the headerRecord object with a hasher. +func (h *headerRecord) HashTreeRootWith(hh ssz.HashWalker) (err error) { + hh.PutBytes(h.Hash[:]) + td := bigToBytes32(h.TotalDifficulty) + hh.PutBytes(td[:]) + hh.Merkleize(0) + return +} + +// bigToBytes32 converts a big.Int into a little-endian 32-byte array. +func bigToBytes32(n *big.Int) (b [32]byte) { + n.FillBytes(b[:]) + reverseOrder(b[:]) + return +} + +// reverseOrder reverses the byte order of a slice. +func reverseOrder(b []byte) []byte { + for i := 0; i < 16; i++ { + b[i], b[32-i-1] = b[32-i-1], b[i] + } + return b +} diff --git a/internal/era/builder.go b/internal/era/builder.go new file mode 100644 index 0000000000..9217c049f3 --- /dev/null +++ b/internal/era/builder.go @@ -0,0 +1,224 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . +package era + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +// Builder is used to create Era1 archives of block data. +// +// Era1 files are themselves e2store files. For more information on this format, +// see https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md. +// +// The overall structure of an Era1 file follows closely the structure of an Era file +// which contains consensus Layer data (and as a byproduct, EL data after the merge). +// +// The structure can be summarized through this definition: +// +// era1 := Version | block-tuple* | other-entries* | Accumulator | BlockIndex +// block-tuple := CompressedHeader | CompressedBody | CompressedReceipts | TotalDifficulty +// +// Each basic element is its own entry: +// +// Version = { type: [0x65, 0x32], data: nil } +// CompressedHeader = { type: [0x03, 0x00], data: snappyFramed(rlp(header)) } +// CompressedBody = { type: [0x04, 0x00], data: snappyFramed(rlp(body)) } +// CompressedReceipts = { type: [0x05, 0x00], data: snappyFramed(rlp(receipts)) } +// TotalDifficulty = { type: [0x06, 0x00], data: uint256(header.total_difficulty) } +// AccumulatorRoot = { type: [0x07, 0x00], data: accumulator-root } +// BlockIndex = { type: [0x32, 0x66], data: block-index } +// +// Accumulator is computed by constructing an SSZ list of header-records of length at most +// 8192 and then calculating the hash_tree_root of that list. +// +// header-record := { block-hash: Bytes32, total-difficulty: Uint256 } +// accumulator := hash_tree_root([]header-record, 8192) +// +// BlockIndex stores relative offsets to each compressed block entry. The +// format is: +// +// block-index := starting-number | index | index | index ... | count +// +// starting-number is the first block number in the archive. Every index is a +// defined relative to beginning of the record. The total number of block +// entries in the file is recorded with count. +// +// Due to the accumulator size limit of 8192, the maximum number of blocks in +// an Era1 batch is also 8192. +type Builder struct { + w *e2store.Writer + startNum *uint64 + startTd *big.Int + indexes []uint64 + hashes []common.Hash + tds []*big.Int + written int + + buf *bytes.Buffer + snappy *snappy.Writer +} + +// NewBuilder returns a new Builder instance. +func NewBuilder(w io.Writer) *Builder { + buf := bytes.NewBuffer(nil) + return &Builder{ + w: e2store.NewWriter(w), + buf: buf, + snappy: snappy.NewBufferedWriter(buf), + } +} + +// Add writes a compressed block entry and compressed receipts entry to the +// underlying e2store file. +func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) error { + eh, err := rlp.EncodeToBytes(block.Header()) + if err != nil { + return err + } + eb, err := rlp.EncodeToBytes(block.Body()) + if err != nil { + return err + } + er, err := rlp.EncodeToBytes(receipts) + if err != nil { + return err + } + return b.AddRLP(eh, eb, er, block.NumberU64(), block.Hash(), td, block.Difficulty()) +} + +// AddRLP writes a compressed block entry and compressed receipts entry to the +// underlying e2store file. +func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error { + // Write Era1 version entry before first block. + if b.startNum == nil { + n, err := b.w.Write(TypeVersion, nil) + if err != nil { + return err + } + startNum := number + b.startNum = &startNum + b.startTd = new(big.Int).Sub(td, difficulty) + b.written += n + } + if len(b.indexes) >= MaxEra1Size { + return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size) + } + + b.indexes = append(b.indexes, uint64(b.written)) + b.hashes = append(b.hashes, hash) + b.tds = append(b.tds, td) + + // Write block data. + if err := b.snappyWrite(TypeCompressedHeader, header); err != nil { + return err + } + if err := b.snappyWrite(TypeCompressedBody, body); err != nil { + return err + } + if err := b.snappyWrite(TypeCompressedReceipts, receipts); err != nil { + return err + } + + // Also write total difficulty, but don't snappy encode. + btd := bigToBytes32(td) + n, err := b.w.Write(TypeTotalDifficulty, btd[:]) + b.written += n + if err != nil { + return err + } + + return nil +} + +// Finalize computes the accumulator and block index values, then writes the +// corresponding e2store entries. +func (b *Builder) Finalize() (common.Hash, error) { + if b.startNum == nil { + return common.Hash{}, fmt.Errorf("finalize called on empty builder") + } + // Compute accumulator root and write entry. + root, err := ComputeAccumulator(b.hashes, b.tds) + if err != nil { + return common.Hash{}, fmt.Errorf("error calculating accumulator root: %w", err) + } + n, err := b.w.Write(TypeAccumulator, root[:]) + b.written += n + if err != nil { + return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err) + } + // Get beginning of index entry to calculate block relative offset. + base := int64(b.written) + + // Construct block index. Detailed format described in Builder + // documentation, but it is essentially encoded as: + // "start | index | index | ... | count" + var ( + count = len(b.indexes) + index = make([]byte, 16+count*8) + ) + binary.LittleEndian.PutUint64(index, *b.startNum) + // Each offset is relative from the position it is encoded in the + // index. This means that even if the same block was to be included in + // the index twice (this would be invalid anyways), the relative offset + // would be different. The idea with this is that after reading a + // relative offset, the corresponding block can be quickly read by + // performing a seek relative to the current position. + for i, offset := range b.indexes { + relative := int64(offset) - base + binary.LittleEndian.PutUint64(index[8+i*8:], uint64(relative)) + } + binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count)) + + // Finally, write the block index entry. + if _, err := b.w.Write(TypeBlockIndex, index); err != nil { + return common.Hash{}, fmt.Errorf("unable to write block index: %w", err) + } + + return root, nil +} + +// snappyWrite is a small helper to take care snappy encoding and writing an e2store entry. +func (b *Builder) snappyWrite(typ uint16, in []byte) error { + var ( + buf = b.buf + s = b.snappy + ) + buf.Reset() + s.Reset(buf) + if _, err := b.snappy.Write(in); err != nil { + return fmt.Errorf("error snappy encoding: %w", err) + } + if err := s.Flush(); err != nil { + return fmt.Errorf("error flushing snappy encoding: %w", err) + } + n, err := b.w.Write(typ, b.buf.Bytes()) + b.written += n + if err != nil { + return fmt.Errorf("error writing e2store entry: %w", err) + } + return nil +} diff --git a/internal/era/e2store/e2store.go b/internal/era/e2store/e2store.go new file mode 100644 index 0000000000..d85b3e44e9 --- /dev/null +++ b/internal/era/e2store/e2store.go @@ -0,0 +1,220 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package e2store + +import ( + "encoding/binary" + "fmt" + "io" +) + +const ( + headerSize = 8 + valueSizeLimit = 1024 * 1024 * 50 +) + +// Entry is a variable-length-data record in an e2store. +type Entry struct { + Type uint16 + Value []byte +} + +// Writer writes entries using e2store encoding. +// For more information on this format, see: +// https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md +type Writer struct { + w io.Writer +} + +// NewWriter returns a new Writer that writes to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{w} +} + +// Write writes a single e2store entry to w. +// An entry is encoded in a type-length-value format. The first 8 bytes of the +// record store the type (2 bytes), the length (4 bytes), and some reserved +// data (2 bytes). The remaining bytes store b. +func (w *Writer) Write(typ uint16, b []byte) (int, error) { + buf := make([]byte, headerSize) + binary.LittleEndian.PutUint16(buf, typ) + binary.LittleEndian.PutUint32(buf[2:], uint32(len(b))) + + // Write header. + if n, err := w.w.Write(buf); err != nil { + return n, err + } + // Write value, return combined write size. + n, err := w.w.Write(b) + return n + headerSize, err +} + +// A Reader reads entries from an e2store-encoded file. +// For more information on this format, see +// https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md +type Reader struct { + r io.ReaderAt + offset int64 +} + +// NewReader returns a new Reader that reads from r. +func NewReader(r io.ReaderAt) *Reader { + return &Reader{r, 0} +} + +// Read reads one Entry from r. +func (r *Reader) Read() (*Entry, error) { + var e Entry + n, err := r.ReadAt(&e, r.offset) + if err != nil { + return nil, err + } + r.offset += int64(n) + return &e, nil +} + +// ReadAt reads one Entry from r at the specified offset. +func (r *Reader) ReadAt(entry *Entry, off int64) (int, error) { + typ, length, err := r.ReadMetadataAt(off) + if err != nil { + return 0, err + } + entry.Type = typ + + // Check length bounds. + if length > valueSizeLimit { + return headerSize, fmt.Errorf("item larger than item size limit %d: have %d", valueSizeLimit, length) + } + if length == 0 { + return headerSize, nil + } + + // Read value. + val := make([]byte, length) + if n, err := r.r.ReadAt(val, off+headerSize); err != nil { + n += headerSize + // An entry with a non-zero length should not return EOF when + // reading the value. + if err == io.EOF { + return n, io.ErrUnexpectedEOF + } + return n, err + } + entry.Value = val + return int(headerSize + length), nil +} + +// ReaderAt returns an io.Reader delivering value data for the entry at +// the specified offset. If the entry type does not match the expected type, an +// error is returned. +func (r *Reader) ReaderAt(expectedType uint16, off int64) (io.Reader, int, error) { + // problem = need to return length+headerSize not just value length via section reader + typ, length, err := r.ReadMetadataAt(off) + if err != nil { + return nil, headerSize, err + } + if typ != expectedType { + return nil, headerSize, fmt.Errorf("wrong type, want %d have %d", expectedType, typ) + } + if length > valueSizeLimit { + return nil, headerSize, fmt.Errorf("item larger than item size limit %d: have %d", valueSizeLimit, length) + } + return io.NewSectionReader(r.r, off+headerSize, int64(length)), headerSize + int(length), nil +} + +// LengthAt reads the header at off and returns the total length of the entry, +// including header. +func (r *Reader) LengthAt(off int64) (int64, error) { + _, length, err := r.ReadMetadataAt(off) + if err != nil { + return 0, err + } + return int64(length) + headerSize, nil +} + +// ReadMetadataAt reads the header metadata at the given offset. +func (r *Reader) ReadMetadataAt(off int64) (typ uint16, length uint32, err error) { + b := make([]byte, headerSize) + if n, err := r.r.ReadAt(b, off); err != nil { + if err == io.EOF && n > 0 { + return 0, 0, io.ErrUnexpectedEOF + } + return 0, 0, err + } + typ = binary.LittleEndian.Uint16(b) + length = binary.LittleEndian.Uint32(b[2:]) + + // Check reserved bytes of header. + if b[6] != 0 || b[7] != 0 { + return 0, 0, fmt.Errorf("reserved bytes are non-zero") + } + + return typ, length, nil +} + +// Find returns the first entry with the matching type. +func (r *Reader) Find(want uint16) (*Entry, error) { + var ( + off int64 + typ uint16 + length uint32 + err error + ) + for { + typ, length, err = r.ReadMetadataAt(off) + if err == io.EOF { + return nil, io.EOF + } else if err != nil { + return nil, err + } + if typ == want { + var e Entry + if _, err := r.ReadAt(&e, off); err != nil { + return nil, err + } + return &e, nil + } + off += int64(headerSize + length) + } +} + +// FindAll returns all entries with the matching type. +func (r *Reader) FindAll(want uint16) ([]*Entry, error) { + var ( + off int64 + typ uint16 + length uint32 + entries []*Entry + err error + ) + for { + typ, length, err = r.ReadMetadataAt(off) + if err == io.EOF { + return entries, nil + } else if err != nil { + return entries, err + } + if typ == want { + e := new(Entry) + if _, err := r.ReadAt(e, off); err != nil { + return entries, err + } + entries = append(entries, e) + } + off += int64(headerSize + length) + } +} diff --git a/internal/era/e2store/e2store_test.go b/internal/era/e2store/e2store_test.go new file mode 100644 index 0000000000..febcffe4cf --- /dev/null +++ b/internal/era/e2store/e2store_test.go @@ -0,0 +1,150 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package e2store + +import ( + "bytes" + "fmt" + "io" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestEncode(t *testing.T) { + for _, test := range []struct { + entries []Entry + want string + name string + }{ + { + name: "emptyEntry", + entries: []Entry{{0xffff, nil}}, + want: "ffff000000000000", + }, + { + name: "beef", + entries: []Entry{{42, common.Hex2Bytes("beef")}}, + want: "2a00020000000000beef", + }, + { + name: "twoEntries", + entries: []Entry{ + {42, common.Hex2Bytes("beef")}, + {9, common.Hex2Bytes("abcdabcd")}, + }, + want: "2a00020000000000beef0900040000000000abcdabcd", + }, + } { + tt := test + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + var ( + b = bytes.NewBuffer(nil) + w = NewWriter(b) + ) + for _, e := range tt.entries { + if _, err := w.Write(e.Type, e.Value); err != nil { + t.Fatalf("encoding error: %v", err) + } + } + if want, have := common.FromHex(tt.want), b.Bytes(); !bytes.Equal(want, have) { + t.Fatalf("encoding mismatch (want %x, have %x", want, have) + } + r := NewReader(bytes.NewReader(b.Bytes())) + for _, want := range tt.entries { + have, err := r.Read() + if err != nil { + t.Fatalf("decoding error: %v", err) + } + if have.Type != want.Type { + t.Fatalf("decoded entry does type mismatch (want %v, got %v)", want.Type, have.Type) + } + if !bytes.Equal(have.Value, want.Value) { + t.Fatalf("decoded entry does not match (want %#x, got %#x)", want.Value, have.Value) + } + } + }) + } +} + +func TestDecode(t *testing.T) { + for i, tt := range []struct { + have string + err error + }{ + { // basic valid decoding + have: "ffff000000000000", + }, + { // basic invalid decoding + have: "ffff000000000001", + err: fmt.Errorf("reserved bytes are non-zero"), + }, + { // no more entries to read, returns EOF + have: "", + err: io.EOF, + }, + { // malformed type + have: "bad", + err: io.ErrUnexpectedEOF, + }, + { // malformed length + have: "badbeef", + err: io.ErrUnexpectedEOF, + }, + { // specified length longer than actual value + have: "beef010000000000", + err: io.ErrUnexpectedEOF, + }, + } { + r := NewReader(bytes.NewReader(common.FromHex(tt.have))) + if tt.err != nil { + _, err := r.Read() + if err == nil && tt.err != nil { + t.Fatalf("test %d, expected error, got none", i) + } + if err != nil && tt.err == nil { + t.Fatalf("test %d, expected no error, got %v", i, err) + } + if err != nil && tt.err != nil && err.Error() != tt.err.Error() { + t.Fatalf("expected error %v, got %v", tt.err, err) + } + continue + } + } +} + +func FuzzCodec(f *testing.F) { + f.Fuzz(func(t *testing.T, input []byte) { + r := NewReader(bytes.NewReader(input)) + entry, err := r.Read() + if err != nil { + return + } + var ( + b = bytes.NewBuffer(nil) + w = NewWriter(b) + ) + w.Write(entry.Type, entry.Value) + output := b.Bytes() + // Only care about the input that was actually consumed + input = input[:r.offset] + if !bytes.Equal(input, output) { + t.Fatalf("decode-encode mismatch, input %#x output %#x", input, output) + } + }) +} diff --git a/internal/era/era.go b/internal/era/era.go new file mode 100644 index 0000000000..a0e701b7e0 --- /dev/null +++ b/internal/era/era.go @@ -0,0 +1,283 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "encoding/binary" + "fmt" + "io" + "math/big" + "os" + "path" + "strconv" + "strings" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +var ( + TypeVersion uint16 = 0x3265 + TypeCompressedHeader uint16 = 0x03 + TypeCompressedBody uint16 = 0x04 + TypeCompressedReceipts uint16 = 0x05 + TypeTotalDifficulty uint16 = 0x06 + TypeAccumulator uint16 = 0x07 + TypeBlockIndex uint16 = 0x3266 + + MaxEra1Size = 8192 +) + +// Filename returns a recognizable Era1-formatted file name for the specified +// epoch and network. +func Filename(network string, epoch int, root common.Hash) string { + return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10]) +} + +// ReadDir reads all the era1 files in a directory for a given network. +// Format: --.era1 +func ReadDir(dir, network string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("error reading directory %s: %w", dir, err) + } + var ( + next = uint64(0) + eras []string + ) + for _, entry := range entries { + if path.Ext(entry.Name()) != ".era1" { + continue + } + parts := strings.Split(entry.Name(), "-") + if len(parts) != 3 || parts[0] != network { + // invalid era1 filename, skip + continue + } + if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil { + return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name()) + } else if epoch != next { + return nil, fmt.Errorf("missing epoch %d", next) + } + next += 1 + eras = append(eras, entry.Name()) + } + return eras, nil +} + +type ReadAtSeekCloser interface { + io.ReaderAt + io.Seeker + io.Closer +} + +// Era reads and Era1 file. +type Era struct { + f ReadAtSeekCloser // backing era1 file + s *e2store.Reader // e2store reader over f + m metadata // start, count, length info + mu *sync.Mutex // lock for buf + buf [8]byte // buffer reading entry offsets +} + +// From returns an Era backed by f. +func From(f ReadAtSeekCloser) (*Era, error) { + m, err := readMetadata(f) + if err != nil { + return nil, err + } + return &Era{ + f: f, + s: e2store.NewReader(f), + m: m, + mu: new(sync.Mutex), + }, nil +} + +// Open returns an Era backed by the given filename. +func Open(filename string) (*Era, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + return From(f) +} + +func (e *Era) Close() error { + return e.f.Close() +} + +func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds") + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + r, n, err := newSnappyReader(e.s, TypeCompressedHeader, off) + if err != nil { + return nil, err + } + var header types.Header + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + r, _, err = newSnappyReader(e.s, TypeCompressedBody, off) + if err != nil { + return nil, err + } + var body types.Body + if err := rlp.Decode(r, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil +} + +// Accumulator reads the accumulator entry in the Era1 file. +func (e *Era) Accumulator() (common.Hash, error) { + entry, err := e.s.Find(TypeAccumulator) + if err != nil { + return common.Hash{}, err + } + return common.BytesToHash(entry.Value), nil +} + +// InitialTD returns initial total difficulty before the difficulty of the +// first block of the Era1 is applied. +func (e *Era) InitialTD() (*big.Int, error) { + var ( + r io.Reader + header types.Header + rawTd []byte + n int64 + off int64 + err error + ) + + // Read first header. + if off, err = e.readOffset(e.m.start); err != nil { + return nil, err + } + if r, n, err = newSnappyReader(e.s, TypeCompressedHeader, off); err != nil { + return nil, err + } + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + + // Skip over next two records. + for i := 0; i < 2; i++ { + length, err := e.s.LengthAt(off) + if err != nil { + return nil, err + } + off += length + } + + // Read total difficulty after first block. + if r, _, err = e.s.ReaderAt(TypeTotalDifficulty, off); err != nil { + return nil, err + } + rawTd, err = io.ReadAll(r) + if err != nil { + return nil, err + } + td := new(big.Int).SetBytes(reverseOrder(rawTd)) + return td.Sub(td, header.Difficulty), nil +} + +// Start returns the listed start block. +func (e *Era) Start() uint64 { + return e.m.start +} + +// Count returns the total number of blocks in the Era1. +func (e *Era) Count() uint64 { + return e.m.count +} + +// readOffset reads a specific block's offset from the block index. The value n +// is the absolute block number desired. +func (e *Era) readOffset(n uint64) (int64, error) { + var ( + blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header + firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num + indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes + offOffset = firstIndex + indexOffset // offset of block offset + ) + e.mu.Lock() + defer e.mu.Unlock() + clearBuffer(e.buf[:]) + if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil { + return 0, err + } + // Since the block offset is relative from the start of the block index record + // we need to add the record offset to it's offset to get the block's absolute + // offset. + return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil +} + +// newReader returns a snappy.Reader for the e2store entry value at off. +func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) { + r, n, err := e.ReaderAt(expectedType, off) + if err != nil { + return nil, 0, err + } + return snappy.NewReader(r), int64(n), err +} + +// clearBuffer zeroes out the buffer. +func clearBuffer(buf []byte) { + for i := 0; i < len(buf); i++ { + buf[i] = 0 + } +} + +// metadata wraps the metadata in the block index. +type metadata struct { + start uint64 + count uint64 + length int64 +} + +// readMetadata reads the metadata stored in an Era1 file's block index. +func readMetadata(f ReadAtSeekCloser) (m metadata, err error) { + // Determine length of reader. + if m.length, err = f.Seek(0, io.SeekEnd); err != nil { + return + } + b := make([]byte, 16) + // Read count. It's the last 8 bytes of the file. + if _, err = f.ReadAt(b[:8], m.length-8); err != nil { + return + } + m.count = binary.LittleEndian.Uint64(b) + // Read start. It's at the offset -sizeof(m.count) - + // count*sizeof(indexEntry) - sizeof(m.start) + if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil { + return + } + m.start = binary.LittleEndian.Uint64(b[8:]) + return +} diff --git a/internal/era/era_test.go b/internal/era/era_test.go new file mode 100644 index 0000000000..ee5d9e82a0 --- /dev/null +++ b/internal/era/era_test.go @@ -0,0 +1,142 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "bytes" + "io" + "math/big" + "os" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +type testchain struct { + headers [][]byte + bodies [][]byte + receipts [][]byte + tds []*big.Int +} + +func TestEra1Builder(t *testing.T) { + // Get temp directory. + f, err := os.CreateTemp("", "era1-test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer f.Close() + + var ( + builder = NewBuilder(f) + chain = testchain{} + ) + for i := 0; i < 128; i++ { + chain.headers = append(chain.headers, []byte{byte('h'), byte(i)}) + chain.bodies = append(chain.bodies, []byte{byte('b'), byte(i)}) + chain.receipts = append(chain.receipts, []byte{byte('r'), byte(i)}) + chain.tds = append(chain.tds, big.NewInt(int64(i))) + } + + // Write blocks to Era1. + for i := 0; i < len(chain.headers); i++ { + var ( + header = chain.headers[i] + body = chain.bodies[i] + receipts = chain.receipts[i] + hash = common.Hash{byte(i)} + td = chain.tds[i] + ) + if err = builder.AddRLP(header, body, receipts, uint64(i), hash, td, big.NewInt(1)); err != nil { + t.Fatalf("error adding entry: %v", err) + } + } + + // Finalize Era1. + if _, err := builder.Finalize(); err != nil { + t.Fatalf("error finalizing era1: %v", err) + } + + // Verify Era1 contents. + e, err := Open(f.Name()) + if err != nil { + t.Fatalf("failed to open era: %v", err) + } + it, err := NewRawIterator(e) + if err != nil { + t.Fatalf("failed to make iterator: %s", err) + } + for i := uint64(0); i < uint64(len(chain.headers)); i++ { + if !it.Next() { + t.Fatalf("expected more entries") + } + if it.Error() != nil { + t.Fatalf("unexpected error %v", it.Error()) + } + // Check headers. + header, err := io.ReadAll(it.Header) + if err != nil { + t.Fatalf("error reading header: %v", err) + } + if !bytes.Equal(header, chain.headers[i]) { + t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], header) + } + // Check bodies. + body, err := io.ReadAll(it.Body) + if err != nil { + t.Fatalf("error reading body: %v", err) + } + if !bytes.Equal(body, chain.bodies[i]) { + t.Fatalf("mismatched body: want %s, got %s", chain.bodies[i], body) + } + // Check receipts. + receipts, err := io.ReadAll(it.Receipts) + if err != nil { + t.Fatalf("error reading receipts: %v", err) + } + if !bytes.Equal(receipts, chain.receipts[i]) { + t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], receipts) + } + + // Check total difficulty. + rawTd, err := io.ReadAll(it.TotalDifficulty) + if err != nil { + t.Fatalf("error reading td: %v", err) + } + td := new(big.Int).SetBytes(reverseOrder(rawTd)) + if td.Cmp(chain.tds[i]) != 0 { + t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td) + } + } +} + +func TestEraFilename(t *testing.T) { + for i, tt := range []struct { + network string + epoch int + root common.Hash + expected string + }{ + {"mainnet", 1, common.Hash{1}, "mainnet-00001-01000000.era1"}, + {"goerli", 99999, common.HexToHash("0xdeadbeef00000000000000000000000000000000000000000000000000000000"), "goerli-99999-deadbeef.era1"}, + } { + got := Filename(tt.network, tt.epoch, tt.root) + if tt.expected != got { + t.Errorf("test %d: invalid filename: want %s, got %s", i, tt.expected, got) + } + } +} diff --git a/internal/era/iterator.go b/internal/era/iterator.go new file mode 100644 index 0000000000..e74a8154b1 --- /dev/null +++ b/internal/era/iterator.go @@ -0,0 +1,197 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// Iterator wraps RawIterator and returns decoded Era1 entries. +type Iterator struct { + inner *RawIterator +} + +// NewRawIterator returns a new Iterator instance. Next must be immediately +// called on new iterators to load the first item. +func NewIterator(e *Era) (*Iterator, error) { + inner, err := NewRawIterator(e) + if err != nil { + return nil, err + } + return &Iterator{inner}, nil +} + +// Next moves the iterator to the next block entry. It returns false when all +// items have been read or an error has halted its progress. Block, Receipts, +// and BlockAndReceipts should no longer be called after false is returned. +func (it *Iterator) Next() bool { + return it.inner.Next() +} + +// Number returns the current number block the iterator will return. +func (it *Iterator) Number() uint64 { + return it.inner.next - 1 +} + +// Error returns the error status of the iterator. It should be called before +// reading from any of the iterator's values. +func (it *Iterator) Error() error { + return it.inner.Error() +} + +// Block returns the block for the iterator's current position. +func (it *Iterator) Block() (*types.Block, error) { + if it.inner.Header == nil || it.inner.Body == nil { + return nil, fmt.Errorf("header and body must be non-nil") + } + var ( + header types.Header + body types.Body + ) + if err := rlp.Decode(it.inner.Header, &header); err != nil { + return nil, err + } + if err := rlp.Decode(it.inner.Body, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil +} + +// Receipts returns the receipts for the iterator's current position. +func (it *Iterator) Receipts() (types.Receipts, error) { + if it.inner.Receipts == nil { + return nil, fmt.Errorf("receipts must be non-nil") + } + var receipts types.Receipts + err := rlp.Decode(it.inner.Receipts, &receipts) + return receipts, err +} + +// BlockAndReceipts returns the block and receipts for the iterator's current +// position. +func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) { + b, err := it.Block() + if err != nil { + return nil, nil, err + } + r, err := it.Receipts() + if err != nil { + return nil, nil, err + } + return b, r, nil +} + +// TotalDifficulty returns the total difficulty for the iterator's current +// position. +func (it *Iterator) TotalDifficulty() (*big.Int, error) { + td, err := io.ReadAll(it.inner.TotalDifficulty) + if err != nil { + return nil, err + } + return new(big.Int).SetBytes(reverseOrder(td)), nil +} + +// RawIterator reads an RLP-encode Era1 entries. +type RawIterator struct { + e *Era // backing Era1 + next uint64 // next block to read + err error // last error + + Header io.Reader + Body io.Reader + Receipts io.Reader + TotalDifficulty io.Reader +} + +// NewRawIterator returns a new RawIterator instance. Next must be immediately +// called on new iterators to load the first item. +func NewRawIterator(e *Era) (*RawIterator, error) { + return &RawIterator{ + e: e, + next: e.m.start, + }, nil +} + +// Next moves the iterator to the next block entry. It returns false when all +// items have been read or an error has halted its progress. Header, Body, +// Receipts, TotalDifficulty will be set to nil in the case returning false or +// finding an error and should therefore no longer be read from. +func (it *RawIterator) Next() bool { + // Clear old errors. + it.err = nil + if it.e.m.start+it.e.m.count <= it.next { + it.clear() + return false + } + off, err := it.e.readOffset(it.next) + if err != nil { + // Error here means block index is corrupted, so don't + // continue. + it.clear() + it.err = err + return false + } + var n int64 + if it.Header, n, it.err = newSnappyReader(it.e.s, TypeCompressedHeader, off); it.err != nil { + it.clear() + return true + } + off += n + if it.Body, n, it.err = newSnappyReader(it.e.s, TypeCompressedBody, off); it.err != nil { + it.clear() + return true + } + off += n + if it.Receipts, n, it.err = newSnappyReader(it.e.s, TypeCompressedReceipts, off); it.err != nil { + it.clear() + return true + } + off += n + if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(TypeTotalDifficulty, off); it.err != nil { + it.clear() + return true + } + it.next += 1 + return true +} + +// Number returns the current number block the iterator will return. +func (it *RawIterator) Number() uint64 { + return it.next - 1 +} + +// Error returns the error status of the iterator. It should be called before +// reading from any of the iterator's values. +func (it *RawIterator) Error() error { + if it.err == io.EOF { + return nil + } + return it.err +} + +// clear sets all the outputs to nil. +func (it *RawIterator) clear() { + it.Header = nil + it.Body = nil + it.Receipts = nil + it.TotalDifficulty = nil +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 898e70ea63..c623647a91 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -27,7 +27,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/accounts/scwallet" "github.com/ethereum/go-ethereum/common" @@ -48,6 +47,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" "github.com/tyler-smith/go-bip39" ) @@ -134,26 +134,28 @@ func (s *EthereumAPI) Syncing() (interface{}, error) { progress := s.b.SyncProgress() // Return not syncing if the synchronisation already completed - if progress.CurrentBlock >= progress.HighestBlock { + if progress.Done() { return false, nil } // Otherwise gather the block sync stats return map[string]interface{}{ - "startingBlock": hexutil.Uint64(progress.StartingBlock), - "currentBlock": hexutil.Uint64(progress.CurrentBlock), - "highestBlock": hexutil.Uint64(progress.HighestBlock), - "syncedAccounts": hexutil.Uint64(progress.SyncedAccounts), - "syncedAccountBytes": hexutil.Uint64(progress.SyncedAccountBytes), - "syncedBytecodes": hexutil.Uint64(progress.SyncedBytecodes), - "syncedBytecodeBytes": hexutil.Uint64(progress.SyncedBytecodeBytes), - "syncedStorage": hexutil.Uint64(progress.SyncedStorage), - "syncedStorageBytes": hexutil.Uint64(progress.SyncedStorageBytes), - "healedTrienodes": hexutil.Uint64(progress.HealedTrienodes), - "healedTrienodeBytes": hexutil.Uint64(progress.HealedTrienodeBytes), - "healedBytecodes": hexutil.Uint64(progress.HealedBytecodes), - "healedBytecodeBytes": hexutil.Uint64(progress.HealedBytecodeBytes), - "healingTrienodes": hexutil.Uint64(progress.HealingTrienodes), - "healingBytecode": hexutil.Uint64(progress.HealingBytecode), + "startingBlock": hexutil.Uint64(progress.StartingBlock), + "currentBlock": hexutil.Uint64(progress.CurrentBlock), + "highestBlock": hexutil.Uint64(progress.HighestBlock), + "syncedAccounts": hexutil.Uint64(progress.SyncedAccounts), + "syncedAccountBytes": hexutil.Uint64(progress.SyncedAccountBytes), + "syncedBytecodes": hexutil.Uint64(progress.SyncedBytecodes), + "syncedBytecodeBytes": hexutil.Uint64(progress.SyncedBytecodeBytes), + "syncedStorage": hexutil.Uint64(progress.SyncedStorage), + "syncedStorageBytes": hexutil.Uint64(progress.SyncedStorageBytes), + "healedTrienodes": hexutil.Uint64(progress.HealedTrienodes), + "healedTrienodeBytes": hexutil.Uint64(progress.HealedTrienodeBytes), + "healedBytecodes": hexutil.Uint64(progress.HealedBytecodes), + "healedBytecodeBytes": hexutil.Uint64(progress.HealedBytecodeBytes), + "healingTrienodes": hexutil.Uint64(progress.HealingTrienodes), + "healingBytecode": hexutil.Uint64(progress.HealingBytecode), + "txIndexFinishedBlocks": hexutil.Uint64(progress.TxIndexFinishedBlocks), + "txIndexRemainingBlocks": hexutil.Uint64(progress.TxIndexRemainingBlocks), }, nil } @@ -649,10 +651,11 @@ func (s *BlockChainAPI) GetBalance(ctx context.Context, address common.Address, if state == nil || err != nil { return nil, err } - return (*hexutil.Big)(state.GetBalance(address)), state.Error() + b := state.GetBalance(address).ToBig() + return (*hexutil.Big)(b), state.Error() } -// Result structs for GetProof +// AccountResult structs for GetProof type AccountResult struct { Address common.Address `json:"address"` AccountProof []string `json:"accountProof"` @@ -747,10 +750,11 @@ func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, st if err := tr.Prove(crypto.Keccak256(address.Bytes()), &accountProof); err != nil { return nil, err } + balance := statedb.GetBalance(address).ToBig() return &AccountResult{ Address: address, AccountProof: accountProof, - Balance: (*hexutil.Big)(statedb.GetBalance(address)), + Balance: (*hexutil.Big)(balance), CodeHash: codeHash, Nonce: hexutil.Uint64(statedb.GetNonce(address)), StorageHash: storageRoot, @@ -973,7 +977,8 @@ func (diff *StateOverride) Apply(state *state.StateDB) error { } // Override account balance. if account.Balance != nil { - state.SetBalance(addr, (*big.Int)(*account.Balance)) + u256Balance, _ := uint256.FromBig((*big.Int)(*account.Balance)) + state.SetBalance(addr, u256Balance) } if account.State != nil && account.StateDiff != nil { return fmt.Errorf("account %s has both 'state' and 'stateDiff'", addr.Hex()) @@ -1133,37 +1138,6 @@ func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash return doCall(ctx, b, args, state, header, overrides, blockOverrides, timeout, globalGasCap) } -func newRevertError(revert []byte) *revertError { - err := vm.ErrExecutionReverted - - reason, errUnpack := abi.UnpackRevert(revert) - if errUnpack == nil { - err = fmt.Errorf("%w: %v", vm.ErrExecutionReverted, reason) - } - return &revertError{ - error: err, - reason: hexutil.Encode(revert), - } -} - -// revertError is an API error that encompasses an EVM revertal with JSON error -// code and a binary data blob. -type revertError struct { - error - reason string // revert reason hex encoded -} - -// ErrorCode returns the JSON error code for a revertal. -// See: https://github.com/ethereum/wiki/wiki/JSON-RPC-Error-Codes-Improvement-Proposal -func (e *revertError) ErrorCode() int { - return 3 -} - -// ErrorData returns the hex encoded revert reason. -func (e *revertError) ErrorData() interface{} { - return e.reason -} - // Call executes the given transaction on the state for the given block number. // // Additionally, the caller can specify a batch of contract for fields overriding. @@ -1647,50 +1621,48 @@ func (s *TransactionAPI) GetTransactionCount(ctx context.Context, address common // GetTransactionByHash returns the transaction for the given hash func (s *TransactionAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { // Try to return an already finalized transaction - tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) + found, tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) + if !found { + // No finalized transaction, try to retrieve it from the pool + if tx := s.b.GetPoolTransaction(hash); tx != nil { + return NewRPCPendingTransaction(tx, s.b.CurrentHeader(), s.b.ChainConfig()), nil + } + if err == nil { + return nil, nil + } + return nil, NewTxIndexingError() + } + header, err := s.b.HeaderByHash(ctx, blockHash) if err != nil { return nil, err } - if tx != nil { - header, err := s.b.HeaderByHash(ctx, blockHash) - if err != nil { - return nil, err - } - return newRPCTransaction(tx, blockHash, blockNumber, header.Time, index, header.BaseFee, s.b.ChainConfig()), nil - } - // No finalized transaction, try to retrieve it from the pool - if tx := s.b.GetPoolTransaction(hash); tx != nil { - return NewRPCPendingTransaction(tx, s.b.CurrentHeader(), s.b.ChainConfig()), nil - } - - // Transaction unknown, return as such - return nil, nil + return newRPCTransaction(tx, blockHash, blockNumber, header.Time, index, header.BaseFee, s.b.ChainConfig()), nil } // GetRawTransactionByHash returns the bytes of the transaction for the given hash. func (s *TransactionAPI) GetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { // Retrieve a finalized transaction, or a pooled otherwise - tx, _, _, _, err := s.b.GetTransaction(ctx, hash) - if err != nil { - return nil, err - } - if tx == nil { - if tx = s.b.GetPoolTransaction(hash); tx == nil { - // Transaction not found anywhere, abort + found, tx, _, _, _, err := s.b.GetTransaction(ctx, hash) + if !found { + if tx = s.b.GetPoolTransaction(hash); tx != nil { + return tx.MarshalBinary() + } + if err == nil { return nil, nil } + return nil, NewTxIndexingError() } - // Serialize to RLP and return return tx.MarshalBinary() } // GetTransactionReceipt returns the transaction receipt for the given transaction hash. func (s *TransactionAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { - tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) - if tx == nil || err != nil { - // When the transaction doesn't exist, the RPC method should return JSON null - // as per specification. - return nil, nil + found, tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) + if err != nil { + return nil, NewTxIndexingError() // transaction is not fully indexed + } + if !found { + return nil, nil // transaction is not existent or reachable } header, err := s.b.HeaderByHash(ctx, blockHash) if err != nil { @@ -1835,13 +1807,14 @@ func (s *TransactionAPI) SendTransaction(ctx context.Context, args TransactionAr // on a given unsigned transaction, and returns it to the caller for further // processing (signing + broadcast). func (s *TransactionAPI) FillTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { + args.blobSidecarAllowed = true + // Set some sanity defaults and terminate on failure if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Assemble the transaction and obtain rlp tx := args.toTransaction() - // TODO(s1na): fill in blob proofs, commitments data, err := tx.MarshalBinary() if err != nil { return nil, err @@ -2080,15 +2053,15 @@ func (api *DebugAPI) GetRawReceipts(ctx context.Context, blockNrOrHash rpc.Block // GetRawTransaction returns the bytes of the transaction for the given hash. func (s *DebugAPI) GetRawTransaction(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { // Retrieve a finalized transaction, or a pooled otherwise - tx, _, _, _, err := s.b.GetTransaction(ctx, hash) - if err != nil { - return nil, err - } - if tx == nil { - if tx = s.b.GetPoolTransaction(hash); tx == nil { - // Transaction not found anywhere, abort + found, tx, _, _, _, err := s.b.GetTransaction(ctx, hash) + if !found { + if tx = s.b.GetPoolTransaction(hash); tx != nil { + return tx.MarshalBinary() + } + if err == nil { return nil, nil } + return nil, NewTxIndexingError() } return tx.MarshalBinary() } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index fd68650193..9328b7e67e 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "crypto/ecdsa" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -45,6 +46,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/blocktest" @@ -583,9 +585,9 @@ func (b testBackend) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) even func (b testBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { panic("implement me") } -func (b testBackend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { +func (b testBackend) GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) { tx, blockHash, blockNumber, index := rawdb.ReadTransaction(b.db, txHash) - return tx, blockHash, blockNumber, index, nil + return true, tx, blockHash, blockNumber, index, nil } func (b testBackend) GetPoolTransactions() (types.Transactions, error) { panic("implement me") } func (b testBackend) GetPoolTransaction(txHash common.Hash) *types.Transaction { panic("implement me") } @@ -1079,6 +1081,195 @@ func TestSendBlobTransaction(t *testing.T) { } } +func TestFillBlobTransaction(t *testing.T) { + t.Parallel() + // Initialize test accounts + var ( + key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + to = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.MergedTestChainConfig, + Alloc: core.GenesisAlloc{}, + } + emptyBlob = kzg4844.Blob{} + emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) + emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) + emptyBlobHash common.Hash = kzg4844.CalcBlobHashV1(sha256.New(), &emptyBlobCommit) + ) + b := newTestBackend(t, 1, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { + b.SetPoS() + }) + api := NewTransactionAPI(b, nil) + type result struct { + Hashes []common.Hash + Sidecar *types.BlobTxSidecar + } + suite := []struct { + name string + args TransactionArgs + err string + want *result + }{ + { + name: "TestInvalidParamsCombination1", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}}, + Proofs: []kzg4844.Proof{{}}, + }, + err: `blob proofs provided while commitments were not`, + }, + { + name: "TestInvalidParamsCombination2", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}}, + Commitments: []kzg4844.Commitment{{}}, + }, + err: `blob commitments provided while proofs were not`, + }, + { + name: "TestInvalidParamsCount1", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}}, + Commitments: []kzg4844.Commitment{{}, {}}, + Proofs: []kzg4844.Proof{{}, {}}, + }, + err: `number of blobs and commitments mismatch (have=2, want=1)`, + }, + { + name: "TestInvalidParamsCount2", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}, {}}, + Commitments: []kzg4844.Commitment{{}, {}}, + Proofs: []kzg4844.Proof{{}}, + }, + err: `number of blobs and proofs mismatch (have=1, want=2)`, + }, + { + name: "TestInvalidProofVerification", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}, {}}, + Commitments: []kzg4844.Commitment{{}, {}}, + Proofs: []kzg4844.Proof{{}, {}}, + }, + err: `failed to verify blob proof: short buffer`, + }, + { + name: "TestGenerateBlobHashes", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + want: &result{ + Hashes: []common.Hash{emptyBlobHash}, + Sidecar: &types.BlobTxSidecar{ + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + }, + }, + { + name: "TestValidBlobHashes", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + BlobHashes: []common.Hash{emptyBlobHash}, + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + want: &result{ + Hashes: []common.Hash{emptyBlobHash}, + Sidecar: &types.BlobTxSidecar{ + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + }, + }, + { + name: "TestInvalidBlobHashes", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + BlobHashes: []common.Hash{{0x01, 0x22}}, + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + err: fmt.Sprintf("blob hash verification failed (have=%s, want=%s)", common.Hash{0x01, 0x22}, emptyBlobHash), + }, + { + name: "TestGenerateBlobProofs", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{emptyBlob}, + }, + want: &result{ + Hashes: []common.Hash{emptyBlobHash}, + Sidecar: &types.BlobTxSidecar{ + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + }, + }, + } + for _, tc := range suite { + t.Run(tc.name, func(t *testing.T) { + res, err := api.FillTransaction(context.Background(), tc.args) + if len(tc.err) > 0 { + if err == nil { + t.Fatalf("missing error. want: %s", tc.err) + } else if err != nil && err.Error() != tc.err { + t.Fatalf("error mismatch. want: %s, have: %s", tc.err, err.Error()) + } + return + } + if err != nil && len(tc.err) == 0 { + t.Fatalf("expected no error. have: %s", err) + } + if res == nil { + t.Fatal("result missing") + } + want, err := json.Marshal(tc.want) + if err != nil { + t.Fatalf("failed to encode expected: %v", err) + } + have, err := json.Marshal(result{Hashes: res.Tx.BlobHashes(), Sidecar: res.Tx.BlobTxSidecar()}) + if err != nil { + t.Fatalf("failed to encode computed sidecar: %v", err) + } + if !bytes.Equal(have, want) { + t.Errorf("blob sidecar mismatch. Have: %s, want: %s", have, want) + } + }) + } +} + func argsFromTransaction(tx *types.Transaction, from common.Address) TransactionArgs { var ( gas = tx.Gas() diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 50f338f5ca..5f408ba20b 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -75,7 +75,7 @@ type Backend interface { // Transaction pool API SendTx(ctx context.Context, signedTx *types.Transaction) error - GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) + GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) GetPoolTransactions() (types.Transactions, error) GetPoolTransaction(txHash common.Hash) *types.Transaction GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) diff --git a/internal/ethapi/errors.go b/internal/ethapi/errors.go new file mode 100644 index 0000000000..b5e668a805 --- /dev/null +++ b/internal/ethapi/errors.go @@ -0,0 +1,78 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/vm" +) + +// revertError is an API error that encompasses an EVM revert with JSON error +// code and a binary data blob. +type revertError struct { + error + reason string // revert reason hex encoded +} + +// ErrorCode returns the JSON error code for a revert. +// See: https://github.com/ethereum/wiki/wiki/JSON-RPC-Error-Codes-Improvement-Proposal +func (e *revertError) ErrorCode() int { + return 3 +} + +// ErrorData returns the hex encoded revert reason. +func (e *revertError) ErrorData() interface{} { + return e.reason +} + +// newRevertError creates a revertError instance with the provided revert data. +func newRevertError(revert []byte) *revertError { + err := vm.ErrExecutionReverted + + reason, errUnpack := abi.UnpackRevert(revert) + if errUnpack == nil { + err = fmt.Errorf("%w: %v", vm.ErrExecutionReverted, reason) + } + return &revertError{ + error: err, + reason: hexutil.Encode(revert), + } +} + +// TxIndexingError is an API error that indicates the transaction indexing is not +// fully finished yet with JSON error code and a binary data blob. +type TxIndexingError struct{} + +// NewTxIndexingError creates a TxIndexingError instance. +func NewTxIndexingError() *TxIndexingError { return &TxIndexingError{} } + +// Error implement error interface, returning the error message. +func (e *TxIndexingError) Error() string { + return "transaction indexing is in progress" +} + +// ErrorCode returns the JSON error code for a revert. +// See: https://github.com/ethereum/wiki/wiki/JSON-RPC-Error-Codes-Improvement-Proposal +func (e *TxIndexingError) ErrorCode() int { + return -32000 // to be decided +} + +// ErrorData returns the hex encoded revert reason. +func (e *TxIndexingError) ErrorData() interface{} { return "transaction indexing is in progress" } diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index d4b7092232..57ce4cdf5b 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -19,6 +19,7 @@ package ethapi import ( "bytes" "context" + "crypto/sha256" "errors" "fmt" "math/big" @@ -29,11 +30,17 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/holiman/uint256" ) +var ( + maxBlobsPerTransaction = params.MaxBlobGasPerBlock / params.BlobTxBlobGasPerBlob +) + // TransactionArgs represents the arguments to construct a new transaction // or a message call. type TransactionArgs struct { @@ -56,9 +63,17 @@ type TransactionArgs struct { AccessList *types.AccessList `json:"accessList,omitempty"` ChainID *hexutil.Big `json:"chainId,omitempty"` - // Introduced by EIP-4844. + // For BlobTxType BlobFeeCap *hexutil.Big `json:"maxFeePerBlobGas"` BlobHashes []common.Hash `json:"blobVersionedHashes,omitempty"` + + // For BlobTxType transactions with blob sidecar + Blobs []kzg4844.Blob `json:"blobs"` + Commitments []kzg4844.Commitment `json:"commitments"` + Proofs []kzg4844.Proof `json:"proofs"` + + // This configures whether blobs are allowed to be passed. + blobSidecarAllowed bool } // from retrieves the transaction sender address. @@ -82,9 +97,13 @@ func (args *TransactionArgs) data() []byte { // setDefaults fills in default values for unspecified tx fields. func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, infiniteGas bool) error { + if err := args.setBlobTxSidecar(ctx, b); err != nil { + return err + } if err := args.setFeeDefaults(ctx, b); err != nil { return err } + if args.Value == nil { args.Value = new(hexutil.Big) } @@ -98,14 +117,23 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, infinit if args.Data != nil && args.Input != nil && !bytes.Equal(*args.Data, *args.Input) { return errors.New(`both "data" and "input" are set and not equal. Please use "input" to pass transaction call data`) } - if args.BlobHashes != nil && args.To == nil { - return errors.New(`blob transactions cannot have the form of a create transaction`) - } + + // BlobTx fields if args.BlobHashes != nil && len(args.BlobHashes) == 0 { return errors.New(`need at least 1 blob for a blob transaction`) } - if args.To == nil && len(args.data()) == 0 { - return errors.New(`contract creation without any data provided`) + if args.BlobHashes != nil && len(args.BlobHashes) > maxBlobsPerTransaction { + return fmt.Errorf(`too many blobs in transaction (have=%d, max=%d)`, len(args.BlobHashes), maxBlobsPerTransaction) + } + + // create check + if args.To == nil { + if args.BlobHashes != nil { + return errors.New(`missing "to" in blob transaction`) + } + if len(args.data()) == 0 { + return errors.New(`contract creation without any data provided`) + } } // Assign a very high gas limit for cases where an accurate gas limit is not critical, // but need to ensure that gas is sufficient. @@ -113,6 +141,7 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, infinit tmp := hexutil.Uint64(math.MaxUint64 / 2) args.Gas = &tmp } + // Estimate the gas usage if necessary. if args.Gas == nil { // These fields are immutable during the estimation, safe to @@ -127,6 +156,8 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, infinit Value: args.Value, Data: (*hexutil.Bytes)(&data), AccessList: args.AccessList, + BlobFeeCap: args.BlobFeeCap, + BlobHashes: args.BlobHashes, } latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, b.RPCGasCap()) @@ -136,6 +167,7 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, infinit args.Gas = &estimated log.Trace("Estimate gas usage automatically", "gas", args.Gas) } + // If chain id is provided, ensure it matches the local chain id. Otherwise, set the local // chain id as the default. want := b.ChainConfig().ChainID @@ -171,10 +203,12 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro } return nil // No need to set anything, user already set MaxFeePerGas and MaxPriorityFeePerGas } + // Sanity check the EIP-4844 fee parameters. if args.BlobFeeCap != nil && args.BlobFeeCap.ToInt().Sign() == 0 { return errors.New("maxFeePerBlobGas must be non-zero") } + // Sanity check the non-EIP-1559 fee parameters. head := b.CurrentHeader() isLondon := b.ChainConfig().IsLondon(head.Number) @@ -256,6 +290,81 @@ func (args *TransactionArgs) setLondonFeeDefaults(ctx context.Context, head *typ return nil } +// setBlobTxSidecar adds the blob tx +func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, b Backend) error { + // No blobs, we're done. + if args.Blobs == nil { + return nil + } + + // Passing blobs is not allowed in all contexts, only in specific methods. + if !args.blobSidecarAllowed { + return errors.New(`"blobs" is not supported for this RPC method`) + } + + n := len(args.Blobs) + // Assume user provides either only blobs (w/o hashes), or + // blobs together with commitments and proofs. + if args.Commitments == nil && args.Proofs != nil { + return errors.New(`blob proofs provided while commitments were not`) + } else if args.Commitments != nil && args.Proofs == nil { + return errors.New(`blob commitments provided while proofs were not`) + } + + // len(blobs) == len(commitments) == len(proofs) == len(hashes) + if args.Commitments != nil && len(args.Commitments) != n { + return fmt.Errorf("number of blobs and commitments mismatch (have=%d, want=%d)", len(args.Commitments), n) + } + if args.Proofs != nil && len(args.Proofs) != n { + return fmt.Errorf("number of blobs and proofs mismatch (have=%d, want=%d)", len(args.Proofs), n) + } + if args.BlobHashes != nil && len(args.BlobHashes) != n { + return fmt.Errorf("number of blobs and hashes mismatch (have=%d, want=%d)", len(args.BlobHashes), n) + } + + if args.Commitments == nil { + // Generate commitment and proof. + commitments := make([]kzg4844.Commitment, n) + proofs := make([]kzg4844.Proof, n) + for i, b := range args.Blobs { + c, err := kzg4844.BlobToCommitment(b) + if err != nil { + return fmt.Errorf("blobs[%d]: error computing commitment: %v", i, err) + } + commitments[i] = c + p, err := kzg4844.ComputeBlobProof(b, c) + if err != nil { + return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) + } + proofs[i] = p + } + args.Commitments = commitments + args.Proofs = proofs + } else { + for i, b := range args.Blobs { + if err := kzg4844.VerifyBlobProof(b, args.Commitments[i], args.Proofs[i]); err != nil { + return fmt.Errorf("failed to verify blob proof: %v", err) + } + } + } + + hashes := make([]common.Hash, n) + hasher := sha256.New() + for i, c := range args.Commitments { + hashes[i] = kzg4844.CalcBlobHashV1(hasher, &c) + } + if args.BlobHashes != nil { + for i, h := range hashes { + if h != args.BlobHashes[i] { + return fmt.Errorf("blob hash verification failed (have=%s, want=%s)", args.BlobHashes[i], h) + } + } + } else { + args.BlobHashes = hashes + } + return nil +} + // ToMessage converts the transaction arguments to the Message type used by the // core evm. This method is used in calls and traces that do not require a real // live transaction. @@ -369,6 +478,14 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { BlobHashes: args.BlobHashes, BlobFeeCap: uint256.MustFromBig((*big.Int)(args.BlobFeeCap)), } + if args.Blobs != nil { + data.(*types.BlobTx).Sidecar = &types.BlobTxSidecar{ + Blobs: args.Blobs, + Commitments: args.Commitments, + Proofs: args.Proofs, + } + } + case args.MaxFeePerGas != nil: al := types.AccessList{} if args.AccessList != nil { @@ -385,6 +502,7 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { Data: args.data(), AccessList: al, } + case args.AccessList != nil: data = &types.AccessListTx{ To: args.To, @@ -396,6 +514,7 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { Data: args.data(), AccessList: *args.AccessList, } + default: data = &types.LegacyTx{ To: args.To, @@ -409,12 +528,6 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { return types.NewTx(data) } -// ToTransaction converts the arguments to a transaction. -// This assumes that setDefaults has been called. -func (args *TransactionArgs) ToTransaction() *types.Transaction { - return args.toTransaction() -} - // IsEIP4844 returns an indicator if the args contains EIP4844 fields. func (args *TransactionArgs) IsEIP4844() bool { return args.BlobHashes != nil || args.BlobFeeCap != nil diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index 8651da4020..f0fdb6d8ee 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -379,8 +379,8 @@ func (b *backendMock) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) eve return nil } func (b *backendMock) SendTx(ctx context.Context, signedTx *types.Transaction) error { return nil } -func (b *backendMock) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { - return nil, [32]byte{}, 0, 0, nil +func (b *backendMock) GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) { + return false, nil, [32]byte{}, 0, 0, nil } func (b *backendMock) GetPoolTransactions() (types.Transactions, error) { return nil, nil } func (b *backendMock) GetPoolTransaction(txHash common.Hash) *types.Transaction { return nil } diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 69e9743556..bf62c53adf 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -256,7 +256,8 @@ type BigFlag struct { Hidden bool HasBeenSet bool - Value *big.Int + Value *big.Int + defaultValue *big.Int Aliases []string EnvVars []string @@ -269,6 +270,10 @@ func (f *BigFlag) IsSet() bool { return f.HasBeenSet } func (f *BigFlag) String() string { return cli.FlagStringer(f) } func (f *BigFlag) Apply(set *flag.FlagSet) error { + // Set default value so that environment wont be able to overwrite it + if f.Value != nil { + f.defaultValue = new(big.Int).Set(f.Value) + } for _, envVar := range f.EnvVars { envVar = strings.TrimSpace(envVar) if value, found := syscall.Getenv(envVar); found { @@ -283,7 +288,6 @@ func (f *BigFlag) Apply(set *flag.FlagSet) error { f.Value = new(big.Int) set.Var((*bigValue)(f.Value), f.Name, f.Usage) }) - return nil } @@ -310,7 +314,7 @@ func (f *BigFlag) GetDefaultText() string { if f.DefaultText != "" { return f.DefaultText } - return f.GetValue() + return f.defaultValue.String() } // bigValue turns *big.Int into a flag.Value diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index 369a931e8a..0112724fa1 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -115,7 +115,7 @@ func doMigrateFlags(ctx *cli.Context) { for _, parent := range ctx.Lineage()[1:] { if parent.IsSet(name) { // When iterating across the lineage, we will be served both - // the 'canon' and alias formats of all commmands. In most cases, + // the 'canon' and alias formats of all commands. In most cases, // it's fine to set it in the ctx multiple times (one for each // name), however, the Slice-flags are not fine. // The slice-flags accumulate, so if we set it once as diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index f23c65584c..0b360e7415 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -2031,7 +2031,7 @@ var fromAscii = function(str) { * * @method transformToFullName * @param {Object} json-abi - * @return {String} full fnction/event name + * @return {String} full function/event name */ var transformToFullName = function (json) { if (json.name.indexOf('(') !== -1) { @@ -2361,7 +2361,7 @@ var isFunction = function (object) { }; /** - * Returns true if object is Objet, otherwise false + * Returns true if object is Object, otherwise false * * @method isObject * @param {Object} @@ -2757,7 +2757,7 @@ var Batch = function (web3) { * Should be called to add create new request to batch request * * @method add - * @param {Object} jsonrpc requet object + * @param {Object} jsonrpc request object */ Batch.prototype.add = function (request) { this.requests.push(request); @@ -3961,6 +3961,8 @@ var outputSyncingFormatter = function(result) { result.healedBytecodeBytes = utils.toDecimal(result.healedBytecodeBytes); result.healingTrienodes = utils.toDecimal(result.healingTrienodes); result.healingBytecode = utils.toDecimal(result.healingBytecode); + result.txIndexFinishedBlocks = utils.toDecimal(result.txIndexFinishedBlocks); + result.txIndexRemainingBlocks = utils.toDecimal(result.txIndexRemainingBlocks); return result; }; @@ -4557,7 +4559,7 @@ Iban.createIndirect = function (options) { }; /** - * Thos method should be used to check if given string is valid iban object + * This method should be used to check if given string is valid iban object * * @method isValid * @param {String} iban string @@ -6706,7 +6708,7 @@ var exchangeAbi = require('../contracts/SmartExchange.json'); * @method transfer * @param {String} from * @param {String} to iban - * @param {Value} value to be tranfered + * @param {Value} value to be transferred * @param {Function} callback, callback */ var transfer = function (eth, from, to, value, callback) { @@ -6736,7 +6738,7 @@ var transfer = function (eth, from, to, value, callback) { * @method transferToAddress * @param {String} from * @param {String} to - * @param {Value} value to be tranfered + * @param {Value} value to be transferred * @param {Function} callback, callback */ var transferToAddress = function (eth, from, to, value, callback) { @@ -7090,7 +7092,7 @@ module.exports = transfer; /** * Initializes a newly created cipher. * - * @param {number} xformMode Either the encryption or decryption transormation mode constant. + * @param {number} xformMode Either the encryption or decryption transformation mode constant. * @param {WordArray} key The key. * @param {Object} cfg (Optional) The configuration options to use for this operation. * @@ -9444,7 +9446,7 @@ module.exports = transfer; var M_offset_14 = M[offset + 14]; var M_offset_15 = M[offset + 15]; - // Working varialbes + // Working variables var a = H[0]; var b = H[1]; var c = H[2]; diff --git a/log/handler_glog.go b/log/handler_glog.go index fb1e03c5b5..f51bae2a4a 100644 --- a/log/handler_glog.go +++ b/log/handler_glog.go @@ -192,7 +192,7 @@ func (h *GlogHandler) Handle(_ context.Context, r slog.Record) error { frame, _ := fs.Next() for _, rule := range h.patterns { - if rule.pattern.MatchString(fmt.Sprintf("%+s", frame.File)) { + if rule.pattern.MatchString(fmt.Sprintf("+%s", frame.File)) { h.siteCache[r.PC], lvl, ok = rule.level, rule.level, true } } diff --git a/metrics/counter.go b/metrics/counter.go index cb81599c21..dbe8e16a90 100644 --- a/metrics/counter.go +++ b/metrics/counter.go @@ -8,7 +8,7 @@ type CounterSnapshot interface { Count() int64 } -// Counters hold an int64 value that can be incremented and decremented. +// Counter hold an int64 value that can be incremented and decremented. type Counter interface { Clear() Dec(int64) diff --git a/metrics/gauge.go b/metrics/gauge.go index 68f8f11abc..5933df3107 100644 --- a/metrics/gauge.go +++ b/metrics/gauge.go @@ -2,12 +2,12 @@ package metrics import "sync/atomic" -// gaugeSnapshot contains a readonly int64. +// GaugeSnapshot contains a readonly int64. type GaugeSnapshot interface { Value() int64 } -// Gauges hold an int64 value that can be set arbitrarily. +// Gauge holds an int64 value that can be set arbitrarily. type Gauge interface { Snapshot() GaugeSnapshot Update(int64) @@ -74,7 +74,7 @@ func (g *StandardGauge) Update(v int64) { g.value.Store(v) } -// Update updates the gauge's value if v is larger then the current valie. +// Update updates the gauge's value if v is larger then the current value. func (g *StandardGauge) UpdateIfGt(v int64) { for { exist := g.value.Load() diff --git a/metrics/gauge_float64.go b/metrics/gauge_float64.go index 967f2bc60e..c1c3c6b6e6 100644 --- a/metrics/gauge_float64.go +++ b/metrics/gauge_float64.go @@ -48,7 +48,7 @@ type gaugeFloat64Snapshot float64 // Value returns the value at the time the snapshot was taken. func (g gaugeFloat64Snapshot) Value() float64 { return float64(g) } -// NilGauge is a no-op Gauge. +// NilGaugeFloat64 is a no-op Gauge. type NilGaugeFloat64 struct{} func (NilGaugeFloat64) Snapshot() GaugeFloat64Snapshot { return NilGaugeFloat64{} } diff --git a/metrics/gauge_info.go b/metrics/gauge_info.go index c44b2d85f3..0010edc324 100644 --- a/metrics/gauge_info.go +++ b/metrics/gauge_info.go @@ -9,7 +9,7 @@ type GaugeInfoSnapshot interface { Value() GaugeInfoValue } -// GaugeInfos hold a GaugeInfoValue value that can be set arbitrarily. +// GaugeInfo holds a GaugeInfoValue value that can be set arbitrarily. type GaugeInfo interface { Update(GaugeInfoValue) Snapshot() GaugeInfoSnapshot diff --git a/metrics/healthcheck.go b/metrics/healthcheck.go index f1ae31e34a..adcd15ab58 100644 --- a/metrics/healthcheck.go +++ b/metrics/healthcheck.go @@ -1,6 +1,6 @@ package metrics -// Healthchecks hold an error value describing an arbitrary up/down status. +// Healthcheck holds an error value describing an arbitrary up/down status. type Healthcheck interface { Check() Error() error diff --git a/metrics/histogram.go b/metrics/histogram.go index 44de588bc1..10259a2463 100644 --- a/metrics/histogram.go +++ b/metrics/histogram.go @@ -4,7 +4,7 @@ type HistogramSnapshot interface { SampleSnapshot } -// Histograms calculate distribution statistics from a series of int64 values. +// Histogram calculates distribution statistics from a series of int64 values. type Histogram interface { Clear() Update(int64) diff --git a/metrics/influxdb/influxdbv2.go b/metrics/influxdb/influxdbv2.go index 0be5137d5e..114d57ae07 100644 --- a/metrics/influxdb/influxdbv2.go +++ b/metrics/influxdb/influxdbv2.go @@ -25,7 +25,7 @@ type v2Reporter struct { write api.WriteAPI } -// InfluxDBWithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags +// InfluxDBV2WithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags func InfluxDBV2WithTags(r metrics.Registry, d time.Duration, endpoint string, token string, bucket string, organization string, namespace string, tags map[string]string) { rep := &v2Reporter{ reg: r, diff --git a/miner/miner.go b/miner/miner.go index b7273948f5..58bb71b557 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -197,6 +197,11 @@ func (miner *Miner) SetExtra(extra []byte) error { return nil } +func (miner *Miner) SetGasTip(tip *big.Int) error { + miner.worker.setGasTip(tip) + return nil +} + // SetRecommitInterval sets the interval for sealing work resubmitting. func (miner *Miner) SetRecommitInterval(interval time.Duration) { miner.worker.setRecommitInterval(interval) diff --git a/miner/ordering.go b/miner/ordering.go index 4c3055f0d3..e686656bb2 100644 --- a/miner/ordering.go +++ b/miner/ordering.go @@ -119,11 +119,11 @@ func newTransactionsByPriceAndNonce(signer types.Signer, txs map[common.Address] } // Peek returns the next transaction by price. -func (t *transactionsByPriceAndNonce) Peek() *txpool.LazyTransaction { +func (t *transactionsByPriceAndNonce) Peek() (*txpool.LazyTransaction, *big.Int) { if len(t.heads) == 0 { - return nil + return nil, nil } - return t.heads[0].tx + return t.heads[0].tx, t.heads[0].fees } // Shift replaces the current best head with the next one from the same account. diff --git a/miner/ordering_test.go b/miner/ordering_test.go index e5868d7a06..d2de9b9f34 100644 --- a/miner/ordering_test.go +++ b/miner/ordering_test.go @@ -104,7 +104,7 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { txset := newTransactionsByPriceAndNonce(signer, groups, baseFee) txs := types.Transactions{} - for tx := txset.Peek(); tx != nil; tx = txset.Peek() { + for tx, _ := txset.Peek(); tx != nil; tx, _ = txset.Peek() { txs = append(txs, tx.Tx) txset.Shift() } @@ -170,7 +170,7 @@ func TestTransactionTimeSort(t *testing.T) { txset := newTransactionsByPriceAndNonce(signer, groups, nil) txs := types.Transactions{} - for tx := txset.Peek(); tx != nil; tx = txset.Peek() { + for tx, _ := txset.Peek(); tx != nil; tx, _ = txset.Peek() { txs = append(txs, tx.Tx) txset.Shift() } diff --git a/miner/payload_building.go b/miner/payload_building.go index 69ffab75b5..719736c479 100644 --- a/miner/payload_building.go +++ b/miner/payload_building.go @@ -35,12 +35,13 @@ import ( // Check engine-api specification for more details. // https://github.com/ethereum/execution-apis/blob/main/src/engine/cancun.md#payloadattributesv3 type BuildPayloadArgs struct { - Parent common.Hash // The parent block to build payload on top - Timestamp uint64 // The provided timestamp of generated payload - FeeRecipient common.Address // The provided recipient address for collecting transaction fee - Random common.Hash // The provided randomness value - Withdrawals types.Withdrawals // The provided withdrawals - BeaconRoot *common.Hash // The provided beaconRoot (Cancun) + Parent common.Hash // The parent block to build payload on top + Timestamp uint64 // The provided timestamp of generated payload + FeeRecipient common.Address // The provided recipient address for collecting transaction fee + Random common.Hash // The provided randomness value + Withdrawals types.Withdrawals // The provided withdrawals + BeaconRoot *common.Hash // The provided beaconRoot (Cancun) + Version engine.PayloadVersion // Versioning byte for payload id calculation. } // Id computes an 8-byte identifier by hashing the components of the payload arguments. @@ -57,6 +58,7 @@ func (args *BuildPayloadArgs) Id() engine.PayloadID { } var out engine.PayloadID copy(out[:], hasher.Sum(nil)[:8]) + out[0] = byte(args.Version) return out } diff --git a/miner/worker.go b/miner/worker.go index 2ed91cc187..052f34ff11 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -205,6 +205,7 @@ type worker struct { mu sync.RWMutex // The lock used to protect the coinbase and extra fields coinbase common.Address extra []byte + tip *big.Int // Minimum tip needed for non-local transaction to include them pendingMu sync.RWMutex pendingTasks map[common.Hash]*task @@ -251,6 +252,7 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus isLocalBlock: isLocalBlock, coinbase: config.Etherbase, extra: config.ExtraData, + tip: config.GasPrice, pendingTasks: make(map[common.Hash]*task), txsCh: make(chan core.NewTxsEvent, txChanSize), chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), @@ -327,6 +329,13 @@ func (w *worker) setExtra(extra []byte) { w.extra = extra } +// setGasTip sets the minimum miner tip needed to include a non-local transaction. +func (w *worker) setGasTip(tip *big.Int) { + w.mu.Lock() + defer w.mu.Unlock() + w.tip = tip +} + // setRecommitInterval updates the interval for miner sealing work recommitting. func (w *worker) setRecommitInterval(interval time.Duration) { select { @@ -554,7 +563,7 @@ func (w *worker) mainLoop() { } txset := newTransactionsByPriceAndNonce(w.current.signer, txs, w.current.header.BaseFee) tcount := w.current.tcount - w.commitTransactions(w.current, txset, nil) + w.commitTransactions(w.current, txset, nil, new(big.Int)) // Only update the snapshot if any new transactions were added // to the pending block @@ -792,7 +801,7 @@ func (w *worker) applyTransaction(env *environment, tx *types.Transaction) (*typ return receipt, err } -func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { +func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAndNonce, interrupt *atomic.Int32, minTip *big.Int) error { gasLimit := env.header.GasLimit if env.gasPool == nil { env.gasPool = new(core.GasPool).AddGas(gasLimit) @@ -812,7 +821,7 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn break } // Retrieve the next transaction and abort if all done. - ltx := txs.Peek() + ltx, tip := txs.Peek() if ltx == nil { break } @@ -827,6 +836,11 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn txs.Pop() continue } + // If we don't receive enough tip for the next transaction, skip the account + if tip.Cmp(minTip) < 0 { + log.Trace("Not enough tip for transaction", "hash", ltx.Hash, "tip", tip, "needed", minTip) + break // If the next-best is too low, surely no better will be available + } // Transaction seems to fit, pull it up from the pool tx := ltx.Resolve() if tx == nil { @@ -888,7 +902,7 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn // generateParams wraps various of settings for generating sealing task. type generateParams struct { - timestamp uint64 // The timstamp for sealing task + timestamp uint64 // The timestamp for sealing task forceTime bool // Flag whether the given timestamp is immutable or not parentHash common.Hash // Parent block hash, empty means the latest chain head coinbase common.Address // The fee recipient address for including transaction @@ -997,15 +1011,19 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err } // Fill the block with all available pending transactions. + w.mu.RLock() + tip := w.tip + w.mu.RUnlock() + if len(localTxs) > 0 { txs := newTransactionsByPriceAndNonce(env.signer, localTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt); err != nil { + if err := w.commitTransactions(env, txs, interrupt, new(big.Int)); err != nil { return err } } if len(remoteTxs) > 0 { txs := newTransactionsByPriceAndNonce(env.signer, remoteTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt); err != nil { + if err := w.commitTransactions(env, txs, interrupt, tip); err != nil { return err } } diff --git a/miner/worker_test.go b/miner/worker_test.go index 59fbbbcdca..675b8d55b9 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) const ( @@ -228,7 +229,7 @@ func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consens taskCh := make(chan struct{}, 2) checkEqual := func(t *testing.T, task *task) { // The work should contain 1 tx - receiptLen, balance := 1, big.NewInt(1000) + receiptLen, balance := 1, uint256.NewInt(1000) if len(task.receipts) != receiptLen { t.Fatalf("receipt number mismatch: have %d, want %d", len(task.receipts), receiptLen) } diff --git a/node/defaults.go b/node/defaults.go index 42d9d4cde0..307d9e186a 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -41,6 +41,7 @@ const ( // needs of all CLs. engineAPIBatchItemLimit = 2000 engineAPIBatchResponseSizeLimit = 250 * 1000 * 1000 + engineAPIBodyLimit = 128 * 1024 * 1024 ) var ( diff --git a/node/node.go b/node/node.go index 41c9971fe8..dfa83d58c7 100644 --- a/node/node.go +++ b/node/node.go @@ -453,14 +453,16 @@ func (n *Node) startRPC() error { jwtSecret: secret, batchItemLimit: engineAPIBatchItemLimit, batchResponseSizeLimit: engineAPIBatchResponseSizeLimit, + httpBodyLimit: engineAPIBodyLimit, } - if err := server.enableRPC(allAPIs, httpConfig{ + err := server.enableRPC(allAPIs, httpConfig{ CorsAllowedOrigins: DefaultAuthCors, Vhosts: n.config.AuthVirtualHosts, Modules: DefaultAuthModules, prefix: DefaultAuthPrefix, rpcEndpointConfig: sharedConfig, - }); err != nil { + }) + if err != nil { return err } servers = append(servers, server) diff --git a/node/rpcstack.go b/node/rpcstack.go index b33c238051..d80d5271a7 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -56,6 +56,7 @@ type rpcEndpointConfig struct { jwtSecret []byte // optional JWT secret batchItemLimit int batchResponseSizeLimit int + httpBodyLimit int } type rpcHandler struct { @@ -304,6 +305,9 @@ func (h *httpServer) enableRPC(apis []rpc.API, config httpConfig) error { // Create RPC server and handler. srv := rpc.NewServer() srv.SetBatchLimits(config.batchItemLimit, config.batchResponseSizeLimit) + if config.httpBodyLimit > 0 { + srv.SetHTTPBodyLimit(config.httpBodyLimit) + } if err := RegisterApis(apis, config.Modules, srv); err != nil { return err } @@ -336,6 +340,9 @@ func (h *httpServer) enableWS(apis []rpc.API, config wsConfig) error { // Create RPC server and handler. srv := rpc.NewServer() srv.SetBatchLimits(config.batchItemLimit, config.batchResponseSizeLimit) + if config.httpBodyLimit > 0 { + srv.SetHTTPBodyLimit(config.httpBodyLimit) + } if err := RegisterApis(apis, config.Modules, srv); err != nil { return err } diff --git a/p2p/discover/metrics.go b/p2p/discover/metrics.go index da8e9cb817..56aae24285 100644 --- a/p2p/discover/metrics.go +++ b/p2p/discover/metrics.go @@ -58,7 +58,7 @@ func newMeteredConn(conn UDPConn) UDPConn { return &meteredUdpConn{UDPConn: conn} } -// Read delegates a network read to the underlying connection, bumping the udp ingress traffic meter along the way. +// ReadFromUDP delegates a network read to the underlying connection, bumping the udp ingress traffic meter along the way. func (c *meteredUdpConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { n, addr, err = c.UDPConn.ReadFromUDP(b) ingressTrafficMeter.Mark(int64(n)) diff --git a/p2p/server_nat.go b/p2p/server_nat.go index 354597cc7a..299d275490 100644 --- a/p2p/server_nat.go +++ b/p2p/server_nat.go @@ -127,7 +127,7 @@ func (srv *Server) portMappingLoop() { } else if !ip.Equal(lastExtIP) { log.Debug("External IP changed", "ip", extip, "interface", srv.NAT) } else { - return + continue } // Here, we either failed to get the external IP, or it has changed. lastExtIP = ip diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index c52917fd0a..349e496b2f 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -172,7 +172,7 @@ type SimNode struct { registerOnce sync.Once } -// Close closes the underlaying node.Node to release +// Close closes the underlying node.Node to release // acquired resources. func (sn *SimNode) Close() error { return sn.node.Close() diff --git a/params/config.go b/params/config.go index 9b4c1338e4..2c80f4f6b0 100644 --- a/params/config.go +++ b/params/config.go @@ -21,6 +21,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/params/forks" ) // Genesis hashes to enforce below configs on. @@ -57,6 +58,7 @@ var ( TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 TerminalTotalDifficultyPassed: true, ShanghaiTime: newUint64(1681338455), + CancunTime: newUint64(1710338135), Ethash: new(EthashConfig), } // HoleskyChainConfig contains the chain parameters to run a node on the Holesky test network. @@ -641,7 +643,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { lastFork.name, cur.name, cur.block) } else { return fmt.Errorf("unsupported fork ordering: %v not enabled, but %v enabled at timestamp %v", - lastFork.name, cur.name, cur.timestamp) + lastFork.name, cur.name, *cur.timestamp) } // Fork (whether defined by block or timestamp) must follow the fork definition sequence @@ -651,7 +653,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { lastFork.name, lastFork.block, cur.name, cur.block) } else if lastFork.timestamp != nil && *lastFork.timestamp > *cur.timestamp { return fmt.Errorf("unsupported fork ordering: %v enabled at timestamp %v, but %v enabled at timestamp %v", - lastFork.name, lastFork.timestamp, cur.name, cur.timestamp) + lastFork.name, *lastFork.timestamp, cur.name, *cur.timestamp) } // Timestamp based forks can follow block based ones, but not the other way around @@ -750,6 +752,23 @@ func (c *ChainConfig) ElasticityMultiplier() uint64 { return DefaultElasticityMultiplier } +// LatestFork returns the latest time-based fork that would be active for the given time. +func (c *ChainConfig) LatestFork(time uint64) forks.Fork { + // Assume last non-time-based fork has passed. + london := c.LondonBlock + + switch { + case c.IsPrague(london, time): + return forks.Prague + case c.IsCancun(london, time): + return forks.Cancun + case c.IsShanghai(london, time): + return forks.Shanghai + default: + return forks.Paris + } +} + // isForkBlockIncompatible returns true if a fork scheduled at block s1 cannot be // rescheduled to block s2 because head is already past the fork. func isForkBlockIncompatible(s1, s2, head *big.Int) bool { diff --git a/params/forks/forks.go b/params/forks/forks.go new file mode 100644 index 0000000000..4f50ff5aed --- /dev/null +++ b/params/forks/forks.go @@ -0,0 +1,42 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package forks + +// Fork is a numerical identifier of specific network upgrades (forks). +type Fork int + +const ( + Frontier = iota + FrontierThawing + Homestead + DAO + TangerineWhistle + SpuriousDragon + Byzantium + Constantinople + Petersburg + Istanbul + MuirGlacier + Berlin + London + ArrowGlacier + GrayGlacier + Paris + Shanghai + Cancun + Prague +) diff --git a/params/version.go b/params/version.go index ba8a0f50d5..7284c07524 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 13 // Minor version component of the current release - VersionPatch = 11 // Patch version component of the current release + VersionPatch = 13 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/rpc/http.go b/rpc/http.go index 741fa1c0eb..dd376b1ecd 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -33,8 +33,8 @@ import ( ) const ( - maxRequestContentLength = 1024 * 1024 * 5 - contentType = "application/json" + defaultBodyLimit = 5 * 1024 * 1024 + contentType = "application/json" ) // https://www.jsonrpc.org/historical/json-rpc-over-http.html#id13 @@ -253,8 +253,8 @@ type httpServerConn struct { r *http.Request } -func newHTTPServerConn(r *http.Request, w http.ResponseWriter) ServerCodec { - body := io.LimitReader(r.Body, maxRequestContentLength) +func (s *Server) newHTTPServerConn(r *http.Request, w http.ResponseWriter) ServerCodec { + body := io.LimitReader(r.Body, int64(s.httpBodyLimit)) conn := &httpServerConn{Reader: body, Writer: w, r: r} encoder := func(v any, isErrorResponse bool) error { @@ -312,7 +312,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) return } - if code, err := validateRequest(r); err != nil { + if code, err := s.validateRequest(r); err != nil { http.Error(w, err.Error(), code) return } @@ -330,19 +330,19 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // until EOF, writes the response to w, and orders the server to process a // single request. w.Header().Set("content-type", contentType) - codec := newHTTPServerConn(r, w) + codec := s.newHTTPServerConn(r, w) defer codec.close() s.serveSingleRequest(ctx, codec) } // validateRequest returns a non-zero response code and error message if the // request is invalid. -func validateRequest(r *http.Request) (int, error) { +func (s *Server) validateRequest(r *http.Request) (int, error) { if r.Method == http.MethodPut || r.Method == http.MethodDelete { return http.StatusMethodNotAllowed, errors.New("method not allowed") } - if r.ContentLength > maxRequestContentLength { - err := fmt.Errorf("content length too large (%d>%d)", r.ContentLength, maxRequestContentLength) + if r.ContentLength > int64(s.httpBodyLimit) { + err := fmt.Errorf("content length too large (%d>%d)", r.ContentLength, s.httpBodyLimit) return http.StatusRequestEntityTooLarge, err } // Allow OPTIONS (regardless of content-type) diff --git a/rpc/http_test.go b/rpc/http_test.go index 584842a9aa..ad86ca15ae 100644 --- a/rpc/http_test.go +++ b/rpc/http_test.go @@ -40,11 +40,13 @@ func confirmStatusCode(t *testing.T, got, want int) { func confirmRequestValidationCode(t *testing.T, method, contentType, body string, expectedStatusCode int) { t.Helper() + + s := NewServer() request := httptest.NewRequest(method, "http://url.com", strings.NewReader(body)) if len(contentType) > 0 { request.Header.Set("Content-Type", contentType) } - code, err := validateRequest(request) + code, err := s.validateRequest(request) if code == 0 { if err != nil { t.Errorf("validation: got error %v, expected nil", err) @@ -64,7 +66,7 @@ func TestHTTPErrorResponseWithPut(t *testing.T) { } func TestHTTPErrorResponseWithMaxContentLength(t *testing.T) { - body := make([]rune, maxRequestContentLength+1) + body := make([]rune, defaultBodyLimit+1) confirmRequestValidationCode(t, http.MethodPost, contentType, string(body), http.StatusRequestEntityTooLarge) } @@ -104,7 +106,7 @@ func TestHTTPResponseWithEmptyGet(t *testing.T) { // This checks that maxRequestContentLength is not applied to the response of a request. func TestHTTPRespBodyUnlimited(t *testing.T) { - const respLength = maxRequestContentLength * 3 + const respLength = defaultBodyLimit * 3 s := NewServer() defer s.Stop() diff --git a/rpc/server.go b/rpc/server.go index 2742adf07b..e2f9120aa2 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -51,13 +51,15 @@ type Server struct { run atomic.Bool batchItemLimit int batchResponseLimit int + httpBodyLimit int } // NewServer creates a new server instance with no registered handlers. func NewServer() *Server { server := &Server{ - idgen: randomIDGenerator(), - codecs: make(map[ServerCodec]struct{}), + idgen: randomIDGenerator(), + codecs: make(map[ServerCodec]struct{}), + httpBodyLimit: defaultBodyLimit, } server.run.Store(true) // Register the default service providing meta information about the RPC service such @@ -78,6 +80,13 @@ func (s *Server) SetBatchLimits(itemLimit, maxResponseSize int) { s.batchResponseLimit = maxResponseSize } +// SetHTTPBodyLimit sets the size limit for HTTP requests. +// +// This method should be called before processing any requests via ServeHTTP. +func (s *Server) SetHTTPBodyLimit(limit int) { + s.httpBodyLimit = limit +} + // RegisterName creates a service for the given receiver type under the given name. When no // methods on the given receiver match the criteria to be either a RPC method or a // subscription an error is returned. Otherwise a new service is created and added to the diff --git a/rpc/websocket_test.go b/rpc/websocket_test.go index d3e15d94c9..8d2bd9d802 100644 --- a/rpc/websocket_test.go +++ b/rpc/websocket_test.go @@ -97,7 +97,7 @@ func TestWebsocketLargeCall(t *testing.T) { // This call sends slightly less than the limit and should work. var result echoResult - arg := strings.Repeat("x", maxRequestContentLength-200) + arg := strings.Repeat("x", defaultBodyLimit-200) if err := client.Call(&result, "test_echo", arg, 1); err != nil { t.Fatalf("valid call didn't work: %v", err) } @@ -106,7 +106,7 @@ func TestWebsocketLargeCall(t *testing.T) { } // This call sends twice the allowed size and shouldn't work. - arg = strings.Repeat("x", maxRequestContentLength*2) + arg = strings.Repeat("x", defaultBodyLimit*2) err = client.Call(&result, "test_echo", arg) if err == nil { t.Fatal("no error for too large call") diff --git a/signer/core/api.go b/signer/core/api.go index ef8c136625..a32f24cb18 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -631,7 +631,7 @@ func (api *SignerAPI) SignGnosisSafeTx(ctx context.Context, signerAddress common } } typedData := gnosisTx.ToTypedData() - // might aswell error early. + // might as well error early. // we are expected to sign. If our calculated hash does not match what they want, // The gnosis safetx input contains a 'safeTxHash' which is the expected safeTxHash that sighash, _, err := apitypes.TypedDataAndHash(typedData) diff --git a/tests/block_test_util.go b/tests/block_test_util.go index ff487255f4..2b6ba6db03 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -328,7 +328,7 @@ func (t *BlockTest) validatePostState(statedb *state.StateDB) error { for addr, acct := range t.json.Post { // address is indirectly verified by the other fields, as it's the db key code2 := statedb.GetCode(addr) - balance2 := statedb.GetBalance(addr) + balance2 := statedb.GetBalance(addr).ToBig() nonce2 := statedb.GetNonce(addr) if !bytes.Equal(code2, acct.Code) { return fmt.Errorf("account code mismatch for addr: %s want: %v have: %s", addr, acct.Code, hex.EncodeToString(code2)) diff --git a/tests/state_test.go b/tests/state_test.go index cc228ea3c6..3a7e83ae3d 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/logger" + "github.com/holiman/uint256" ) func TestState(t *testing.T) { @@ -279,7 +280,7 @@ func runBenchmark(b *testing.B, t *StateTest) { start := time.Now() // Execute the message. - _, leftOverGas, err := evm.Call(sender, *msg.To, msg.Data, msg.GasLimit, msg.Value) + _, leftOverGas, err := evm.Call(sender, *msg.To, msg.Data, msg.GasLimit, uint256.MustFromBig(msg.Value)) if err != nil { b.Error(err) return diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 919730089a..eb5738242e 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -42,6 +42,7 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) @@ -315,7 +316,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh // - the coinbase self-destructed, or // - there are only 'bad' transactions, which aren't executed. In those cases, // the coinbase gets no txfee, so isn't created, and thus needs to be touched - statedb.AddBalance(block.Coinbase(), new(big.Int)) + statedb.AddBalance(block.Coinbase(), new(uint256.Int)) // Commit state mutations into database. root, _ := statedb.Commit(block.NumberU64(), config.IsEIP158(block.Number())) @@ -339,7 +340,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo for addr, a := range accounts { statedb.SetCode(addr, a.Code) statedb.SetNonce(addr, a.Nonce) - statedb.SetBalance(addr, a.Balance) + statedb.SetBalance(addr, uint256.MustFromBig(a.Balance)) for k, v := range a.Storage { statedb.SetState(addr, k, v) } diff --git a/trie/proof.go b/trie/proof.go index a526a53402..fd892fb4be 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -389,7 +389,7 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error } else { if bytes.Compare(cld.Key, key[pos:]) > 0 { // The key of fork shortnode is greater than the - // path(it belongs to the range), unset the entrie + // path(it belongs to the range), unset the entries // branch. The parent must be a fullnode. fn := parent.(*fullNode) fn.Children[key[pos-1]] = nil diff --git a/trie/trie_test.go b/trie/trie_test.go index c5bd3faf53..b799a0c3ed 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -23,7 +23,6 @@ import ( "fmt" "hash" "io" - "math/big" "math/rand" "reflect" "testing" @@ -37,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) @@ -333,7 +333,7 @@ func TestLargeValue(t *testing.T) { trie.Hash() } -// TestRandomCases tests som cases that were found via random fuzzing +// TestRandomCases tests some cases that were found via random fuzzing func TestRandomCases(t *testing.T) { var rt = []randTestStep{ {op: 6, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 0 @@ -796,7 +796,7 @@ func makeAccounts(size int) (addresses [][20]byte, accounts [][]byte) { numBytes := random.Uint32() % 33 // [0, 32] bytes balanceBytes := make([]byte, numBytes) random.Read(balanceBytes) - balance := new(big.Int).SetBytes(balanceBytes) + balance := new(uint256.Int).SetBytes(balanceBytes) data, _ := rlp.EncodeToBytes(&types.StateAccount{Nonce: nonce, Balance: balance, Root: root, CodeHash: code}) accounts[i] = data } diff --git a/trie/triedb/pathdb/database_test.go b/trie/triedb/pathdb/database_test.go index 5509682c39..e7bd469993 100644 --- a/trie/triedb/pathdb/database_test.go +++ b/trie/triedb/pathdb/database_test.go @@ -20,7 +20,6 @@ import ( "bytes" "errors" "fmt" - "math/big" "math/rand" "testing" @@ -32,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/trie/testutil" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/holiman/uint256" ) func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[common.Hash][]byte) (common.Hash, *trienode.NodeSet) { @@ -53,7 +53,7 @@ func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[comm func generateAccount(storageRoot common.Hash) types.StateAccount { return types.StateAccount{ Nonce: uint64(rand.Intn(100)), - Balance: big.NewInt(rand.Int63()), + Balance: uint256.NewInt(rand.Uint64()), CodeHash: testutil.RandBytes(32), Root: storageRoot, } diff --git a/trie/verkle.go b/trie/verkle.go index 89e2e53408..c21a796a0f 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -20,7 +20,6 @@ import ( "encoding/binary" "errors" "fmt" - "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -108,7 +107,7 @@ func (t *VerkleTrie) GetAccount(addr common.Address) (*types.StateAccount, error for i := 0; i < len(balance)/2; i++ { balance[len(balance)-i-1], balance[i] = balance[i], balance[len(balance)-i-1] } - acc.Balance = new(big.Int).SetBytes(balance[:]) + acc.Balance = new(uint256.Int).SetBytes32(balance[:]) // Decode codehash acc.CodeHash = values[utils.CodeKeccakLeafKey] diff --git a/trie/verkle_test.go b/trie/verkle_test.go index bd31ea3879..1c65b673aa 100644 --- a/trie/verkle_test.go +++ b/trie/verkle_test.go @@ -18,7 +18,6 @@ package trie import ( "bytes" - "math/big" "reflect" "testing" @@ -27,18 +26,19 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/utils" + "github.com/holiman/uint256" ) var ( accounts = map[common.Address]*types.StateAccount{ {1}: { Nonce: 100, - Balance: big.NewInt(100), + Balance: uint256.NewInt(100), CodeHash: common.Hash{0x1}.Bytes(), }, {2}: { Nonce: 200, - Balance: big.NewInt(200), + Balance: uint256.NewInt(200), CodeHash: common.Hash{0x2}.Bytes(), }, }