diff --git a/accounts/abi/abigen/bind_test.go b/accounts/abi/abigen/bind_test.go index b3c52e81e5..1651e637c8 100644 --- a/accounts/abi/abigen/bind_test.go +++ b/accounts/abi/abigen/bind_test.go @@ -485,13 +485,13 @@ var bindTests = []struct { contract Defaulter { address public caller; - function() { + fallback() external payable { caller = msg.sender; } } `, - []string{`6060604052606a8060106000396000f360606040523615601d5760e060020a6000350463fc9c8d3981146040575b605e6000805473ffffffffffffffffffffffffffffffffffffffff191633179055565b606060005473ffffffffffffffffffffffffffffffffffffffff1681565b005b6060908152602090f3`}, - []string{`[{"constant":true,"inputs":[],"name":"caller","outputs":[{"name":"","type":"address"}],"type":"function"}]`}, + []string{`608060405234801561000f575f80fd5b5061013d8061001d5f395ff3fe608060405260043610610021575f3560e01c8063fc9c8d391461006257610022565b5b335f806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055005b34801561006d575f80fd5b5061007661008c565b60405161008391906100ee565b60405180910390f35b5f8054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b5f73ffffffffffffffffffffffffffffffffffffffff82169050919050565b5f6100d8826100af565b9050919050565b6100e8816100ce565b82525050565b5f6020820190506101015f8301846100df565b9291505056fea26469706673582212201e9273ecfb1f534644c77f09a25c21baaba81cf1c444ebc071e12a225a23c72964736f6c63430008140033`}, + []string{`[{"stateMutability":"payable","type":"fallback"},{"inputs":[],"name":"caller","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"}]`}, ` "math/big" diff --git a/accounts/abi/bind/v2/base.go b/accounts/abi/bind/v2/base.go index f714848efb..4f2013b4a3 100644 --- a/accounts/abi/bind/v2/base.go +++ b/accounts/abi/bind/v2/base.go @@ -277,8 +277,10 @@ func (c *BoundContract) RawCreationTransact(opts *TransactOpts, calldata []byte) // Transfer initiates a plain transaction to move funds to the contract, calling // its default method if one is available. func (c *BoundContract) Transfer(opts *TransactOpts) (*types.Transaction, error) { - // todo(rjl493456442) check the payable fallback or receive is defined - // or not, reject invalid transaction at the first place + // Check if payable fallback or receive is defined + if !c.abi.HasReceive() && !(c.abi.HasFallback() && c.abi.Fallback.IsPayable()) { + return nil, fmt.Errorf("contract does not have a payable fallback or receive function") + } return c.transact(opts, &c.address, nil) } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 2465b52ad1..cc294b2f30 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -188,6 +188,8 @@ var ( utils.AllowUnprotectedTxs, utils.BatchRequestLimit, utils.BatchResponseMaxSize, + utils.RPCTxSyncDefaultTimeoutFlag, + utils.RPCTxSyncMaxTimeoutFlag, } metricsFlags = []cli.Flag{ diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index 6490f832af..67dc7257c0 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -201,7 +201,7 @@ func expandVerkle(ctx *cli.Context) error { } for i, key := range keylist { - log.Info("Reading key", "index", i, "key", keylist[0]) + log.Info("Reading key", "index", i, "key", key) root.Get(key, chaindb.Get) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c9da08578c..0c5db9e6d8 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -615,6 +615,18 @@ var ( Value: ethconfig.Defaults.LogQueryLimit, Category: flags.APICategory, } + RPCTxSyncDefaultTimeoutFlag = &cli.DurationFlag{ + Name: "rpc.txsync.defaulttimeout", + Usage: "Default timeout for eth_sendRawTransactionSync (e.g. 2s, 500ms)", + Value: ethconfig.Defaults.TxSyncDefaultTimeout, + Category: flags.APICategory, + } + RPCTxSyncMaxTimeoutFlag = &cli.DurationFlag{ + Name: "rpc.txsync.maxtimeout", + Usage: "Maximum allowed timeout for eth_sendRawTransactionSync (e.g. 5m)", + Value: ethconfig.Defaults.TxSyncMaxTimeout, + Category: flags.APICategory, + } // Authenticated RPC HTTP settings AuthListenFlag = &cli.StringFlag{ Name: "authrpc.addr", @@ -1717,6 +1729,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(RPCGlobalLogQueryLimit.Name) { cfg.LogQueryLimit = ctx.Int(RPCGlobalLogQueryLimit.Name) } + if ctx.IsSet(RPCTxSyncDefaultTimeoutFlag.Name) { + cfg.TxSyncDefaultTimeout = ctx.Duration(RPCTxSyncDefaultTimeoutFlag.Name) + } + if ctx.IsSet(RPCTxSyncMaxTimeoutFlag.Name) { + cfg.TxSyncMaxTimeout = ctx.Duration(RPCTxSyncMaxTimeoutFlag.Name) + } if !ctx.Bool(SnapshotFlag.Name) || cfg.SnapshotCache == 0 { // If snap-sync is requested, this flag is also required if cfg.SyncMode == ethconfig.SnapSync { diff --git a/cmd/workload/filtertest.go b/cmd/workload/filtertest.go index 9f0b6cab44..d77cbc5768 100644 --- a/cmd/workload/filtertest.go +++ b/cmd/workload/filtertest.go @@ -182,13 +182,14 @@ func (s *filterTestSuite) loadQueries() error { // filterQuery is a single query for testing. type filterQuery struct { - FromBlock int64 `json:"fromBlock"` - ToBlock int64 `json:"toBlock"` - Address []common.Address `json:"address"` - Topics [][]common.Hash `json:"topics"` - ResultHash *common.Hash `json:"resultHash,omitempty"` - results []types.Log - Err error `json:"error,omitempty"` + FromBlock int64 `json:"fromBlock"` + ToBlock int64 `json:"toBlock"` + lastBlockHash common.Hash + Address []common.Address `json:"address"` + Topics [][]common.Hash `json:"topics"` + ResultHash *common.Hash `json:"resultHash,omitempty"` + results []types.Log + Err error `json:"error,omitempty"` } func (fq *filterQuery) isWildcard() bool { diff --git a/cmd/workload/filtertestfuzz.go b/cmd/workload/filtertestfuzz.go new file mode 100644 index 0000000000..3549f4db56 --- /dev/null +++ b/cmd/workload/filtertestfuzz.go @@ -0,0 +1,337 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "context" + "fmt" + "math/big" + "reflect" + "slices" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" + "github.com/urfave/cli/v2" +) + +const maxFilterRangeForTestFuzz = 300 + +var ( + filterFuzzCommand = &cli.Command{ + Name: "filterfuzz", + Usage: "Generates queries and compares results against matches derived from receipts", + ArgsUsage: "", + Action: filterFuzzCmd, + Flags: []cli.Flag{}, + } +) + +// filterFuzzCmd is the main function of the filter fuzzer. +func filterFuzzCmd(ctx *cli.Context) error { + f := newFilterTestGen(ctx, maxFilterRangeForTestFuzz) + var lastHead *types.Header + headerCache := lru.NewCache[common.Hash, *types.Header](200) + + commonAncestor := func(oldPtr, newPtr *types.Header) *types.Header { + if oldPtr == nil || newPtr == nil { + return nil + } + if newPtr.Number.Uint64() > oldPtr.Number.Uint64()+100 || oldPtr.Number.Uint64() > newPtr.Number.Uint64()+100 { + return nil + } + for oldPtr.Hash() != newPtr.Hash() { + if newPtr.Number.Uint64() >= oldPtr.Number.Uint64() { + if parent, _ := headerCache.Get(newPtr.ParentHash); parent != nil { + newPtr = parent + } else { + newPtr, _ = getHeaderByHash(f.client, newPtr.ParentHash) + if newPtr == nil { + return nil + } + headerCache.Add(newPtr.Hash(), newPtr) + } + } + if oldPtr.Number.Uint64() > newPtr.Number.Uint64() { + oldPtr, _ = headerCache.Get(oldPtr.ParentHash) + if oldPtr == nil { + return nil + } + } + } + return newPtr + } + + fetchHead := func() (*types.Header, bool) { + currentHead, err := getLatestHeader(f.client) + if err != nil { + fmt.Println("Could not fetch head block", err) + return nil, false + } + headerCache.Add(currentHead.Hash(), currentHead) + if lastHead != nil && currentHead.Hash() == lastHead.Hash() { + return currentHead, false + } + f.blockLimit = currentHead.Number.Int64() + ca := commonAncestor(lastHead, currentHead) + fmt.Print("*** New head ", f.blockLimit) + if ca == nil { + fmt.Println(" ") + } else { + if reorged := lastHead.Number.Uint64() - ca.Number.Uint64(); reorged > 0 { + fmt.Print(" reorged ", reorged) + } + if missed := currentHead.Number.Uint64() - ca.Number.Uint64() - 1; missed > 0 { + fmt.Print(" missed ", missed) + } + fmt.Println() + } + lastHead = currentHead + return currentHead, true + } + + tryExtendQuery := func(query *filterQuery) *filterQuery { + for { + extQuery := f.extendRange(query) + if extQuery == nil { + return query + } + extQuery.checkLastBlockHash(f.client) + extQuery.run(f.client, nil) + if extQuery.Err == nil && len(extQuery.results) == 0 { + // query is useless now due to major reorg; abandon and continue + fmt.Println("Zero length results") + return nil + } + if extQuery.Err != nil { + extQuery.printError() + return nil + } + if len(extQuery.results) > maxFilterResultSize { + return query + } + query = extQuery + } + } + + var ( + mmQuery *filterQuery + mmRetry, mmNextRetry int + ) + +mainLoop: + for { + select { + case <-ctx.Done(): + return nil + default: + } + var query *filterQuery + if mmQuery != nil { + if mmRetry == 0 { + query = mmQuery + mmRetry = mmNextRetry + mmNextRetry *= 2 + query.checkLastBlockHash(f.client) + query.run(f.client, nil) + if query.Err != nil { + query.printError() + continue + } + fmt.Println("Retrying query from:", query.FromBlock, "to:", query.ToBlock, "results:", len(query.results)) + } else { + mmRetry-- + } + } + if query == nil { + currentHead, isNewHead := fetchHead() + if currentHead == nil { + select { + case <-ctx.Done(): + return nil + case <-time.After(time.Second): + } + continue mainLoop + } + if isNewHead { + query = f.newHeadSeedQuery(currentHead.Number.Int64()) + } else { + query = f.newQuery() + } + query.checkLastBlockHash(f.client) + query.run(f.client, nil) + if query.Err != nil { + query.printError() + continue + } + fmt.Println("New query from:", query.FromBlock, "to:", query.ToBlock, "results:", len(query.results)) + if len(query.results) == 0 || len(query.results) > maxFilterResultSize { + continue mainLoop + } + if query = tryExtendQuery(query); query == nil { + continue mainLoop + } + } + if !query.checkLastBlockHash(f.client) { + fmt.Println("Reorg during search") + continue mainLoop + } + // now we have a new query; check results + results, err := query.getResultsFromReceipts(f.client) + if err != nil { + fmt.Println("Could not fetch results from receipts", err) + continue mainLoop + } + if !query.checkLastBlockHash(f.client) { + fmt.Println("Reorg during search") + continue mainLoop + } + if !reflect.DeepEqual(query.results, results) { + fmt.Println("Results mismatch from:", query.FromBlock, "to:", query.ToBlock, "addresses:", query.Address, "topics:", query.Topics) + resShared, resGetLogs, resReceipts := compareResults(query.results, results) + fmt.Println(" shared:", len(resShared)) + fmt.Println(" only from getLogs:", len(resGetLogs), resGetLogs) + fmt.Println(" only from receipts:", len(resReceipts), resReceipts) + if mmQuery != query { + mmQuery = query + mmRetry = 0 + mmNextRetry = 1 + } + continue mainLoop + } + fmt.Println("Successful query from:", query.FromBlock, "to:", query.ToBlock, "results:", len(query.results)) + f.storeQuery(query) + } +} + +func compareResults(a, b []types.Log) (shared, onlya, onlyb []types.Log) { + for len(a) > 0 && len(b) > 0 { + if reflect.DeepEqual(a[0], b[0]) { + shared = append(shared, a[0]) + a = a[1:] + b = b[1:] + } else { + for i := 1; ; i++ { + if i >= len(a) { // b[0] not found in a + onlyb = append(onlyb, b[0]) + b = b[1:] + break + } + if i >= len(b) { // a[0] not found in b + onlya = append(onlya, a[0]) + a = a[1:] + break + } + if reflect.DeepEqual(b[0], a[i]) { // a[:i] not found in b + onlya = append(onlya, a[:i]...) + a = a[i:] + break + } + if reflect.DeepEqual(a[0], b[i]) { // b[:i] not found in a + onlyb = append(onlyb, b[:i]...) + b = b[i:] + break + } + } + } + } + onlya = append(onlya, a...) + onlyb = append(onlyb, b...) + return +} + +func getLatestHeader(client *client) (*types.Header, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + return client.Eth.HeaderByNumber(ctx, big.NewInt(int64(rpc.LatestBlockNumber))) +} + +func getHeaderByHash(client *client, hash common.Hash) (*types.Header, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + return client.Eth.HeaderByHash(ctx, hash) +} + +// newHeadSeedQuery creates a query that gets all logs from the latest head. +func (s *filterTestGen) newHeadSeedQuery(head int64) *filterQuery { + return &filterQuery{ + FromBlock: head, + ToBlock: head, + } +} + +func (fq *filterQuery) checkLastBlockHash(client *client) bool { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + header, err := client.Eth.HeaderByNumber(ctx, big.NewInt(fq.ToBlock)) + if err != nil { + fmt.Println("Cound not fetch last block hash of query number:", fq.ToBlock, "error:", err) + fq.lastBlockHash = common.Hash{} + return false + } + hash := header.Hash() + if fq.lastBlockHash == hash { + return true + } + fq.lastBlockHash = hash + return false +} + +func (fq *filterQuery) filterLog(log *types.Log) bool { + if len(fq.Address) > 0 && !slices.Contains(fq.Address, log.Address) { + return false + } + // If the to filtered topics is greater than the amount of topics in logs, skip. + if len(fq.Topics) > len(log.Topics) { + return false + } + for i, sub := range fq.Topics { + if len(sub) == 0 { + continue // empty rule set == wildcard + } + if !slices.Contains(sub, log.Topics[i]) { + return false + } + } + return true +} + +func (fq *filterQuery) getResultsFromReceipts(client *client) ([]types.Log, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + var results []types.Log + for blockNumber := fq.FromBlock; blockNumber <= fq.ToBlock; blockNumber++ { + receipts, err := client.Eth.BlockReceipts(ctx, rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(blockNumber))) + if err != nil { + return nil, err + } + for _, receipt := range receipts { + for _, log := range receipt.Logs { + if fq.filterLog(log) { + results = append(results, *log) + } + } + } + } + return results, nil +} diff --git a/cmd/workload/filtertestgen.go b/cmd/workload/filtertestgen.go index 6d1f639819..603e3dea67 100644 --- a/cmd/workload/filtertestgen.go +++ b/cmd/workload/filtertestgen.go @@ -32,6 +32,17 @@ import ( "github.com/urfave/cli/v2" ) +const ( + // Parameter of the random filter query generator. + maxFilterRangeForTestGen = 100000000000 + maxFilterResultSize = 1000 + filterBuckets = 10 + maxFilterBucketSize = 100 + filterSeedChance = 10 + filterMergeChance = 45 + filterExtendChance = 50 +) + var ( filterGenerateCommand = &cli.Command{ Name: "filtergen", @@ -58,7 +69,7 @@ var ( // filterGenCmd is the main function of the filter tests generator. func filterGenCmd(ctx *cli.Context) error { - f := newFilterTestGen(ctx) + f := newFilterTestGen(ctx, maxFilterRangeForTestGen) lastWrite := time.Now() for { select { @@ -67,7 +78,7 @@ func filterGenCmd(ctx *cli.Context) error { default: } - f.updateFinalizedBlock() + f.setLimitToFinalizedBlock() query := f.newQuery() query.run(f.client, nil) if query.Err != nil { @@ -75,7 +86,7 @@ func filterGenCmd(ctx *cli.Context) error { exit("filter query failed") } if len(query.results) > 0 && len(query.results) <= maxFilterResultSize { - for { + for rand.Intn(100) < filterExtendChance { extQuery := f.extendRange(query) if extQuery == nil { break @@ -108,39 +119,32 @@ func filterGenCmd(ctx *cli.Context) error { // filterTestGen is the filter query test generator. type filterTestGen struct { - client *client - queryFile string + client *client + queryFile string + maxFilterRange int64 - finalizedBlock int64 - queries [filterBuckets][]*filterQuery + blockLimit int64 + queries [filterBuckets][]*filterQuery } -func newFilterTestGen(ctx *cli.Context) *filterTestGen { +func newFilterTestGen(ctx *cli.Context, maxFilterRange int64) *filterTestGen { return &filterTestGen{ - client: makeClient(ctx), - queryFile: ctx.String(filterQueryFileFlag.Name), + client: makeClient(ctx), + queryFile: ctx.String(filterQueryFileFlag.Name), + maxFilterRange: maxFilterRange, } } -func (s *filterTestGen) updateFinalizedBlock() { - s.finalizedBlock = mustGetFinalizedBlock(s.client) +func (s *filterTestGen) setLimitToFinalizedBlock() { + s.blockLimit = mustGetFinalizedBlock(s.client) } -const ( - // Parameter of the random filter query generator. - maxFilterRange = 10000000 - maxFilterResultSize = 300 - filterBuckets = 10 - maxFilterBucketSize = 100 - filterSeedChance = 10 - filterMergeChance = 45 -) - // storeQuery adds a filter query to the output file. func (s *filterTestGen) storeQuery(query *filterQuery) { query.ResultHash = new(common.Hash) *query.ResultHash = query.calculateHash() - logRatio := math.Log(float64(len(query.results))*float64(s.finalizedBlock)/float64(query.ToBlock+1-query.FromBlock)) / math.Log(float64(s.finalizedBlock)*maxFilterResultSize) + maxFilterRange := min(s.maxFilterRange, s.blockLimit) + logRatio := math.Log(float64(len(query.results))*float64(maxFilterRange)/float64(query.ToBlock+1-query.FromBlock)) / math.Log(float64(maxFilterRange)*maxFilterResultSize) bucket := int(math.Floor(logRatio * filterBuckets)) if bucket >= filterBuckets { bucket = filterBuckets - 1 @@ -160,13 +164,13 @@ func (s *filterTestGen) storeQuery(query *filterQuery) { func (s *filterTestGen) extendRange(q *filterQuery) *filterQuery { rangeLen := q.ToBlock + 1 - q.FromBlock extLen := rand.Int63n(rangeLen) + 1 - if rangeLen+extLen > s.finalizedBlock { + if rangeLen+extLen > min(s.maxFilterRange, s.blockLimit) { return nil } extBefore := min(rand.Int63n(extLen+1), q.FromBlock) extAfter := extLen - extBefore - if q.ToBlock+extAfter > s.finalizedBlock { - d := q.ToBlock + extAfter - s.finalizedBlock + if q.ToBlock+extAfter > s.blockLimit { + d := q.ToBlock + extAfter - s.blockLimit extAfter -= d if extBefore+d <= q.FromBlock { extBefore += d @@ -203,7 +207,7 @@ func (s *filterTestGen) newQuery() *filterQuery { // newSeedQuery creates a query that gets all logs in a random non-finalized block. func (s *filterTestGen) newSeedQuery() *filterQuery { - block := rand.Int63n(s.finalizedBlock + 1) + block := rand.Int63n(s.blockLimit + 1) return &filterQuery{ FromBlock: block, ToBlock: block, @@ -358,6 +362,7 @@ func (s *filterTestGen) writeQueries() { func mustGetFinalizedBlock(client *client) int64 { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() + header, err := client.Eth.HeaderByNumber(ctx, big.NewInt(int64(rpc.FinalizedBlockNumber))) if err != nil { exit(fmt.Errorf("could not fetch finalized header (error: %v)", err)) diff --git a/cmd/workload/main.go b/cmd/workload/main.go index 32618d6a79..8ac0e5b6cb 100644 --- a/cmd/workload/main.go +++ b/cmd/workload/main.go @@ -49,6 +49,7 @@ func init() { filterGenerateCommand, traceGenerateCommand, filterPerfCommand, + filterFuzzCommand, } } diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go index 298ad04f40..714c1f77d6 100644 --- a/core/rawdb/accessors_state.go +++ b/core/rawdb/accessors_state.go @@ -188,24 +188,16 @@ func ReadStateAccountIndex(db ethdb.AncientReaderOp, id uint64) []byte { // data in the concatenated storage data table. Compute the position of state // history in freezer by minus one since the id of first state history starts // from one (zero for initial state). -func ReadStateStorageIndex(db ethdb.AncientReaderOp, id uint64) []byte { - blob, err := db.Ancient(stateHistoryStorageIndex, id-1) - if err != nil { - return nil - } - return blob +func ReadStateStorageIndex(db ethdb.AncientReaderOp, id uint64, offset, length int) ([]byte, error) { + return db.AncientBytes(stateHistoryStorageIndex, id-1, uint64(offset), uint64(length)) } // ReadStateAccountHistory retrieves the concatenated account data blob for the // specified state history. Offsets and lengths are resolved via the account // index. Compute the position of state history in freezer by minus one since // the id of first state history starts from one (zero for initial state). -func ReadStateAccountHistory(db ethdb.AncientReaderOp, id uint64) []byte { - blob, err := db.Ancient(stateHistoryAccountData, id-1) - if err != nil { - return nil - } - return blob +func ReadStateAccountHistory(db ethdb.AncientReaderOp, id uint64, offset, length int) ([]byte, error) { + return db.AncientBytes(stateHistoryAccountData, id-1, uint64(offset), uint64(length)) } // ReadStateStorageHistory retrieves the concatenated storage slot data blob for @@ -213,12 +205,8 @@ func ReadStateAccountHistory(db ethdb.AncientReaderOp, id uint64) []byte { // storage indexes. Compute the position of state history in freezer by minus // one since the id of first state history starts from one (zero for initial // state). -func ReadStateStorageHistory(db ethdb.AncientReaderOp, id uint64) []byte { - blob, err := db.Ancient(stateHistoryStorageData, id-1) - if err != nil { - return nil - } - return blob +func ReadStateStorageHistory(db ethdb.AncientReaderOp, id uint64, offset, length int) ([]byte, error) { + return db.AncientBytes(stateHistoryStorageData, id-1, uint64(offset), uint64(length)) } // ReadStateHistory retrieves the state history from database with provided id. diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index c12f2ab8fe..d33f7ce33d 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -403,6 +403,10 @@ func (f *chainFreezer) AncientRange(kind string, start, count, maxBytes uint64) return f.ancients.AncientRange(kind, start, count, maxBytes) } +func (f *chainFreezer) AncientBytes(kind string, id, offset, length uint64) ([]byte, error) { + return f.ancients.AncientBytes(kind, id, offset, length) +} + func (f *chainFreezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (int64, error) { return f.ancients.ModifyAncients(fn) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 626d390c0d..724c90ead6 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -100,6 +100,12 @@ func (db *nofreezedb) AncientRange(kind string, start, max, maxByteSize uint64) return nil, errNotSupported } +// AncientBytes retrieves the value segment of the element specified by the id +// and value offsets. +func (db *nofreezedb) AncientBytes(kind string, id, offset, length uint64) ([]byte, error) { + return nil, errNotSupported +} + // Ancients returns an error as we don't have a backing chain freezer. func (db *nofreezedb) Ancients() (uint64, error) { return 0, errNotSupported diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 98ad174ce0..42cd2a7999 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -202,6 +202,15 @@ func (f *Freezer) AncientRange(kind string, start, count, maxBytes uint64) ([][] return nil, errUnknownTable } +// AncientBytes retrieves the value segment of the element specified by the id +// and value offsets. +func (f *Freezer) AncientBytes(kind string, id, offset, length uint64) ([]byte, error) { + if table := f.tables[kind]; table != nil { + return table.RetrieveBytes(id, offset, length) + } + return nil, errUnknownTable +} + // Ancients returns the length of the frozen items. func (f *Freezer) Ancients() (uint64, error) { return f.frozen.Load(), nil diff --git a/core/rawdb/freezer_memory.go b/core/rawdb/freezer_memory.go index f5621ac4c6..8cb4cc2006 100644 --- a/core/rawdb/freezer_memory.go +++ b/core/rawdb/freezer_memory.go @@ -412,3 +412,28 @@ func (f *MemoryFreezer) Reset() error { func (f *MemoryFreezer) AncientDatadir() (string, error) { return "", nil } + +// AncientBytes retrieves the value segment of the element specified by the id +// and value offsets. +func (f *MemoryFreezer) AncientBytes(kind string, id, offset, length uint64) ([]byte, error) { + f.lock.RLock() + defer f.lock.RUnlock() + + table := f.tables[kind] + if table == nil { + return nil, errUnknownTable + } + entries, err := table.retrieve(id, 1, 0) + if err != nil { + return nil, err + } + if len(entries) == 0 { + return nil, errOutOfBounds + } + data := entries[0] + + if offset > uint64(len(data)) || offset+length > uint64(len(data)) { + return nil, fmt.Errorf("requested range out of bounds: item size %d, offset %d, length %d", len(data), offset, length) + } + return data[offset : offset+length], nil +} diff --git a/core/rawdb/freezer_resettable.go b/core/rawdb/freezer_resettable.go index 9db71cfd0e..f531e668c3 100644 --- a/core/rawdb/freezer_resettable.go +++ b/core/rawdb/freezer_resettable.go @@ -126,6 +126,15 @@ func (f *resettableFreezer) AncientRange(kind string, start, count, maxBytes uin return f.freezer.AncientRange(kind, start, count, maxBytes) } +// AncientBytes retrieves the value segment of the element specified by the id +// and value offsets. +func (f *resettableFreezer) AncientBytes(kind string, id, offset, length uint64) ([]byte, error) { + f.lock.RLock() + defer f.lock.RUnlock() + + return f.freezer.AncientBytes(kind, id, offset, length) +} + // Ancients returns the length of the frozen items. func (f *resettableFreezer) Ancients() (uint64, error) { f.lock.RLock() diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index d3a29a73c6..01a754c5c8 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -1107,6 +1107,71 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i return output, sizes, nil } +// RetrieveBytes retrieves the value segment of the element specified by the id +// and value offsets. +func (t *freezerTable) RetrieveBytes(item, offset, length uint64) ([]byte, error) { + t.lock.RLock() + defer t.lock.RUnlock() + + if t.index == nil || t.head == nil || t.metadata.file == nil { + return nil, errClosed + } + items, hidden := t.items.Load(), t.itemHidden.Load() + if items <= item || hidden > item { + return nil, errOutOfBounds + } + + // Retrieves the index entries for the specified ID and its immediate successor + indices, err := t.getIndices(item, 1) + if err != nil { + return nil, err + } + index0, index1 := indices[0], indices[1] + + itemStart, itemLimit, fileId := index0.bounds(index1) + itemSize := itemLimit - itemStart + + dataFile, exist := t.files[fileId] + if !exist { + return nil, fmt.Errorf("missing data file %d", fileId) + } + + // Perform the partial read if no-compression was enabled upon + if t.config.noSnappy { + if offset > uint64(itemSize) || offset+length > uint64(itemSize) { + return nil, fmt.Errorf("requested range out of bounds: item size %d, offset %d, length %d", itemSize, offset, length) + } + itemStart += uint32(offset) + + buf := make([]byte, length) + _, err = dataFile.ReadAt(buf, int64(itemStart)) + if err != nil { + return nil, err + } + t.readMeter.Mark(int64(length)) + return buf, nil + } else { + // If compressed, read the full item, decompress, then slice. + // Unfortunately, in this case, there is no performance gain + // by performing the partial read at all. + buf := make([]byte, itemSize) + _, err = dataFile.ReadAt(buf, int64(itemStart)) + if err != nil { + return nil, err + } + t.readMeter.Mark(int64(itemSize)) + + data, err := snappy.Decode(nil, buf) + if err != nil { + return nil, err + } + if offset > uint64(len(data)) || offset+length > uint64(len(data)) { + return nil, fmt.Errorf("requested range out of bounds: item size %d, offset %d, length %d", len(data), offset, length) + } + return data[offset : offset+length], nil + } +} + // size returns the total data size in the freezer table. func (t *freezerTable) size() (uint64, error) { t.lock.RLock() diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 96edac7e4a..fc21ea6c63 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -1571,3 +1571,65 @@ func TestTailTruncationCrash(t *testing.T) { t.Fatalf("Unexpected index flush offset, want: %d, got: %d", 26*indexEntrySize, f.metadata.flushOffset) } } + +func TestFreezerAncientBytes(t *testing.T) { + t.Parallel() + types := []struct { + name string + config freezerTableConfig + }{ + {"uncompressed", freezerTableConfig{noSnappy: true}}, + {"compressed", freezerTableConfig{noSnappy: false}}, + } + for _, typ := range types { + t.Run(typ.name, func(t *testing.T) { + f, err := newTable(os.TempDir(), fmt.Sprintf("ancientbytes-%s-%d", typ.name, rand.Uint64()), metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 1000, typ.config, false) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + for i := 0; i < 10; i++ { + data := getChunk(100, i) + batch := f.newBatch() + require.NoError(t, batch.AppendRaw(uint64(i), data)) + require.NoError(t, batch.commit()) + } + + for i := 0; i < 10; i++ { + full, err := f.Retrieve(uint64(i)) + require.NoError(t, err) + + // Full read + got, err := f.RetrieveBytes(uint64(i), 0, uint64(len(full))) + require.NoError(t, err) + if !bytes.Equal(got, full) { + t.Fatalf("full read mismatch for entry %d", i) + } + // Empty read + got, err = f.RetrieveBytes(uint64(i), 0, 0) + require.NoError(t, err) + if !bytes.Equal(got, full[:0]) { + t.Fatalf("empty read mismatch for entry %d", i) + } + // Middle slice + got, err = f.RetrieveBytes(uint64(i), 10, 50) + require.NoError(t, err) + if !bytes.Equal(got, full[10:60]) { + t.Fatalf("middle slice mismatch for entry %d", i) + } + // Single byte + got, err = f.RetrieveBytes(uint64(i), 99, 1) + require.NoError(t, err) + if !bytes.Equal(got, full[99:100]) { + t.Fatalf("single byte mismatch for entry %d", i) + } + // Out of bounds + _, err = f.RetrieveBytes(uint64(i), 100, 1) + if err == nil { + t.Fatalf("expected error for out-of-bounds read for entry %d", i) + } + } + }) + } +} diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 45c8aecf0c..d38afdaa35 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -62,6 +62,12 @@ func (t *table) AncientRange(kind string, start, count, maxBytes uint64) ([][]by return t.db.AncientRange(kind, start, count, maxBytes) } +// AncientBytes is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) AncientBytes(kind string, id, offset, length uint64) ([]byte, error) { + return t.db.AncientBytes(kind, id, offset, length) +} + // Ancients is a noop passthrough that just forwards the request to the underlying // database. func (t *table) Ancients() (uint64, error) { diff --git a/core/state/database.go b/core/state/database.go index 3a0ac422ee..58d0ccfe82 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -302,6 +302,8 @@ func mustCopyTrie(t Trie) Trie { return t.Copy() case *trie.VerkleTrie: return t.Copy() + case *trie.TransitionTrie: + return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) } diff --git a/core/state/state_object.go b/core/state/state_object.go index 2938750503..fdeb4254c1 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/holiman/uint256" ) @@ -494,8 +495,20 @@ func (s *stateObject) deepCopy(db *StateDB) *stateObject { selfDestructed: s.selfDestructed, newContract: s.newContract, } - if s.trie != nil { + + switch s.trie.(type) { + case *trie.VerkleTrie: + // Verkle uses only one tree, and the copy has already been + // made in mustCopyTrie. + obj.trie = db.trie + case *trie.TransitionTrie: + // Same thing for the transition tree, since the MPT is + // read-only. + obj.trie = db.trie + case *trie.StateTrie: obj.trie = mustCopyTrie(s.trie) + case nil: + // do nothing } return obj } diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index e199d21c7a..ceedc74a53 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -23,7 +23,6 @@ import ( "math" "math/big" "slices" - "sort" "sync" "sync/atomic" "time" @@ -238,11 +237,10 @@ type LegacyPool struct { pendingNonces *noncer // Pending state tracking virtual nonces reserver txpool.Reserver // Address reserver to ensure exclusivity across subpools - pending map[common.Address]*list // All currently processable transactions - queue map[common.Address]*list // Queued but non-processable transactions - beats map[common.Address]time.Time // Last heartbeat from each known account - all *lookup // All transactions to allow lookups - priced *pricedList // All transactions sorted by price + pending map[common.Address]*list // All currently processable transactions + queue *queue + all *lookup // All transactions to allow lookups + priced *pricedList // All transactions sorted by price reqResetCh chan *txpoolResetRequest reqPromoteCh chan *accountSet @@ -266,14 +264,14 @@ func New(config Config, chain BlockChain) *LegacyPool { config = (&config).sanitize() // Create the transaction pool with its initial settings + signer := types.LatestSigner(chain.Config()) pool := &LegacyPool{ config: config, chain: chain, chainconfig: chain.Config(), - signer: types.LatestSigner(chain.Config()), + signer: signer, pending: make(map[common.Address]*list), - queue: make(map[common.Address]*list), - beats: make(map[common.Address]time.Time), + queue: newQueue(config, signer), all: newLookup(), reqResetCh: make(chan *txpoolResetRequest), reqPromoteCh: make(chan *accountSet), @@ -369,15 +367,8 @@ func (pool *LegacyPool) loop() { // Handle inactive account transaction eviction case <-evict.C: pool.mu.Lock() - for addr := range pool.queue { - // Any old enough should be removed - if time.Since(pool.beats[addr]) > pool.config.Lifetime { - list := pool.queue[addr].Flatten() - for _, tx := range list { - pool.removeTx(tx.Hash(), true, true) - } - queuedEvictionMeter.Mark(int64(len(list))) - } + for _, hash := range pool.queue.evictList() { + pool.removeTx(hash, true, true) } pool.mu.Unlock() } @@ -459,11 +450,7 @@ func (pool *LegacyPool) stats() (int, int) { for _, list := range pool.pending { pending += list.Len() } - queued := 0 - for _, list := range pool.queue { - queued += list.Len() - } - return pending, queued + return pending, pool.queue.stats() } // Content retrieves the data content of the transaction pool, returning all the @@ -476,10 +463,7 @@ func (pool *LegacyPool) Content() (map[common.Address][]*types.Transaction, map[ for addr, list := range pool.pending { pending[addr] = list.Flatten() } - queued := make(map[common.Address][]*types.Transaction, len(pool.queue)) - for addr, list := range pool.queue { - queued[addr] = list.Flatten() - } + queued := pool.queue.content() return pending, queued } @@ -493,10 +477,7 @@ func (pool *LegacyPool) ContentFrom(addr common.Address) ([]*types.Transaction, if list, ok := pool.pending[addr]; ok { pending = list.Flatten() } - var queued []*types.Transaction - if list, ok := pool.queue[addr]; ok { - queued = list.Flatten() - } + queued := pool.queue.contentFrom(addr) return pending, queued } @@ -644,7 +625,7 @@ func (pool *LegacyPool) validateAuth(tx *types.Transaction) error { if pending := pool.pending[auth]; pending != nil { count += pending.Len() } - if queue := pool.queue[auth]; queue != nil { + if queue, ok := pool.queue.get(auth); ok { count += queue.Len() } if count > 1 { @@ -691,7 +672,7 @@ func (pool *LegacyPool) add(tx *types.Transaction) (replaced bool, err error) { // only by this subpool until all transactions are evicted var ( _, hasPending = pool.pending[from] - _, hasQueued = pool.queue[from] + _, hasQueued = pool.queue.get(from) ) if !hasPending && !hasQueued { if err := pool.reserver.Hold(from); err != nil { @@ -790,7 +771,7 @@ func (pool *LegacyPool) add(tx *types.Transaction) (replaced bool, err error) { log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) // Successful promotion, bump the heartbeat - pool.beats[from] = time.Now() + pool.queue.bump(from) return old != nil, nil } // New transaction isn't replacing a pending one, push into queue @@ -815,7 +796,7 @@ func (pool *LegacyPool) isGapped(from common.Address, tx *types.Transaction) boo } // The transaction has a nonce gap with pending list, it's only considered // as executable if transactions in queue can fill up the nonce gap. - queue, ok := pool.queue[from] + queue, ok := pool.queue.get(from) if !ok { return true } @@ -831,25 +812,12 @@ func (pool *LegacyPool) isGapped(from common.Address, tx *types.Transaction) boo // // Note, this method assumes the pool lock is held! func (pool *LegacyPool) enqueueTx(hash common.Hash, tx *types.Transaction, addAll bool) (bool, error) { - // Try to insert the transaction into the future queue - from, _ := types.Sender(pool.signer, tx) // already validated - if pool.queue[from] == nil { - pool.queue[from] = newList(false) + replaced, err := pool.queue.add(tx) + if err != nil { + return false, err } - inserted, old := pool.queue[from].Add(tx, pool.config.PriceBump) - if !inserted { - // An older transaction was better, discard this - queuedDiscardMeter.Mark(1) - return false, txpool.ErrReplaceUnderpriced - } - // Discard any previous transaction and mark this - if old != nil { - pool.all.Remove(old.Hash()) - pool.priced.Removed(1) - queuedReplaceMeter.Mark(1) - } else { - // Nothing was replaced, bump the queued counter - queuedGauge.Inc(1) + if replaced != nil { + pool.removeTx(*replaced, true, true) } // If the transaction isn't in lookup set but it's expected to be there, // show the error log. @@ -860,11 +828,7 @@ func (pool *LegacyPool) enqueueTx(hash common.Hash, tx *types.Transaction, addAl pool.all.Add(tx) pool.priced.Put(tx) } - // If we never record the heartbeat, do it right now. - if _, exist := pool.beats[from]; !exist { - pool.beats[from] = time.Now() - } - return old != nil, nil + return replaced != nil, nil } // promoteTx adds a transaction to the pending (processable) list of transactions @@ -899,7 +863,7 @@ func (pool *LegacyPool) promoteTx(addr common.Address, hash common.Hash, tx *typ pool.pendingNonces.set(addr, tx.Nonce()+1) // Successful promotion, bump the heartbeat - pool.beats[addr] = time.Now() + pool.queue.bump(addr) return true } @@ -1019,7 +983,7 @@ func (pool *LegacyPool) Status(hash common.Hash) txpool.TxStatus { if txList := pool.pending[from]; txList != nil && txList.txs.items[tx.Nonce()] != nil { return txpool.TxStatusPending - } else if txList := pool.queue[from]; txList != nil && txList.txs.items[tx.Nonce()] != nil { + } else if txList, ok := pool.queue.get(from); ok && txList.txs.items[tx.Nonce()] != nil { return txpool.TxStatusQueued } return txpool.TxStatusUnknown @@ -1096,7 +1060,7 @@ func (pool *LegacyPool) removeTx(hash common.Hash, outofbound bool, unreserve bo defer func() { var ( _, hasPending = pool.pending[addr] - _, hasQueued = pool.queue[addr] + _, hasQueued = pool.queue.get(addr) ) if !hasPending && !hasQueued { pool.reserver.Release(addr) @@ -1128,16 +1092,7 @@ func (pool *LegacyPool) removeTx(hash common.Hash, outofbound bool, unreserve bo } } // Transaction is in the future queue - if future := pool.queue[addr]; future != nil { - if removed, _ := future.Remove(tx); removed { - // Reduce the queued counter - queuedGauge.Dec(1) - } - if future.Empty() { - delete(pool.queue, addr) - delete(pool.beats, addr) - } - } + pool.queue.remove(addr, tx) return 0 } @@ -1285,10 +1240,7 @@ func (pool *LegacyPool) runReorg(done chan struct{}, reset *txpoolResetRequest, } } // Reset needs promote for all addresses - promoteAddrs = make([]common.Address, 0, len(pool.queue)) - for addr := range pool.queue { - promoteAddrs = append(promoteAddrs, addr) - } + promoteAddrs = pool.queue.addresses() } // Check for pending transactions for every account that sent new ones promoted := pool.promoteExecutables(promoteAddrs) @@ -1442,60 +1394,29 @@ func (pool *LegacyPool) reset(oldHead, newHead *types.Header) { // future queue to the set of pending transactions. During this process, all // invalidated transactions (low nonce, low balance) are deleted. func (pool *LegacyPool) promoteExecutables(accounts []common.Address) []*types.Transaction { - // Track the promoted transactions to broadcast them at once - var promoted []*types.Transaction - - // Iterate over all accounts and promote any executable transactions gasLimit := pool.currentHead.Load().GasLimit - for _, addr := range accounts { - list := pool.queue[addr] - if list == nil { - continue // Just in case someone calls with a non existing account - } - // Drop all transactions that are deemed too old (low nonce) - forwards := list.Forward(pool.currentState.GetNonce(addr)) - for _, tx := range forwards { - pool.all.Remove(tx.Hash()) - } - log.Trace("Removed old queued transactions", "count", len(forwards)) - // Drop all transactions that are too costly (low balance or out of gas) - drops, _ := list.Filter(pool.currentState.GetBalance(addr), gasLimit) - for _, tx := range drops { - pool.all.Remove(tx.Hash()) - } - log.Trace("Removed unpayable queued transactions", "count", len(drops)) - queuedNofundsMeter.Mark(int64(len(drops))) + promotable, dropped, removedAddresses := pool.queue.promoteExecutables(accounts, gasLimit, pool.currentState, pool.pendingNonces) - // Gather all executable transactions and promote them - readies := list.Ready(pool.pendingNonces.get(addr)) - for _, tx := range readies { - hash := tx.Hash() - if pool.promoteTx(addr, hash, tx) { - promoted = append(promoted, tx) - } + // promote all promotable transactions + promoted := make([]*types.Transaction, 0, len(promotable)) + for _, tx := range promotable { + from, _ := pool.signer.Sender(tx) + if pool.promoteTx(from, tx.Hash(), tx) { + promoted = append(promoted, tx) } - log.Trace("Promoted queued transactions", "count", len(promoted)) - queuedGauge.Dec(int64(len(readies))) + } - // Drop all transactions over the allowed limit - var caps = list.Cap(int(pool.config.AccountQueue)) - for _, tx := range caps { - hash := tx.Hash() - pool.all.Remove(hash) - log.Trace("Removed cap-exceeding queued transaction", "hash", hash) - } - queuedRateLimitMeter.Mark(int64(len(caps))) - // Mark all the items dropped as removed - pool.priced.Removed(len(forwards) + len(drops) + len(caps)) - queuedGauge.Dec(int64(len(forwards) + len(drops) + len(caps))) + // remove all removable transactions + for _, hash := range dropped { + pool.all.Remove(hash) + } + pool.priced.Removed(len(dropped)) - // Delete the entire queue entry if it became empty. - if list.Empty() { - delete(pool.queue, addr) - delete(pool.beats, addr) - if _, ok := pool.pending[addr]; !ok { - pool.reserver.Release(addr) - } + // release all accounts that have no more transactions in the pool + for _, addr := range removedAddresses { + _, hasPending := pool.pending[addr] + if !hasPending { + pool.reserver.Release(addr) } } return promoted @@ -1585,43 +1506,18 @@ func (pool *LegacyPool) truncatePending() { // truncateQueue drops the oldest transactions in the queue if the pool is above the global queue limit. func (pool *LegacyPool) truncateQueue() { - queued := uint64(0) - for _, list := range pool.queue { - queued += uint64(list.Len()) - } - if queued <= pool.config.GlobalQueue { - return + removed, removedAddresses := pool.queue.truncate() + + // Remove all removable transactions from the lookup and global price list + for _, hash := range removed { + pool.all.Remove(hash) } + pool.priced.Removed(len(removed)) - // Sort all accounts with queued transactions by heartbeat - addresses := make(addressesByHeartbeat, 0, len(pool.queue)) - for addr := range pool.queue { - addresses = append(addresses, addressByHeartbeat{addr, pool.beats[addr]}) - } - sort.Sort(sort.Reverse(addresses)) - - // Drop transactions until the total is below the limit - for drop := queued - pool.config.GlobalQueue; drop > 0 && len(addresses) > 0; { - addr := addresses[len(addresses)-1] - list := pool.queue[addr.address] - - addresses = addresses[:len(addresses)-1] - - // Drop all transactions if they are less than the overflow - if size := uint64(list.Len()); size <= drop { - for _, tx := range list.Flatten() { - pool.removeTx(tx.Hash(), true, true) - } - drop -= size - queuedRateLimitMeter.Mark(int64(size)) - continue - } - // Otherwise drop only last few transactions - txs := list.Flatten() - for i := len(txs) - 1; i >= 0 && drop > 0; i-- { - pool.removeTx(txs[i].Hash(), true, true) - drop-- - queuedRateLimitMeter.Mark(1) + for _, addr := range removedAddresses { + _, hasPending := pool.pending[addr] + if !hasPending { + pool.reserver.Release(addr) } } } @@ -1679,25 +1575,13 @@ func (pool *LegacyPool) demoteUnexecutables() { // Delete the entire pending entry if it became empty. if list.Empty() { delete(pool.pending, addr) - if _, ok := pool.queue[addr]; !ok { + if _, ok := pool.queue.get(addr); !ok { pool.reserver.Release(addr) } } } } -// addressByHeartbeat is an account address tagged with its last activity timestamp. -type addressByHeartbeat struct { - address common.Address - heartbeat time.Time -} - -type addressesByHeartbeat []addressByHeartbeat - -func (a addressesByHeartbeat) Len() int { return len(a) } -func (a addressesByHeartbeat) Less(i, j int) bool { return a[i].heartbeat.Before(a[j].heartbeat) } -func (a addressesByHeartbeat) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - // accountSet is simply a set of addresses to check for existence, and a signer // capable of deriving addresses from transactions. type accountSet struct { @@ -1938,17 +1822,17 @@ func (pool *LegacyPool) Clear() { // acquire the subpool lock until the transaction addition is completed. for addr := range pool.pending { - if _, ok := pool.queue[addr]; !ok { + if _, ok := pool.queue.get(addr); !ok { pool.reserver.Release(addr) } } - for addr := range pool.queue { + for _, addr := range pool.queue.addresses() { pool.reserver.Release(addr) } pool.all.Clear() pool.priced.Reheap() pool.pending = make(map[common.Address]*list) - pool.queue = make(map[common.Address]*list) + pool.queue = newQueue(pool.config, pool.signer) pool.pendingNonces = newNoncer(pool.currentState) } diff --git a/core/txpool/legacypool/legacypool_test.go b/core/txpool/legacypool/legacypool_test.go index 0c8642659d..fb994d8208 100644 --- a/core/txpool/legacypool/legacypool_test.go +++ b/core/txpool/legacypool/legacypool_test.go @@ -466,8 +466,8 @@ func TestQueue(t *testing.T) { if _, ok := pool.pending[from].txs.items[tx.Nonce()]; ok { t.Error("expected transaction to be in tx pool") } - if len(pool.queue) > 0 { - t.Error("expected transaction queue to be empty. is", len(pool.queue)) + if len(pool.queue.queued) > 0 { + t.Error("expected transaction queue to be empty. is", len(pool.queue.queued)) } } @@ -492,8 +492,8 @@ func TestQueue2(t *testing.T) { if len(pool.pending) != 1 { t.Error("expected pending length to be 1, got", len(pool.pending)) } - if pool.queue[from].Len() != 2 { - t.Error("expected len(queue) == 2, got", pool.queue[from].Len()) + if list, _ := pool.queue.get(from); list.Len() != 2 { + t.Error("expected len(queue) == 2, got", list.Len()) } } @@ -639,8 +639,8 @@ func TestMissingNonce(t *testing.T) { if len(pool.pending) != 0 { t.Error("expected 0 pending transactions, got", len(pool.pending)) } - if pool.queue[addr].Len() != 1 { - t.Error("expected 1 queued transaction, got", pool.queue[addr].Len()) + if list, _ := pool.queue.get(addr); list.Len() != 1 { + t.Error("expected 1 queued transaction, got", list.Len()) } if pool.all.Count() != 1 { t.Error("expected 1 total transactions, got", pool.all.Count()) @@ -712,8 +712,8 @@ func TestDropping(t *testing.T) { if pool.pending[account].Len() != 3 { t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), 3) } - if pool.queue[account].Len() != 3 { - t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 3) + if list, _ := pool.queue.get(account); list.Len() != 3 { + t.Errorf("queued transaction mismatch: have %d, want %d", list.Len(), 3) } if pool.all.Count() != 6 { t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 6) @@ -722,8 +722,8 @@ func TestDropping(t *testing.T) { if pool.pending[account].Len() != 3 { t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), 3) } - if pool.queue[account].Len() != 3 { - t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 3) + if list, _ := pool.queue.get(account); list.Len() != 3 { + t.Errorf("queued transaction mismatch: have %d, want %d", list.Len(), 3) } if pool.all.Count() != 6 { t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 6) @@ -741,13 +741,14 @@ func TestDropping(t *testing.T) { if _, ok := pool.pending[account].txs.items[tx2.Nonce()]; ok { t.Errorf("out-of-fund pending transaction present: %v", tx1) } - if _, ok := pool.queue[account].txs.items[tx10.Nonce()]; !ok { + list, _ := pool.queue.get(account) + if _, ok := list.txs.items[tx10.Nonce()]; !ok { t.Errorf("funded queued transaction missing: %v", tx10) } - if _, ok := pool.queue[account].txs.items[tx11.Nonce()]; !ok { + if _, ok := list.txs.items[tx11.Nonce()]; !ok { t.Errorf("funded queued transaction missing: %v", tx10) } - if _, ok := pool.queue[account].txs.items[tx12.Nonce()]; ok { + if _, ok := list.txs.items[tx12.Nonce()]; ok { t.Errorf("out-of-fund queued transaction present: %v", tx11) } if pool.all.Count() != 4 { @@ -763,10 +764,11 @@ func TestDropping(t *testing.T) { if _, ok := pool.pending[account].txs.items[tx1.Nonce()]; ok { t.Errorf("over-gased pending transaction present: %v", tx1) } - if _, ok := pool.queue[account].txs.items[tx10.Nonce()]; !ok { + list, _ = pool.queue.get(account) + if _, ok := list.txs.items[tx10.Nonce()]; !ok { t.Errorf("funded queued transaction missing: %v", tx10) } - if _, ok := pool.queue[account].txs.items[tx11.Nonce()]; ok { + if _, ok := list.txs.items[tx11.Nonce()]; ok { t.Errorf("over-gased queued transaction present: %v", tx11) } if pool.all.Count() != 2 { @@ -820,8 +822,8 @@ func TestPostponing(t *testing.T) { if pending := pool.pending[accs[0]].Len() + pool.pending[accs[1]].Len(); pending != len(txs) { t.Errorf("pending transaction mismatch: have %d, want %d", pending, len(txs)) } - if len(pool.queue) != 0 { - t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0) + if len(pool.queue.addresses()) != 0 { + t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue.addresses()), 0) } if pool.all.Count() != len(txs) { t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), len(txs)) @@ -830,8 +832,8 @@ func TestPostponing(t *testing.T) { if pending := pool.pending[accs[0]].Len() + pool.pending[accs[1]].Len(); pending != len(txs) { t.Errorf("pending transaction mismatch: have %d, want %d", pending, len(txs)) } - if len(pool.queue) != 0 { - t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0) + if len(pool.queue.addresses()) != 0 { + t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue.addresses()), 0) } if pool.all.Count() != len(txs) { t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), len(txs)) @@ -847,7 +849,8 @@ func TestPostponing(t *testing.T) { if _, ok := pool.pending[accs[0]].txs.items[txs[0].Nonce()]; !ok { t.Errorf("tx %d: valid and funded transaction missing from pending pool: %v", 0, txs[0]) } - if _, ok := pool.queue[accs[0]].txs.items[txs[0].Nonce()]; ok { + list, _ := pool.queue.get(accs[0]) + if _, ok := list.txs.items[txs[0].Nonce()]; ok { t.Errorf("tx %d: valid and funded transaction present in future queue: %v", 0, txs[0]) } for i, tx := range txs[1:100] { @@ -855,14 +858,14 @@ func TestPostponing(t *testing.T) { if _, ok := pool.pending[accs[0]].txs.items[tx.Nonce()]; ok { t.Errorf("tx %d: valid but future transaction present in pending pool: %v", i+1, tx) } - if _, ok := pool.queue[accs[0]].txs.items[tx.Nonce()]; !ok { + if _, ok := list.txs.items[tx.Nonce()]; !ok { t.Errorf("tx %d: valid but future transaction missing from future queue: %v", i+1, tx) } } else { if _, ok := pool.pending[accs[0]].txs.items[tx.Nonce()]; ok { t.Errorf("tx %d: out-of-fund transaction present in pending pool: %v", i+1, tx) } - if _, ok := pool.queue[accs[0]].txs.items[tx.Nonce()]; ok { + if _, ok := list.txs.items[tx.Nonce()]; ok { t.Errorf("tx %d: out-of-fund transaction present in future queue: %v", i+1, tx) } } @@ -872,13 +875,14 @@ func TestPostponing(t *testing.T) { if pool.pending[accs[1]] != nil { t.Errorf("invalidated account still has pending transactions") } + list, _ = pool.queue.get(accs[1]) for i, tx := range txs[100:] { if i%2 == 1 { - if _, ok := pool.queue[accs[1]].txs.items[tx.Nonce()]; !ok { + if _, ok := list.txs.items[tx.Nonce()]; !ok { t.Errorf("tx %d: valid but future transaction missing from future queue: %v", 100+i, tx) } } else { - if _, ok := pool.queue[accs[1]].txs.items[tx.Nonce()]; ok { + if _, ok := list.txs.items[tx.Nonce()]; ok { t.Errorf("tx %d: out-of-fund transaction present in future queue: %v", 100+i, tx) } } @@ -963,13 +967,14 @@ func TestQueueAccountLimiting(t *testing.T) { if len(pool.pending) != 0 { t.Errorf("tx %d: pending pool size mismatch: have %d, want %d", i, len(pool.pending), 0) } + list, _ := pool.queue.get(account) if i <= testTxPoolConfig.AccountQueue { - if pool.queue[account].Len() != int(i) { - t.Errorf("tx %d: queue size mismatch: have %d, want %d", i, pool.queue[account].Len(), i) + if list.Len() != int(i) { + t.Errorf("tx %d: queue size mismatch: have %d, want %d", i, list.Len(), i) } } else { - if pool.queue[account].Len() != int(testTxPoolConfig.AccountQueue) { - t.Errorf("tx %d: queue limit mismatch: have %d, want %d", i, pool.queue[account].Len(), testTxPoolConfig.AccountQueue) + if list.Len() != int(testTxPoolConfig.AccountQueue) { + t.Errorf("tx %d: queue limit mismatch: have %d, want %d", i, list.Len(), testTxPoolConfig.AccountQueue) } } } @@ -1020,7 +1025,7 @@ func TestQueueGlobalLimiting(t *testing.T) { pool.addRemotesSync(txs) queued := 0 - for addr, list := range pool.queue { + for addr, list := range pool.queue.queued { if list.Len() > int(config.AccountQueue) { t.Errorf("addr %x: queued accounts overflown allowance: %d > %d", addr, list.Len(), config.AccountQueue) } @@ -1179,8 +1184,8 @@ func TestPendingLimiting(t *testing.T) { if pool.pending[account].Len() != int(i)+1 { t.Errorf("tx %d: pending pool size mismatch: have %d, want %d", i, pool.pending[account].Len(), i+1) } - if len(pool.queue) != 0 { - t.Errorf("tx %d: queue size mismatch: have %d, want %d", i, pool.queue[account].Len(), 0) + if len(pool.queue.addresses()) != 0 { + t.Errorf("tx %d: queue size mismatch: have %d, want %d", i, len(pool.queue.addresses()), 0) } } if pool.all.Count() != int(testTxPoolConfig.AccountQueue+5) { diff --git a/core/txpool/legacypool/queue.go b/core/txpool/legacypool/queue.go new file mode 100644 index 0000000000..a889debe37 --- /dev/null +++ b/core/txpool/legacypool/queue.go @@ -0,0 +1,275 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package legacypool + +import ( + "sort" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +// queue manages nonce-gapped transactions that have been validated but are +// not yet processable. +type queue struct { + config Config + signer types.Signer + queued map[common.Address]*list // Queued but non-processable transactions + beats map[common.Address]time.Time // Last heartbeat from each known account +} + +func newQueue(config Config, signer types.Signer) *queue { + return &queue{ + signer: signer, + config: config, + queued: make(map[common.Address]*list), + beats: make(map[common.Address]time.Time), + } +} + +// evictList returns the hashes of transactions that are old enough to be evicted. +func (q *queue) evictList() []common.Hash { + var removed []common.Hash + for addr, list := range q.queued { + if time.Since(q.beats[addr]) > q.config.Lifetime { + for _, tx := range list.Flatten() { + removed = append(removed, tx.Hash()) + } + } + } + queuedEvictionMeter.Mark(int64(len(removed))) + return removed +} + +func (q *queue) stats() int { + queued := 0 + for _, list := range q.queued { + queued += list.Len() + } + return queued +} + +func (q *queue) content() map[common.Address][]*types.Transaction { + queued := make(map[common.Address][]*types.Transaction, len(q.queued)) + for addr, list := range q.queued { + queued[addr] = list.Flatten() + } + return queued +} + +func (q *queue) contentFrom(addr common.Address) []*types.Transaction { + var queued []*types.Transaction + if list, ok := q.get(addr); ok { + queued = list.Flatten() + } + return queued +} + +func (q *queue) get(addr common.Address) (*list, bool) { + l, ok := q.queued[addr] + return l, ok +} + +func (q *queue) bump(addr common.Address) { + q.beats[addr] = time.Now() +} + +func (q *queue) addresses() []common.Address { + addrs := make([]common.Address, 0, len(q.queued)) + for addr := range q.queued { + addrs = append(addrs, addr) + } + return addrs +} + +func (q *queue) remove(addr common.Address, tx *types.Transaction) { + if future := q.queued[addr]; future != nil { + if txOld := future.txs.Get(tx.Nonce()); txOld != nil && txOld.Hash() != tx.Hash() { + // Edge case, a different transaction + // with the same nonce is in the queued, just ignore + return + } + if removed, _ := future.Remove(tx); removed { + // Reduce the queued counter + queuedGauge.Dec(1) + } + if future.Empty() { + delete(q.queued, addr) + delete(q.beats, addr) + } + } +} + +func (q *queue) add(tx *types.Transaction) (*common.Hash, error) { + // Try to insert the transaction into the future queue + from, _ := types.Sender(q.signer, tx) // already validated + if q.queued[from] == nil { + q.queued[from] = newList(false) + } + inserted, old := q.queued[from].Add(tx, q.config.PriceBump) + if !inserted { + // An older transaction was better, discard this + queuedDiscardMeter.Mark(1) + return nil, txpool.ErrReplaceUnderpriced + } + // If we never record the heartbeat, do it right now. + if _, exist := q.beats[from]; !exist { + q.beats[from] = time.Now() + } + if old == nil { + // Nothing was replaced, bump the queued counter + queuedGauge.Inc(1) + return nil, nil + } + h := old.Hash() + // Transaction was replaced, bump the replacement counter + queuedReplaceMeter.Mark(1) + return &h, nil +} + +// promoteExecutables iterates over all accounts with queued transactions, selecting +// for promotion any that are now executable. It also drops any transactions that are +// deemed too old (nonce too low) or too costly (insufficient funds or over gas limit). +// +// Returns three lists: +// - all transactions that were removed from the queue and selected for promotion; +// - all other transactions that were removed from the queue and dropped; +// - the list of addresses removed. +func (q *queue) promoteExecutables(accounts []common.Address, gasLimit uint64, currentState *state.StateDB, nonces *noncer) ([]*types.Transaction, []common.Hash, []common.Address) { + // Track the promotable transactions to broadcast them at once + var ( + promotable []*types.Transaction + dropped []common.Hash + removedAddresses []common.Address + ) + // Iterate over all accounts and promote any executable transactions + for _, addr := range accounts { + list := q.queued[addr] + if list == nil { + continue // Just in case someone calls with a non existing account + } + // Drop all transactions that are deemed too old (low nonce) + forwards := list.Forward(currentState.GetNonce(addr)) + for _, tx := range forwards { + dropped = append(dropped, tx.Hash()) + } + log.Trace("Removing old queued transactions", "count", len(forwards)) + + // Drop all transactions that are too costly (low balance or out of gas) + drops, _ := list.Filter(currentState.GetBalance(addr), gasLimit) + for _, tx := range drops { + dropped = append(dropped, tx.Hash()) + } + log.Trace("Removing unpayable queued transactions", "count", len(drops)) + queuedNofundsMeter.Mark(int64(len(drops))) + + // Gather all executable transactions and promote them + readies := list.Ready(nonces.get(addr)) + promotable = append(promotable, readies...) + log.Trace("Promoting queued transactions", "count", len(promotable)) + queuedGauge.Dec(int64(len(readies))) + + // Drop all transactions over the allowed limit + var caps = list.Cap(int(q.config.AccountQueue)) + for _, tx := range caps { + hash := tx.Hash() + dropped = append(dropped, hash) + log.Trace("Removing cap-exceeding queued transaction", "hash", hash) + } + queuedRateLimitMeter.Mark(int64(len(caps))) + + // Delete the entire queue entry if it became empty. + if list.Empty() { + delete(q.queued, addr) + delete(q.beats, addr) + removedAddresses = append(removedAddresses, addr) + } + } + queuedGauge.Dec(int64(len(dropped))) + return promotable, dropped, removedAddresses +} + +// truncate drops the oldest transactions from the queue until the total +// number is below the configured limit. Returns the hashes of all dropped +// transactions and the addresses of accounts that became empty due to +// the truncation. +func (q *queue) truncate() ([]common.Hash, []common.Address) { + queued := uint64(0) + for _, list := range q.queued { + queued += uint64(list.Len()) + } + if queued <= q.config.GlobalQueue { + return nil, nil + } + + // Sort all accounts with queued transactions by heartbeat + addresses := make(addressesByHeartbeat, 0, len(q.queued)) + for addr := range q.queued { + addresses = append(addresses, addressByHeartbeat{addr, q.beats[addr]}) + } + sort.Sort(sort.Reverse(addresses)) + + // Drop transactions until the total is below the limit + var ( + removed = make([]common.Hash, 0) + removedAddresses = make([]common.Address, 0) + ) + for drop := queued - q.config.GlobalQueue; drop > 0 && len(addresses) > 0; { + addr := addresses[len(addresses)-1] + list := q.queued[addr.address] + + addresses = addresses[:len(addresses)-1] + + // Drop all transactions if they are less than the overflow + if size := uint64(list.Len()); size <= drop { + for _, tx := range list.Flatten() { + q.remove(addr.address, tx) + removed = append(removed, tx.Hash()) + } + drop -= size + queuedRateLimitMeter.Mark(int64(size)) + removedAddresses = append(removedAddresses, addr.address) + continue + } + // Otherwise drop only last few transactions + txs := list.Flatten() + for i := len(txs) - 1; i >= 0 && drop > 0; i-- { + q.remove(addr.address, txs[i]) + removed = append(removed, txs[i].Hash()) + drop-- + queuedRateLimitMeter.Mark(1) + } + } + // No need to clear empty accounts, remove already does that + return removed, removedAddresses +} + +// addressByHeartbeat is an account address tagged with its last activity timestamp. +type addressByHeartbeat struct { + address common.Address + heartbeat time.Time +} + +type addressesByHeartbeat []addressByHeartbeat + +func (a addressesByHeartbeat) Len() int { return len(a) } +func (a addressesByHeartbeat) Less(i, j int) bool { return a[i].heartbeat.Before(a[j].heartbeat) } +func (a addressesByHeartbeat) Swap(i, j int) { a[i], a[j] = a[j], a[i] } diff --git a/core/types/bloom9.go b/core/types/bloom9.go index 5a6e49c220..1d57e8e4bc 100644 --- a/core/types/bloom9.go +++ b/core/types/bloom9.go @@ -21,6 +21,7 @@ import ( "fmt" "math/big" + "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" ) @@ -125,9 +126,7 @@ func MergeBloom(receipts Receipts) Bloom { for _, receipt := range receipts { if len(receipt.Logs) != 0 { bl := receipt.Bloom.Bytes() - for i := range bin { - bin[i] |= bl[i] - } + bitutil.ORBytes(bin[:], bin[:], bl) } } return bin diff --git a/eth/api_backend.go b/eth/api_backend.go index 3ae73e78af..766a99fc1e 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -486,3 +486,11 @@ func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, re func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (*types.Transaction, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { return b.eth.stateAtTransaction(ctx, block, txIndex, reexec) } + +func (b *EthAPIBackend) RPCTxSyncDefaultTimeout() time.Duration { + return b.eth.config.TxSyncDefaultTimeout +} + +func (b *EthAPIBackend) RPCTxSyncMaxTimeout() time.Duration { + return b.eth.config.TxSyncMaxTimeout +} diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 6dfe24f729..75b263bf6b 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -47,7 +47,6 @@ import ( // Register adds the engine API to the full node. func Register(stack *node.Node, backend *eth.Ethereum) error { - log.Warn("Engine API enabled", "protocol", "eth") stack.RegisterAPIs([]rpc.API{ { Namespace: "engine", diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 6020387bcd..c4a0956b3b 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -49,27 +49,29 @@ var FullNodeGPO = gasprice.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ - HistoryMode: history.KeepAll, - SyncMode: SnapSync, - NetworkId: 0, // enable auto configuration of networkID == chainID - TxLookupLimit: 2350000, - TransactionHistory: 2350000, - LogHistory: 2350000, - StateHistory: params.FullImmutabilityThreshold, - DatabaseCache: 512, - TrieCleanCache: 154, - TrieDirtyCache: 256, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 102, - FilterLogCacheSize: 32, - LogQueryLimit: 1000, - Miner: miner.DefaultConfig, - TxPool: legacypool.DefaultConfig, - BlobPool: blobpool.DefaultConfig, - RPCGasCap: 50000000, - RPCEVMTimeout: 5 * time.Second, - GPO: FullNodeGPO, - RPCTxFeeCap: 1, // 1 ether + HistoryMode: history.KeepAll, + SyncMode: SnapSync, + NetworkId: 0, // enable auto configuration of networkID == chainID + TxLookupLimit: 2350000, + TransactionHistory: 2350000, + LogHistory: 2350000, + StateHistory: params.FullImmutabilityThreshold, + DatabaseCache: 512, + TrieCleanCache: 154, + TrieDirtyCache: 256, + TrieTimeout: 60 * time.Minute, + SnapshotCache: 102, + FilterLogCacheSize: 32, + LogQueryLimit: 1000, + Miner: miner.DefaultConfig, + TxPool: legacypool.DefaultConfig, + BlobPool: blobpool.DefaultConfig, + RPCGasCap: 50000000, + RPCEVMTimeout: 5 * time.Second, + GPO: FullNodeGPO, + RPCTxFeeCap: 1, // 1 ether + TxSyncDefaultTimeout: 20 * time.Second, + TxSyncMaxTimeout: 1 * time.Minute, } //go:generate go run github.com/fjl/gencodec -type Config -formats toml -out gen_config.go @@ -183,6 +185,10 @@ type Config struct { // OverrideVerkle (TODO: remove after the fork) OverrideVerkle *uint64 `toml:",omitempty"` + + // EIP-7966: eth_sendRawTransactionSync timeouts + TxSyncDefaultTimeout time.Duration `toml:",omitempty"` + TxSyncMaxTimeout time.Duration `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain config. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 6f6e541368..6f18dc34c5 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -58,10 +58,12 @@ func (c Config) MarshalTOML() (interface{}, error) { RPCGasCap uint64 RPCEVMTimeout time.Duration RPCTxFeeCap float64 - OverrideOsaka *uint64 `toml:",omitempty"` - OverrideBPO1 *uint64 `toml:",omitempty"` - OverrideBPO2 *uint64 `toml:",omitempty"` - OverrideVerkle *uint64 `toml:",omitempty"` + OverrideOsaka *uint64 `toml:",omitempty"` + OverrideBPO1 *uint64 `toml:",omitempty"` + OverrideBPO2 *uint64 `toml:",omitempty"` + OverrideVerkle *uint64 `toml:",omitempty"` + TxSyncDefaultTimeout time.Duration `toml:",omitempty"` + TxSyncMaxTimeout time.Duration `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -109,6 +111,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.OverrideBPO1 = c.OverrideBPO1 enc.OverrideBPO2 = c.OverrideBPO2 enc.OverrideVerkle = c.OverrideVerkle + enc.TxSyncDefaultTimeout = c.TxSyncDefaultTimeout + enc.TxSyncMaxTimeout = c.TxSyncMaxTimeout return &enc, nil } @@ -156,10 +160,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { RPCGasCap *uint64 RPCEVMTimeout *time.Duration RPCTxFeeCap *float64 - OverrideOsaka *uint64 `toml:",omitempty"` - OverrideBPO1 *uint64 `toml:",omitempty"` - OverrideBPO2 *uint64 `toml:",omitempty"` - OverrideVerkle *uint64 `toml:",omitempty"` + OverrideOsaka *uint64 `toml:",omitempty"` + OverrideBPO1 *uint64 `toml:",omitempty"` + OverrideBPO2 *uint64 `toml:",omitempty"` + OverrideVerkle *uint64 `toml:",omitempty"` + TxSyncDefaultTimeout *time.Duration `toml:",omitempty"` + TxSyncMaxTimeout *time.Duration `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -300,5 +306,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.OverrideVerkle != nil { c.OverrideVerkle = dec.OverrideVerkle } + if dec.TxSyncDefaultTimeout != nil { + c.TxSyncDefaultTimeout = *dec.TxSyncDefaultTimeout + } + if dec.TxSyncMaxTimeout != nil { + c.TxSyncMaxTimeout = *dec.TxSyncMaxTimeout + } return nil } diff --git a/eth/filters/api.go b/eth/filters/api.go index de70e5e310..b69442163f 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -144,6 +144,7 @@ func (api *FilterAPI) NewPendingTransactionFilter(fullTx *bool) rpc.ID { api.filtersMu.Unlock() go func() { + defer pendingTxSub.Unsubscribe() for { select { case pTx := <-pendingTxs: @@ -156,7 +157,6 @@ func (api *FilterAPI) NewPendingTransactionFilter(fullTx *bool) rpc.ID { api.filtersMu.Lock() delete(api.filters, pendingTxSub.ID) api.filtersMu.Unlock() - pendingTxSub.Unsubscribe() return } } @@ -219,6 +219,7 @@ func (api *FilterAPI) NewBlockFilter() rpc.ID { api.filtersMu.Unlock() go func() { + defer headerSub.Unsubscribe() for { select { case h := <-headers: @@ -405,6 +406,7 @@ func (api *FilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { api.filtersMu.Unlock() go func() { + defer logsSub.Unsubscribe() for { select { case l := <-logs: diff --git a/eth/handler.go b/eth/handler.go index 304560a158..ff970e2ba6 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -181,8 +181,7 @@ func newHandler(config *handlerConfig) (*handler, error) { } else { head := h.chain.CurrentBlock() if head.Number.Uint64() > 0 && h.chain.HasState(head.Root) { - // Print warning log if database is not empty to run snap sync. - log.Warn("Switch sync mode from snap sync to full sync", "reason", "snap sync complete") + log.Info("Switch sync mode from snap sync to full sync", "reason", "snap sync complete") } else { // If snap sync was requested and our database is empty, grant it h.snapSync.Store(true) diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 15ad048bcf..aad3353d88 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -494,12 +494,19 @@ func handleTransactions(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(&txs); err != nil { return err } + // Duplicate transactions are not allowed + seen := make(map[common.Hash]struct{}) for i, tx := range txs { // Validate and mark the remote transaction if tx == nil { return fmt.Errorf("Transactions: transaction %d is nil", i) } - peer.markTransaction(tx.Hash()) + hash := tx.Hash() + if _, exists := seen[hash]; exists { + return fmt.Errorf("Transactions: multiple copies of the same hash %v", hash) + } + seen[hash] = struct{}{} + peer.markTransaction(hash) } return backend.Handle(peer, &txs) } @@ -514,12 +521,19 @@ func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(&txs); err != nil { return err } + // Duplicate transactions are not allowed + seen := make(map[common.Hash]struct{}) for i, tx := range txs.PooledTransactionsResponse { // Validate and mark the remote transaction if tx == nil { return fmt.Errorf("PooledTransactions: transaction %d is nil", i) } - peer.markTransaction(tx.Hash()) + hash := tx.Hash() + if _, exists := seen[hash]; exists { + return fmt.Errorf("PooledTransactions: multiple copies of the same hash %v", hash) + } + seen[hash] = struct{}{} + peer.markTransaction(hash) } requestTracker.Fulfil(peer.id, peer.version, PooledTransactionsMsg, txs.RequestId) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 8b26f5b3ca..5008378da6 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -705,6 +706,39 @@ func (ec *Client) SendTransaction(ctx context.Context, tx *types.Transaction) er return ec.c.CallContext(ctx, nil, "eth_sendRawTransaction", hexutil.Encode(data)) } +// SendTransactionSync submits a signed tx and waits for a receipt (or until +// the optional timeout elapses on the server side). If timeout == 0, the server +// uses its default. +func (ec *Client) SendTransactionSync( + ctx context.Context, + tx *types.Transaction, + timeout *time.Duration, +) (*types.Receipt, error) { + raw, err := tx.MarshalBinary() + if err != nil { + return nil, err + } + return ec.SendRawTransactionSync(ctx, raw, timeout) +} + +func (ec *Client) SendRawTransactionSync( + ctx context.Context, + rawTx []byte, + timeout *time.Duration, +) (*types.Receipt, error) { + var ms *hexutil.Uint64 + if timeout != nil { + if d := hexutil.Uint64(timeout.Milliseconds()); d > 0 { + ms = &d + } + } + var receipt types.Receipt + if err := ec.c.CallContext(ctx, &receipt, "eth_sendRawTransactionSync", hexutil.Bytes(rawTx), ms); err != nil { + return nil, err + } + return &receipt, nil +} + // RevertErrorData returns the 'revert reason' data of a contract call. // // This can be used with CallContract and EstimateGas, and only when the server is Geth. @@ -828,3 +862,89 @@ func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress { StateIndexRemaining: uint64(p.StateIndexRemaining), } } + +// SimulateOptions represents the options for eth_simulateV1. +type SimulateOptions struct { + BlockStateCalls []SimulateBlock `json:"blockStateCalls"` + TraceTransfers bool `json:"traceTransfers"` + Validation bool `json:"validation"` + ReturnFullTransactions bool `json:"returnFullTransactions"` +} + +// SimulateBlock represents a batch of calls to be simulated. +type SimulateBlock struct { + BlockOverrides *ethereum.BlockOverrides `json:"blockOverrides,omitempty"` + StateOverrides map[common.Address]ethereum.OverrideAccount `json:"stateOverrides,omitempty"` + Calls []ethereum.CallMsg `json:"calls"` +} + +// MarshalJSON implements json.Marshaler for SimulateBlock. +func (s SimulateBlock) MarshalJSON() ([]byte, error) { + type Alias struct { + BlockOverrides *ethereum.BlockOverrides `json:"blockOverrides,omitempty"` + StateOverrides map[common.Address]ethereum.OverrideAccount `json:"stateOverrides,omitempty"` + Calls []interface{} `json:"calls"` + } + calls := make([]interface{}, len(s.Calls)) + for i, call := range s.Calls { + calls[i] = toCallArg(call) + } + return json.Marshal(Alias{ + BlockOverrides: s.BlockOverrides, + StateOverrides: s.StateOverrides, + Calls: calls, + }) +} + +//go:generate go run github.com/fjl/gencodec -type SimulateCallResult -field-override simulateCallResultMarshaling -out gen_simulate_call_result.go + +// SimulateCallResult is the result of a simulated call. +type SimulateCallResult struct { + ReturnValue []byte `json:"returnData"` + Logs []*types.Log `json:"logs"` + GasUsed uint64 `json:"gasUsed"` + Status uint64 `json:"status"` + Error *CallError `json:"error,omitempty"` +} + +type simulateCallResultMarshaling struct { + ReturnValue hexutil.Bytes + GasUsed hexutil.Uint64 + Status hexutil.Uint64 +} + +// CallError represents an error from a simulated call. +type CallError struct { + Code int `json:"code"` + Message string `json:"message"` + Data string `json:"data,omitempty"` +} + +//go:generate go run github.com/fjl/gencodec -type SimulateBlockResult -field-override simulateBlockResultMarshaling -out gen_simulate_block_result.go + +// SimulateBlockResult represents the result of a simulated block. +type SimulateBlockResult struct { + Number *big.Int `json:"number"` + Hash common.Hash `json:"hash"` + Timestamp uint64 `json:"timestamp"` + GasLimit uint64 `json:"gasLimit"` + GasUsed uint64 `json:"gasUsed"` + FeeRecipient common.Address `json:"miner"` + BaseFeePerGas *big.Int `json:"baseFeePerGas,omitempty"` + Calls []SimulateCallResult `json:"calls"` +} + +type simulateBlockResultMarshaling struct { + Number *hexutil.Big + Timestamp hexutil.Uint64 + GasLimit hexutil.Uint64 + GasUsed hexutil.Uint64 + BaseFeePerGas *hexutil.Big +} + +// SimulateV1 executes transactions on top of a base state. +func (ec *Client) SimulateV1(ctx context.Context, opts SimulateOptions, blockNrOrHash *rpc.BlockNumberOrHash) ([]SimulateBlockResult, error) { + var result []SimulateBlockResult + err := ec.c.CallContext(ctx, &result, "eth_simulateV1", opts, blockNrOrHash) + return result, err +} diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 815bc29de4..302ccf2e16 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -754,3 +754,250 @@ func ExampleRevertErrorData() { // revert: 08c379a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000a75736572206572726f72 // message: user error } + +func TestSimulateV1(t *testing.T) { + backend, _, err := newTestBackend(nil) + if err != nil { + t.Fatalf("Failed to create test backend: %v", err) + } + defer backend.Close() + + client := ethclient.NewClient(backend.Attach()) + defer client.Close() + + ctx := context.Background() + + // Get current base fee + header, err := client.HeaderByNumber(ctx, nil) + if err != nil { + t.Fatalf("Failed to get header: %v", err) + } + + // Simple test: transfer ETH from one account to another + from := testAddr + to := common.HexToAddress("0x0000000000000000000000000000000000000001") + value := big.NewInt(100) + gas := uint64(100000) + maxFeePerGas := new(big.Int).Mul(header.BaseFee, big.NewInt(2)) + + opts := ethclient.SimulateOptions{ + BlockStateCalls: []ethclient.SimulateBlock{ + { + Calls: []ethereum.CallMsg{ + { + From: from, + To: &to, + Value: value, + Gas: gas, + GasFeeCap: maxFeePerGas, + }, + }, + }, + }, + Validation: true, + } + + results, err := client.SimulateV1(ctx, opts, nil) + if err != nil { + t.Fatalf("SimulateV1 failed: %v", err) + } + + if len(results) != 1 { + t.Fatalf("expected 1 block result, got %d", len(results)) + } + + if len(results[0].Calls) != 1 { + t.Fatalf("expected 1 call result, got %d", len(results[0].Calls)) + } + + // Check that the transaction succeeded + if results[0].Calls[0].Status != 1 { + t.Errorf("expected status 1 (success), got %d", results[0].Calls[0].Status) + } + + if results[0].Calls[0].Error != nil { + t.Errorf("expected no error, got %v", results[0].Calls[0].Error) + } +} + +func TestSimulateV1WithBlockOverrides(t *testing.T) { + backend, _, err := newTestBackend(nil) + if err != nil { + t.Fatalf("Failed to create test backend: %v", err) + } + defer backend.Close() + + client := ethclient.NewClient(backend.Attach()) + defer client.Close() + + ctx := context.Background() + + // Get current base fee + header, err := client.HeaderByNumber(ctx, nil) + if err != nil { + t.Fatalf("Failed to get header: %v", err) + } + + from := testAddr + to := common.HexToAddress("0x0000000000000000000000000000000000000001") + value := big.NewInt(100) + gas := uint64(100000) + maxFeePerGas := new(big.Int).Mul(header.BaseFee, big.NewInt(2)) + + // Override timestamp only + timestamp := uint64(1234567890) + + opts := ethclient.SimulateOptions{ + BlockStateCalls: []ethclient.SimulateBlock{ + { + BlockOverrides: ðereum.BlockOverrides{ + Time: timestamp, + }, + Calls: []ethereum.CallMsg{ + { + From: from, + To: &to, + Value: value, + Gas: gas, + GasFeeCap: maxFeePerGas, + }, + }, + }, + }, + Validation: true, + } + + results, err := client.SimulateV1(ctx, opts, nil) + if err != nil { + t.Fatalf("SimulateV1 with block overrides failed: %v", err) + } + + if len(results) != 1 { + t.Fatalf("expected 1 block result, got %d", len(results)) + } + + // Verify the timestamp was overridden + if results[0].Timestamp != timestamp { + t.Errorf("expected timestamp %d, got %d", timestamp, results[0].Timestamp) + } +} + +func TestSimulateV1WithStateOverrides(t *testing.T) { + backend, _, err := newTestBackend(nil) + if err != nil { + t.Fatalf("Failed to create test backend: %v", err) + } + defer backend.Close() + + client := ethclient.NewClient(backend.Attach()) + defer client.Close() + + ctx := context.Background() + + // Get current base fee + header, err := client.HeaderByNumber(ctx, nil) + if err != nil { + t.Fatalf("Failed to get header: %v", err) + } + + from := testAddr + to := common.HexToAddress("0x0000000000000000000000000000000000000001") + value := big.NewInt(1000000000000000000) // 1 ETH + gas := uint64(100000) + maxFeePerGas := new(big.Int).Mul(header.BaseFee, big.NewInt(2)) + + // Override the balance of the 'from' address + balanceStr := "1000000000000000000000" + balance := new(big.Int) + balance.SetString(balanceStr, 10) + + stateOverrides := map[common.Address]ethereum.OverrideAccount{ + from: { + Balance: balance, + }, + } + + opts := ethclient.SimulateOptions{ + BlockStateCalls: []ethclient.SimulateBlock{ + { + StateOverrides: stateOverrides, + Calls: []ethereum.CallMsg{ + { + From: from, + To: &to, + Value: value, + Gas: gas, + GasFeeCap: maxFeePerGas, + }, + }, + }, + }, + Validation: true, + } + + results, err := client.SimulateV1(ctx, opts, nil) + if err != nil { + t.Fatalf("SimulateV1 with state overrides failed: %v", err) + } + + if len(results) != 1 { + t.Fatalf("expected 1 block result, got %d", len(results)) + } + + if results[0].Calls[0].Status != 1 { + t.Errorf("expected status 1 (success), got %d", results[0].Calls[0].Status) + } +} + +func TestSimulateV1WithBlockNumberOrHash(t *testing.T) { + backend, _, err := newTestBackend(nil) + if err != nil { + t.Fatalf("Failed to create test backend: %v", err) + } + defer backend.Close() + + client := ethclient.NewClient(backend.Attach()) + defer client.Close() + + ctx := context.Background() + + // Get current base fee + header, err := client.HeaderByNumber(ctx, nil) + if err != nil { + t.Fatalf("Failed to get header: %v", err) + } + + from := testAddr + to := common.HexToAddress("0x0000000000000000000000000000000000000001") + value := big.NewInt(100) + gas := uint64(100000) + maxFeePerGas := new(big.Int).Mul(header.BaseFee, big.NewInt(2)) + + opts := ethclient.SimulateOptions{ + BlockStateCalls: []ethclient.SimulateBlock{ + { + Calls: []ethereum.CallMsg{ + { + From: from, + To: &to, + Value: value, + Gas: gas, + GasFeeCap: maxFeePerGas, + }, + }, + }, + }, + Validation: true, + } + + // Simulate on the latest block + latest := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + results, err := client.SimulateV1(ctx, opts, &latest) + if err != nil { + t.Fatalf("SimulateV1 with latest block failed: %v", err) + } + + if len(results) != 1 { + t.Fatalf("expected 1 block result, got %d", len(results)) + } +} diff --git a/ethclient/gen_simulate_block_result.go b/ethclient/gen_simulate_block_result.go new file mode 100644 index 0000000000..b8cd6ebf2f --- /dev/null +++ b/ethclient/gen_simulate_block_result.go @@ -0,0 +1,80 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package ethclient + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*simulateBlockResultMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (s SimulateBlockResult) MarshalJSON() ([]byte, error) { + type SimulateBlockResult struct { + Number *hexutil.Big `json:"number"` + Hash common.Hash `json:"hash"` + Timestamp hexutil.Uint64 `json:"timestamp"` + GasLimit hexutil.Uint64 `json:"gasLimit"` + GasUsed hexutil.Uint64 `json:"gasUsed"` + FeeRecipient common.Address `json:"miner"` + BaseFeePerGas *hexutil.Big `json:"baseFeePerGas,omitempty"` + Calls []SimulateCallResult `json:"calls"` + } + var enc SimulateBlockResult + enc.Number = (*hexutil.Big)(s.Number) + enc.Hash = s.Hash + enc.Timestamp = hexutil.Uint64(s.Timestamp) + enc.GasLimit = hexutil.Uint64(s.GasLimit) + enc.GasUsed = hexutil.Uint64(s.GasUsed) + enc.FeeRecipient = s.FeeRecipient + enc.BaseFeePerGas = (*hexutil.Big)(s.BaseFeePerGas) + enc.Calls = s.Calls + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (s *SimulateBlockResult) UnmarshalJSON(input []byte) error { + type SimulateBlockResult struct { + Number *hexutil.Big `json:"number"` + Hash *common.Hash `json:"hash"` + Timestamp *hexutil.Uint64 `json:"timestamp"` + GasLimit *hexutil.Uint64 `json:"gasLimit"` + GasUsed *hexutil.Uint64 `json:"gasUsed"` + FeeRecipient *common.Address `json:"miner"` + BaseFeePerGas *hexutil.Big `json:"baseFeePerGas,omitempty"` + Calls []SimulateCallResult `json:"calls"` + } + var dec SimulateBlockResult + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Number != nil { + s.Number = (*big.Int)(dec.Number) + } + if dec.Hash != nil { + s.Hash = *dec.Hash + } + if dec.Timestamp != nil { + s.Timestamp = uint64(*dec.Timestamp) + } + if dec.GasLimit != nil { + s.GasLimit = uint64(*dec.GasLimit) + } + if dec.GasUsed != nil { + s.GasUsed = uint64(*dec.GasUsed) + } + if dec.FeeRecipient != nil { + s.FeeRecipient = *dec.FeeRecipient + } + if dec.BaseFeePerGas != nil { + s.BaseFeePerGas = (*big.Int)(dec.BaseFeePerGas) + } + if dec.Calls != nil { + s.Calls = dec.Calls + } + return nil +} diff --git a/ethclient/gen_simulate_call_result.go b/ethclient/gen_simulate_call_result.go new file mode 100644 index 0000000000..55e14cd697 --- /dev/null +++ b/ethclient/gen_simulate_call_result.go @@ -0,0 +1,61 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package ethclient + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" +) + +var _ = (*simulateCallResultMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (s SimulateCallResult) MarshalJSON() ([]byte, error) { + type SimulateCallResult struct { + ReturnValue hexutil.Bytes `json:"returnData"` + Logs []*types.Log `json:"logs"` + GasUsed hexutil.Uint64 `json:"gasUsed"` + Status hexutil.Uint64 `json:"status"` + Error *CallError `json:"error,omitempty"` + } + var enc SimulateCallResult + enc.ReturnValue = s.ReturnValue + enc.Logs = s.Logs + enc.GasUsed = hexutil.Uint64(s.GasUsed) + enc.Status = hexutil.Uint64(s.Status) + enc.Error = s.Error + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (s *SimulateCallResult) UnmarshalJSON(input []byte) error { + type SimulateCallResult struct { + ReturnValue *hexutil.Bytes `json:"returnData"` + Logs []*types.Log `json:"logs"` + GasUsed *hexutil.Uint64 `json:"gasUsed"` + Status *hexutil.Uint64 `json:"status"` + Error *CallError `json:"error,omitempty"` + } + var dec SimulateCallResult + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.ReturnValue != nil { + s.ReturnValue = *dec.ReturnValue + } + if dec.Logs != nil { + s.Logs = dec.Logs + } + if dec.GasUsed != nil { + s.GasUsed = uint64(*dec.GasUsed) + } + if dec.Status != nil { + s.Status = uint64(*dec.Status) + } + if dec.Error != nil { + s.Error = dec.Error + } + return nil +} diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 54997cbf51..6a0f5eb312 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -19,7 +19,6 @@ package gethclient import ( "context" - "encoding/json" "fmt" "math/big" "runtime" @@ -280,97 +279,8 @@ func toCallArg(msg ethereum.CallMsg) interface{} { return arg } -// OverrideAccount specifies the state of an account to be overridden. -type OverrideAccount struct { - // Nonce sets nonce of the account. Note: the nonce override will only - // be applied when it is set to a non-zero value. - Nonce uint64 +// OverrideAccount is an alias for ethereum.OverrideAccount. +type OverrideAccount = ethereum.OverrideAccount - // Code sets the contract code. The override will be applied - // when the code is non-nil, i.e. setting empty code is possible - // using an empty slice. - Code []byte - - // Balance sets the account balance. - Balance *big.Int - - // State sets the complete storage. The override will be applied - // when the given map is non-nil. Using an empty map wipes the - // entire contract storage during the call. - State map[common.Hash]common.Hash - - // StateDiff allows overriding individual storage slots. - StateDiff map[common.Hash]common.Hash -} - -func (a OverrideAccount) MarshalJSON() ([]byte, error) { - type acc struct { - Nonce hexutil.Uint64 `json:"nonce,omitempty"` - Code string `json:"code,omitempty"` - Balance *hexutil.Big `json:"balance,omitempty"` - State interface{} `json:"state,omitempty"` - StateDiff map[common.Hash]common.Hash `json:"stateDiff,omitempty"` - } - - output := acc{ - Nonce: hexutil.Uint64(a.Nonce), - Balance: (*hexutil.Big)(a.Balance), - StateDiff: a.StateDiff, - } - if a.Code != nil { - output.Code = hexutil.Encode(a.Code) - } - if a.State != nil { - output.State = a.State - } - return json.Marshal(output) -} - -// BlockOverrides specifies the set of header fields to override. -type BlockOverrides struct { - // Number overrides the block number. - Number *big.Int - // Difficulty overrides the block difficulty. - Difficulty *big.Int - // Time overrides the block timestamp. Time is applied only when - // it is non-zero. - Time uint64 - // GasLimit overrides the block gas limit. GasLimit is applied only when - // it is non-zero. - GasLimit uint64 - // Coinbase overrides the block coinbase. Coinbase is applied only when - // it is different from the zero address. - Coinbase common.Address - // Random overrides the block extra data which feeds into the RANDOM opcode. - // Random is applied only when it is a non-zero hash. - Random common.Hash - // BaseFee overrides the block base fee. - BaseFee *big.Int -} - -func (o BlockOverrides) MarshalJSON() ([]byte, error) { - type override struct { - Number *hexutil.Big `json:"number,omitempty"` - Difficulty *hexutil.Big `json:"difficulty,omitempty"` - Time hexutil.Uint64 `json:"time,omitempty"` - GasLimit hexutil.Uint64 `json:"gasLimit,omitempty"` - Coinbase *common.Address `json:"feeRecipient,omitempty"` - Random *common.Hash `json:"prevRandao,omitempty"` - BaseFee *hexutil.Big `json:"baseFeePerGas,omitempty"` - } - - output := override{ - Number: (*hexutil.Big)(o.Number), - Difficulty: (*hexutil.Big)(o.Difficulty), - Time: hexutil.Uint64(o.Time), - GasLimit: hexutil.Uint64(o.GasLimit), - BaseFee: (*hexutil.Big)(o.BaseFee), - } - if o.Coinbase != (common.Address{}) { - output.Coinbase = &o.Coinbase - } - if o.Random != (common.Hash{}) { - output.Random = &o.Random - } - return json.Marshal(output) -} +// BlockOverrides is an alias for ethereum.BlockOverrides. +type BlockOverrides = ethereum.BlockOverrides diff --git a/ethdb/database.go b/ethdb/database.go index e665a84a61..534fcad4fc 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -121,6 +121,10 @@ type AncientReaderOp interface { // - if maxBytes is not specified, 'count' items will be returned if they are present AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) + // AncientBytes retrieves the value segment of the element specified by the id + // and value offsets. + AncientBytes(kind string, id, offset, length uint64) ([]byte, error) + // Ancients returns the ancient item numbers in the ancient store. Ancients() (uint64, error) diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index 7fe154ea95..0d0d854fe4 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -140,6 +140,10 @@ func (db *Database) Close() error { return nil } +func (db *Database) AncientBytes(kind string, id, offset, length uint64) ([]byte, error) { + panic("not supported") +} + func New(client *rpc.Client) ethdb.Database { if client == nil { return nil diff --git a/interfaces.go b/interfaces.go index 2828af1cc9..21d42c6d34 100644 --- a/interfaces.go +++ b/interfaces.go @@ -19,10 +19,12 @@ package ethereum import ( "context" + "encoding/json" "errors" "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" ) @@ -293,3 +295,98 @@ type BlockNumberReader interface { type ChainIDReader interface { ChainID(ctx context.Context) (*big.Int, error) } + +// OverrideAccount specifies the state of an account to be overridden. +type OverrideAccount struct { + // Nonce sets nonce of the account. Note: the nonce override will only + // be applied when it is set to a non-zero value. + Nonce uint64 + + // Code sets the contract code. The override will be applied + // when the code is non-nil, i.e. setting empty code is possible + // using an empty slice. + Code []byte + + // Balance sets the account balance. + Balance *big.Int + + // State sets the complete storage. The override will be applied + // when the given map is non-nil. Using an empty map wipes the + // entire contract storage during the call. + State map[common.Hash]common.Hash + + // StateDiff allows overriding individual storage slots. + StateDiff map[common.Hash]common.Hash +} + +func (a OverrideAccount) MarshalJSON() ([]byte, error) { + type acc struct { + Nonce hexutil.Uint64 `json:"nonce,omitempty"` + Code string `json:"code,omitempty"` + Balance *hexutil.Big `json:"balance,omitempty"` + State interface{} `json:"state,omitempty"` + StateDiff map[common.Hash]common.Hash `json:"stateDiff,omitempty"` + } + + output := acc{ + Nonce: hexutil.Uint64(a.Nonce), + Balance: (*hexutil.Big)(a.Balance), + StateDiff: a.StateDiff, + } + if a.Code != nil { + output.Code = hexutil.Encode(a.Code) + } + if a.State != nil { + output.State = a.State + } + return json.Marshal(output) +} + +// BlockOverrides specifies the set of header fields to override. +type BlockOverrides struct { + // Number overrides the block number. + Number *big.Int + // Difficulty overrides the block difficulty. + Difficulty *big.Int + // Time overrides the block timestamp. Time is applied only when + // it is non-zero. + Time uint64 + // GasLimit overrides the block gas limit. GasLimit is applied only when + // it is non-zero. + GasLimit uint64 + // Coinbase overrides the block coinbase. Coinbase is applied only when + // it is different from the zero address. + Coinbase common.Address + // Random overrides the block extra data which feeds into the RANDOM opcode. + // Random is applied only when it is a non-zero hash. + Random common.Hash + // BaseFee overrides the block base fee. + BaseFee *big.Int +} + +func (o BlockOverrides) MarshalJSON() ([]byte, error) { + type override struct { + Number *hexutil.Big `json:"number,omitempty"` + Difficulty *hexutil.Big `json:"difficulty,omitempty"` + Time hexutil.Uint64 `json:"time,omitempty"` + GasLimit hexutil.Uint64 `json:"gasLimit,omitempty"` + Coinbase *common.Address `json:"feeRecipient,omitempty"` + Random *common.Hash `json:"prevRandao,omitempty"` + BaseFee *hexutil.Big `json:"baseFeePerGas,omitempty"` + } + + output := override{ + Number: (*hexutil.Big)(o.Number), + Difficulty: (*hexutil.Big)(o.Difficulty), + Time: hexutil.Uint64(o.Time), + GasLimit: hexutil.Uint64(o.GasLimit), + BaseFee: (*hexutil.Big)(o.BaseFee), + } + if o.Coinbase != (common.Address{}) { + output.Coinbase = &o.Coinbase + } + if o.Random != (common.Hash{}) { + output.Random = &o.Random + } + return json.Marshal(output) +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index a2cb28d3b2..d7cf47468c 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -55,6 +55,7 @@ import ( const estimateGasErrorRatio = 0.015 var errBlobTxNotSupported = errors.New("signing blob transactions not supported") +var errSubClosed = errors.New("chain subscription closed") // EthereumAPI provides an API to access Ethereum related information. type EthereumAPI struct { @@ -1619,16 +1620,9 @@ func (api *TransactionAPI) SendTransaction(ctx context.Context, args Transaction // processing (signing + broadcast). func (api *TransactionAPI) FillTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { // Set some sanity defaults and terminate on failure - sidecarVersion := types.BlobSidecarVersion0 - if len(args.Blobs) > 0 { - h := api.b.CurrentHeader() - if api.b.ChainConfig().IsOsaka(h.Number, h.Time) { - sidecarVersion = types.BlobSidecarVersion1 - } - } config := sidecarConfig{ blobSidecarAllowed: true, - blobSidecarVersion: sidecarVersion, + blobSidecarVersion: api.currentBlobSidecarVersion(), } if err := args.setDefaults(ctx, api.b, config); err != nil { return nil, err @@ -1642,6 +1636,14 @@ func (api *TransactionAPI) FillTransaction(ctx context.Context, args Transaction return &SignTransactionResult{data, tx}, nil } +func (api *TransactionAPI) currentBlobSidecarVersion() byte { + h := api.b.CurrentHeader() + if api.b.ChainConfig().IsOsaka(h.Number, h.Time) { + return types.BlobSidecarVersion1 + } + return types.BlobSidecarVersion0 +} + // SendRawTransaction will add the signed transaction to the transaction pool. // The sender is responsible for signing the transaction and using the correct nonce. func (api *TransactionAPI) SendRawTransaction(ctx context.Context, input hexutil.Bytes) (common.Hash, error) { @@ -1649,9 +1651,119 @@ func (api *TransactionAPI) SendRawTransaction(ctx context.Context, input hexutil if err := tx.UnmarshalBinary(input); err != nil { return common.Hash{}, err } + + // Convert legacy blob transaction proofs. + // TODO: remove in go-ethereum v1.17.x + if sc := tx.BlobTxSidecar(); sc != nil { + exp := api.currentBlobSidecarVersion() + if sc.Version == types.BlobSidecarVersion0 && exp == types.BlobSidecarVersion1 { + if err := sc.ToV1(); err != nil { + return common.Hash{}, fmt.Errorf("blob sidecar conversion failed: %v", err) + } + tx = tx.WithBlobTxSidecar(sc) + } + } + return SubmitTransaction(ctx, api.b, tx) } +// SendRawTransactionSync will add the signed transaction to the transaction pool +// and wait until the transaction has been included in a block and return the receipt, or the timeout. +func (api *TransactionAPI) SendRawTransactionSync(ctx context.Context, input hexutil.Bytes, timeoutMs *hexutil.Uint64) (map[string]interface{}, error) { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(input); err != nil { + return nil, err + } + + // Convert legacy blob transaction proofs. + // TODO: remove in go-ethereum v1.17.x + if sc := tx.BlobTxSidecar(); sc != nil { + exp := api.currentBlobSidecarVersion() + if sc.Version == types.BlobSidecarVersion0 && exp == types.BlobSidecarVersion1 { + if err := sc.ToV1(); err != nil { + return nil, fmt.Errorf("blob sidecar conversion failed: %v", err) + } + tx = tx.WithBlobTxSidecar(sc) + } + } + + ch := make(chan core.ChainEvent, 128) + sub := api.b.SubscribeChainEvent(ch) + defer sub.Unsubscribe() + + hash, err := SubmitTransaction(ctx, api.b, tx) + if err != nil { + return nil, err + } + + var ( + maxTimeout = api.b.RPCTxSyncMaxTimeout() + defaultTimeout = api.b.RPCTxSyncDefaultTimeout() + timeout = defaultTimeout + ) + if timeoutMs != nil && *timeoutMs > 0 { + req := time.Duration(*timeoutMs) * time.Millisecond + if req > maxTimeout { + timeout = maxTimeout + } else { + timeout = req + } + } + receiptCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // Fast path. + if r, err := api.GetTransactionReceipt(receiptCtx, hash); err == nil && r != nil { + return r, nil + } + + // Monitor the receipts + for { + select { + case <-receiptCtx.Done(): + // If server-side wait window elapsed, return the structured timeout. + if errors.Is(receiptCtx.Err(), context.DeadlineExceeded) { + return nil, &txSyncTimeoutError{ + msg: fmt.Sprintf("The transaction was added to the transaction pool but wasn't processed in %v", timeout), + hash: hash, + } + } + return nil, receiptCtx.Err() + + case err, ok := <-sub.Err(): + if !ok { + return nil, errSubClosed + } + return nil, err + + case ev, ok := <-ch: + if !ok { + return nil, errSubClosed + } + rs, txs := ev.Receipts, ev.Transactions + if len(rs) == 0 || len(rs) != len(txs) { + continue + } + for i := range rs { + if rs[i].TxHash == hash { + if rs[i].BlockNumber != nil && rs[i].BlockHash != (common.Hash{}) { + signer := types.LatestSigner(api.b.ChainConfig()) + return MarshalReceipt( + rs[i], + rs[i].BlockHash, + rs[i].BlockNumber.Uint64(), + signer, + txs[i], + int(rs[i].TransactionIndex), + ), nil + } + return api.GetTransactionReceipt(receiptCtx, hash) + } + } + } + } +} + // Sign calculates an ECDSA signature for: // keccak256("\x19Ethereum Signed Message:\n" + len(message) + message). // diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index d3278c04e7..aaa002b5ec 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -440,6 +440,19 @@ type testBackend struct { pending *types.Block pendingReceipts types.Receipts + + chainFeed *event.Feed + autoMine bool + + sentTx *types.Transaction + sentTxHash common.Hash + + syncDefaultTimeout time.Duration + syncMaxTimeout time.Duration +} + +func fakeBlockHash(txh common.Hash) common.Hash { + return crypto.Keccak256Hash([]byte("testblock"), txh.Bytes()) } func newTestBackend(t *testing.T, n int, gspec *core.Genesis, engine consensus.Engine, generator func(i int, b *core.BlockGen)) *testBackend { @@ -466,6 +479,7 @@ func newTestBackend(t *testing.T, n int, gspec *core.Genesis, engine consensus.E acc: acc, pending: blocks[n], pendingReceipts: receipts[n], + chainFeed: new(event.Feed), } return backend } @@ -587,19 +601,64 @@ func (b testBackend) GetEVM(ctx context.Context, state *state.StateDB, header *t return vm.NewEVM(context, state, b.chain.Config(), *vmConfig) } func (b testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { - panic("implement me") + return b.chainFeed.Subscribe(ch) } func (b testBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { panic("implement me") } -func (b testBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { - panic("implement me") +func (b *testBackend) SendTx(ctx context.Context, tx *types.Transaction) error { + b.sentTx = tx + b.sentTxHash = tx.Hash() + + if b.autoMine { + // Synthesize a "mined" receipt at head+1 + num := b.chain.CurrentHeader().Number.Uint64() + 1 + receipt := &types.Receipt{ + TxHash: tx.Hash(), + Status: types.ReceiptStatusSuccessful, + BlockHash: fakeBlockHash(tx.Hash()), + BlockNumber: new(big.Int).SetUint64(num), + TransactionIndex: 0, + CumulativeGasUsed: 21000, + GasUsed: 21000, + } + // Broadcast a ChainEvent that includes the receipts and txs + b.chainFeed.Send(core.ChainEvent{ + Header: &types.Header{ + Number: new(big.Int).SetUint64(num), + }, + Receipts: types.Receipts{receipt}, + Transactions: types.Transactions{tx}, + }) + } + return nil } -func (b testBackend) GetCanonicalTransaction(txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64) { +func (b *testBackend) GetCanonicalTransaction(txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64) { + // Treat the auto-mined tx as canonically placed at head+1. + if b.autoMine && txHash == b.sentTxHash { + num := b.chain.CurrentHeader().Number.Uint64() + 1 + return true, b.sentTx, fakeBlockHash(txHash), num, 0 + } tx, blockHash, blockNumber, index := rawdb.ReadCanonicalTransaction(b.db, txHash) return tx != nil, tx, blockHash, blockNumber, index } -func (b testBackend) GetCanonicalReceipt(tx *types.Transaction, blockHash common.Hash, blockNumber, blockIndex uint64) (*types.Receipt, error) { +func (b *testBackend) GetCanonicalReceipt(tx *types.Transaction, blockHash common.Hash, blockNumber, blockIndex uint64) (*types.Receipt, error) { + if b.autoMine && tx != nil && tx.Hash() == b.sentTxHash && + blockHash == fakeBlockHash(tx.Hash()) && + blockIndex == 0 && + blockNumber == b.chain.CurrentHeader().Number.Uint64()+1 { + return &types.Receipt{ + Type: tx.Type(), + Status: types.ReceiptStatusSuccessful, + CumulativeGasUsed: 21000, + GasUsed: 21000, + EffectiveGasPrice: big.NewInt(1), + BlockHash: blockHash, + BlockNumber: new(big.Int).SetUint64(blockNumber), + TransactionIndex: 0, + TxHash: tx.Hash(), + }, nil + } return b.chain.GetCanonicalReceipt(tx, blockHash, blockNumber, blockIndex) } func (b testBackend) TxIndexDone() bool { @@ -3889,3 +3948,109 @@ func (b configTimeBackend) HeaderByNumber(_ context.Context, n rpc.BlockNumber) func (b configTimeBackend) CurrentHeader() *types.Header { return &types.Header{Time: b.time} } + +func (b *testBackend) RPCTxSyncDefaultTimeout() time.Duration { + if b.syncDefaultTimeout != 0 { + return b.syncDefaultTimeout + } + return 2 * time.Second +} +func (b *testBackend) RPCTxSyncMaxTimeout() time.Duration { + if b.syncMaxTimeout != 0 { + return b.syncMaxTimeout + } + return 5 * time.Minute +} +func (b *backendMock) RPCTxSyncDefaultTimeout() time.Duration { return 2 * time.Second } +func (b *backendMock) RPCTxSyncMaxTimeout() time.Duration { return 5 * time.Minute } + +func makeSignedRaw(t *testing.T, api *TransactionAPI, from, to common.Address, value *big.Int) (hexutil.Bytes, *types.Transaction) { + t.Helper() + + fillRes, err := api.FillTransaction(context.Background(), TransactionArgs{ + From: &from, + To: &to, + Value: (*hexutil.Big)(value), + }) + if err != nil { + t.Fatalf("FillTransaction failed: %v", err) + } + signRes, err := api.SignTransaction(context.Background(), argsFromTransaction(fillRes.Tx, from)) + if err != nil { + t.Fatalf("SignTransaction failed: %v", err) + } + return signRes.Raw, signRes.Tx +} + +// makeSelfSignedRaw is a convenience for a 0-ETH self-transfer. +func makeSelfSignedRaw(t *testing.T, api *TransactionAPI, addr common.Address) (hexutil.Bytes, *types.Transaction) { + return makeSignedRaw(t, api, addr, addr, big.NewInt(0)) +} + +func TestSendRawTransactionSync_Success(t *testing.T) { + t.Parallel() + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{}, + } + b := newTestBackend(t, 0, genesis, ethash.NewFaker(), nil) + b.autoMine = true // immediately “mines” the tx in-memory + + api := NewTransactionAPI(b, new(AddrLocker)) + + raw, _ := makeSelfSignedRaw(t, api, b.acc.Address) + + receipt, err := api.SendRawTransactionSync(context.Background(), raw, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if receipt == nil { + t.Fatalf("expected non-nil receipt") + } + if _, ok := receipt["blockNumber"]; !ok { + t.Fatalf("expected blockNumber in receipt, got %#v", receipt) + } +} + +func TestSendRawTransactionSync_Timeout(t *testing.T) { + t.Parallel() + + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{}, + } + b := newTestBackend(t, 0, genesis, ethash.NewFaker(), nil) + b.autoMine = false // don't mine, should time out + + api := NewTransactionAPI(b, new(AddrLocker)) + + raw, _ := makeSelfSignedRaw(t, api, b.acc.Address) + + timeout := hexutil.Uint64(200) // 200ms + receipt, err := api.SendRawTransactionSync(context.Background(), raw, &timeout) + + if receipt != nil { + t.Fatalf("expected nil receipt, got %#v", receipt) + } + if err == nil { + t.Fatalf("expected timeout error, got nil") + } + // assert error shape & data (hash) + var de interface { + ErrorCode() int + ErrorData() interface{} + } + if !errors.As(err, &de) { + t.Fatalf("expected data error with code/data, got %T %v", err, err) + } + if de.ErrorCode() != errCodeTxSyncTimeout { + t.Fatalf("expected code %d, got %d", errCodeTxSyncTimeout, de.ErrorCode()) + } + tx := new(types.Transaction) + if e := tx.UnmarshalBinary(raw); e != nil { + t.Fatal(e) + } + if got, want := de.ErrorData(), tx.Hash().Hex(); got != want { + t.Fatalf("expected ErrorData=%s, got %v", want, got) + } +} diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index f709a1fcdc..af3d592b82 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -53,6 +53,8 @@ type Backend interface { RPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection RPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs UnprotectedAllowed() bool // allows only for EIP155 transactions. + RPCTxSyncDefaultTimeout() time.Duration + RPCTxSyncMaxTimeout() time.Duration // Blockchain API SetHead(number uint64) diff --git a/internal/ethapi/errors.go b/internal/ethapi/errors.go index 154938fa0e..30711a0167 100644 --- a/internal/ethapi/errors.go +++ b/internal/ethapi/errors.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/vm" @@ -33,6 +34,11 @@ type revertError struct { reason string // revert reason hex encoded } +type txSyncTimeoutError struct { + msg string + hash common.Hash +} + // ErrorCode returns the JSON error code for a revert. // See: https://ethereum.org/en/developers/docs/apis/json-rpc/#error-codes func (e *revertError) ErrorCode() int { @@ -108,6 +114,7 @@ const ( errCodeInvalidParams = -32602 errCodeReverted = -32000 errCodeVMError = -32015 + errCodeTxSyncTimeout = 4 ) func txValidationError(err error) *invalidTxError { @@ -168,3 +175,7 @@ type blockGasLimitReachedError struct{ message string } func (e *blockGasLimitReachedError) Error() string { return e.message } func (e *blockGasLimitReachedError) ErrorCode() int { return errCodeBlockGasLimitReached } + +func (e *txSyncTimeoutError) Error() string { return e.msg } +func (e *txSyncTimeoutError) ErrorCode() int { return errCodeTxSyncTimeout } +func (e *txSyncTimeoutError) ErrorData() interface{} { return e.hash.Hex() } diff --git a/node/rpcstack.go b/node/rpcstack.go index 655f7db9e4..a1cc832f9f 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -500,7 +500,7 @@ func (w *gzipResponseWriter) init() { hdr := w.resp.Header() length := hdr.Get("content-length") if len(length) > 0 { - if n, err := strconv.ParseUint(length, 10, 64); err != nil { + if n, err := strconv.ParseUint(length, 10, 64); err == nil { w.hasLength = true w.contentLength = n } diff --git a/p2p/discover/lookup.go b/p2p/discover/lookup.go index 9cca0118ac..416256fb36 100644 --- a/p2p/discover/lookup.go +++ b/p2p/discover/lookup.go @@ -153,6 +153,7 @@ type lookupIterator struct { cancel func() lookup *lookup tabRefreshing <-chan struct{} + lastLookup time.Time } type lookupFunc func(ctx context.Context) *lookup @@ -185,6 +186,9 @@ func (it *lookupIterator) Next() bool { return false } if it.lookup == nil { + // Ensure enough time has passed between lookup creations. + it.slowdown() + it.lookup = it.nextLookup(it.ctx) if it.lookup.empty() { // If the lookup is empty right after creation, it means the local table @@ -235,6 +239,25 @@ func (it *lookupIterator) lookupFailed(tab *Table, timeout time.Duration) { tab.waitForNodes(tout, 1) } +// slowdown applies a delay between creating lookups. This exists to prevent hot-spinning +// in some test environments where lookups don't yield any results. +func (it *lookupIterator) slowdown() { + const minInterval = 1 * time.Second + + now := time.Now() + diff := now.Sub(it.lastLookup) + it.lastLookup = now + if diff > minInterval { + return + } + wait := time.NewTimer(diff) + defer wait.Stop() + select { + case <-wait.C: + case <-it.ctx.Done(): + } +} + // Close ends the iterator. func (it *lookupIterator) Close() { it.cancel() diff --git a/p2p/discover/table.go b/p2p/discover/table.go index 6a1c7494ee..e5b2c7c8c5 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -54,9 +54,8 @@ const ( bucketIPLimit, bucketSubnet = 2, 24 // at most 2 addresses from the same /24 tableIPLimit, tableSubnet = 10, 24 - seedMinTableTime = 5 * time.Minute - seedCount = 30 - seedMaxAge = 5 * 24 * time.Hour + seedCount = 30 + seedMaxAge = 5 * 24 * time.Hour ) // Table is the 'node table', a Kademlia-like index of neighbor nodes. The table keeps diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index 44863183fa..287f0c34fa 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -575,6 +575,13 @@ func startLocalhostV4(t *testing.T, cfg Config) *UDPv4 { if err != nil { t.Fatal(err) } + + // Wait for bootstrap to complete. + select { + case <-udp.tab.initDone: + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for table initialization") + } return udp } diff --git a/p2p/enode/node.go b/p2p/enode/node.go index d6f2ac7ff5..dafde51d6a 100644 --- a/p2p/enode/node.go +++ b/p2p/enode/node.go @@ -19,6 +19,7 @@ package enode import ( "crypto/ecdsa" "encoding/base64" + "encoding/binary" "encoding/hex" "errors" "fmt" @@ -358,9 +359,10 @@ func ParseID(in string) (ID, error) { // Returns -1 if a is closer to target, 1 if b is closer to target // and 0 if they are equal. func DistCmp(target, a, b ID) int { - for i := range target { - da := a[i] ^ target[i] - db := b[i] ^ target[i] + for i := 0; i < len(target); i += 8 { + tn := binary.BigEndian.Uint64(target[i : i+8]) + da := tn ^ binary.BigEndian.Uint64(a[i:i+8]) + db := tn ^ binary.BigEndian.Uint64(b[i:i+8]) if da > db { return 1 } else if da < db { @@ -373,12 +375,14 @@ func DistCmp(target, a, b ID) int { // LogDist returns the logarithmic distance between a and b, log2(a ^ b). func LogDist(a, b ID) int { lz := 0 - for i := range a { - x := a[i] ^ b[i] + for i := 0; i < len(a); i += 8 { + ai := binary.BigEndian.Uint64(a[i : i+8]) + bi := binary.BigEndian.Uint64(b[i : i+8]) + x := ai ^ bi if x == 0 { - lz += 8 + lz += 64 } else { - lz += bits.LeadingZeros8(x) + lz += bits.LeadingZeros64(x) break } } diff --git a/p2p/enode/node_test.go b/p2p/enode/node_test.go index e9fe631f34..51bc4ebe15 100644 --- a/p2p/enode/node_test.go +++ b/p2p/enode/node_test.go @@ -368,6 +368,16 @@ func TestID_distcmpEqual(t *testing.T) { } } +func BenchmarkDistCmp(b *testing.B) { + base := ID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + aID := ID{15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0} + bID := ID{15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 1} + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = DistCmp(base, aID, bID) + } +} + func TestID_logdist(t *testing.T) { logdistBig := func(a, b ID) int { abig, bbig := new(big.Int).SetBytes(a[:]), new(big.Int).SetBytes(b[:]) @@ -378,6 +388,28 @@ func TestID_logdist(t *testing.T) { } } +func makeIDs() (ID, ID) { + var a, b ID + size := len(a) + // last byte differs + for i := 0; i < size-1; i++ { + a[i] = 0xAA + b[i] = 0xAA + } + a[size-1] = 0xAA + b[size-1] = 0xAB + return a, b +} + +// Benchmark LogDist +func BenchmarkLogDist(b *testing.B) { + aID, bID := makeIDs() // 256-bit ID + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = LogDist(aID, bID) + } +} + // The random tests is likely to miss the case where a and b are equal, // this test checks it explicitly. func TestID_logdistEqual(t *testing.T) { diff --git a/rpc/server_test.go b/rpc/server_test.go index a38a64b080..8334d4e80d 100644 --- a/rpc/server_test.go +++ b/rpc/server_test.go @@ -273,7 +273,8 @@ func TestServerWebsocketReadLimit(t *testing.T) { } } else if !errors.Is(err, websocket.ErrReadLimit) && !strings.Contains(strings.ToLower(err.Error()), "1009") && - !strings.Contains(strings.ToLower(err.Error()), "message too big") { + !strings.Contains(strings.ToLower(err.Error()), "message too big") && + !strings.Contains(strings.ToLower(err.Error()), "connection reset by peer") { // Not the error we expect from exceeding the message size limit. t.Fatalf("unexpected error for read limit violation: %v", err) } diff --git a/trie/transition.go b/trie/transition.go index da49c6cdc2..c6eecd3937 100644 --- a/trie/transition.go +++ b/trie/transition.go @@ -211,7 +211,8 @@ func (t *TransitionTrie) UpdateStem(key []byte, values [][]byte) error { func (t *TransitionTrie) Copy() *TransitionTrie { return &TransitionTrie{ overlay: t.overlay.Copy(), - base: t.base.Copy(), + // base in immutable, so there is no need to copy it + base: t.base, storage: t.storage, } } diff --git a/triedb/pathdb/history_reader.go b/triedb/pathdb/history_reader.go index ce6aa693d1..1bf4cf648d 100644 --- a/triedb/pathdb/history_reader.go +++ b/triedb/pathdb/history_reader.go @@ -144,25 +144,17 @@ func (r *historyReader) readAccountMetadata(address common.Address, historyID ui // readStorageMetadata resolves the storage slot metadata within the specified // state history. func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash common.Hash, historyID uint64, slotOffset, slotNumber int) ([]byte, error) { - // TODO(rj493456442) optimize it with partial read - blob := rawdb.ReadStateStorageIndex(r.freezer, historyID) - if len(blob) == 0 { - return nil, fmt.Errorf("storage index is truncated, historyID: %d", historyID) + data, err := rawdb.ReadStateStorageIndex(r.freezer, historyID, slotIndexSize*slotOffset, slotIndexSize*slotNumber) + if err != nil { + msg := fmt.Sprintf("id: %d, slot-offset: %d, slot-length: %d", historyID, slotOffset, slotNumber) + return nil, fmt.Errorf("storage indices corrupted, %s, %w", msg, err) } - if len(blob)%slotIndexSize != 0 { - return nil, fmt.Errorf("storage indices is corrupted, historyID: %d, size: %d", historyID, len(blob)) - } - if slotIndexSize*(slotOffset+slotNumber) > len(blob) { - return nil, fmt.Errorf("storage indices is truncated, historyID: %d, size: %d, offset: %d, length: %d", historyID, len(blob), slotOffset, slotNumber) - } - subSlice := blob[slotIndexSize*slotOffset : slotIndexSize*(slotOffset+slotNumber)] - // TODO(rj493456442) get rid of the metadata resolution var ( m meta target common.Hash ) - blob = rawdb.ReadStateHistoryMeta(r.freezer, historyID) + blob := rawdb.ReadStateHistoryMeta(r.freezer, historyID) if err := m.decode(blob); err != nil { return nil, err } @@ -172,17 +164,17 @@ func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash target = storageKey } pos := sort.Search(slotNumber, func(i int) bool { - slotID := subSlice[slotIndexSize*i : slotIndexSize*i+common.HashLength] + slotID := data[slotIndexSize*i : slotIndexSize*i+common.HashLength] return bytes.Compare(slotID, target.Bytes()) >= 0 }) if pos == slotNumber { return nil, fmt.Errorf("storage metadata is not found, slot key: %#x, historyID: %d", storageKey, historyID) } offset := slotIndexSize * pos - if target != common.BytesToHash(subSlice[offset:offset+common.HashLength]) { + if target != common.BytesToHash(data[offset:offset+common.HashLength]) { return nil, fmt.Errorf("storage metadata is not found, slot key: %#x, historyID: %d", storageKey, historyID) } - return subSlice[offset : slotIndexSize*(pos+1)], nil + return data[offset : slotIndexSize*(pos+1)], nil } // readAccount retrieves the account data from the specified state history. @@ -194,12 +186,11 @@ func (r *historyReader) readAccount(address common.Address, historyID uint64) ([ length := int(metadata[common.AddressLength]) // one byte for account data length offset := int(binary.BigEndian.Uint32(metadata[common.AddressLength+1 : common.AddressLength+5])) // four bytes for the account data offset - // TODO(rj493456442) optimize it with partial read - data := rawdb.ReadStateAccountHistory(r.freezer, historyID) - if len(data) < length+offset { + data, err := rawdb.ReadStateAccountHistory(r.freezer, historyID, offset, length) + if err != nil { return nil, fmt.Errorf("account data is truncated, address: %#x, historyID: %d, size: %d, offset: %d, len: %d", address, historyID, len(data), offset, length) } - return data[offset : offset+length], nil + return data, nil } // readStorage retrieves the storage slot data from the specified state history. @@ -222,12 +213,11 @@ func (r *historyReader) readStorage(address common.Address, storageKey common.Ha length := int(slotMetadata[common.HashLength]) // one byte for slot data length offset := int(binary.BigEndian.Uint32(slotMetadata[common.HashLength+1 : common.HashLength+5])) // four bytes for slot data offset - // TODO(rj493456442) optimize it with partial read - data := rawdb.ReadStateStorageHistory(r.freezer, historyID) - if len(data) < offset+length { + data, err := rawdb.ReadStateStorageHistory(r.freezer, historyID, offset, length) + if err != nil { return nil, fmt.Errorf("storage data is truncated, address: %#x, key: %#x, historyID: %d, size: %d, offset: %d, len: %d", address, storageKey, historyID, len(data), offset, length) } - return data[offset : offset+length], nil + return data, nil } // read retrieves the state element data associated with the stateID. diff --git a/triedb/pathdb/history_trienode.go b/triedb/pathdb/history_trienode.go index 2a4459d4ad..f5eb590a9a 100644 --- a/triedb/pathdb/history_trienode.go +++ b/triedb/pathdb/history_trienode.go @@ -161,7 +161,7 @@ func newTrienodeHistory(root common.Hash, parent common.Hash, block uint64, node // sharedLen returns the length of the common prefix shared by a and b. func sharedLen(a, b []byte) int { n := min(len(a), len(b)) - for i := 0; i < n; i++ { + for i := range n { if a[i] != b[i] { return i } @@ -295,7 +295,7 @@ func decodeHeader(data []byte) (*trienodeMetadata, []common.Hash, []uint32, []ui keyOffsets = make([]uint32, 0, count) valOffsets = make([]uint32, 0, count) ) - for i := 0; i < count; i++ { + for i := range count { n := trienodeMetadataSize + trienodeTrieHeaderSize*i owner := common.BytesToHash(data[n : n+common.HashLength]) if i != 0 && bytes.Compare(owner.Bytes(), owners[i-1].Bytes()) <= 0 { @@ -348,7 +348,7 @@ func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) ([]st if len(keySection) < int(8*nRestarts)+4 { return nil, fmt.Errorf("key section too short, restarts: %d, size: %d", nRestarts, len(keySection)) } - for i := 0; i < int(nRestarts); i++ { + for i := range int(nRestarts) { o := len(keySection) - 4 - (int(nRestarts)-i)*8 keyOffset := binary.BigEndian.Uint32(keySection[o : o+4]) if i != 0 && keyOffset <= keyOffsets[i-1] { @@ -469,7 +469,7 @@ func (h *trienodeHistory) decode(header []byte, keySection []byte, valueSection h.nodeList = make(map[common.Hash][]string) h.nodes = make(map[common.Hash]map[string][]byte) - for i := 0; i < len(owners); i++ { + for i := range len(owners) { // Resolve the boundary of key section keyStart := keyOffsets[i] keyLimit := len(keySection) @@ -524,7 +524,7 @@ func newSingleTrienodeHistoryReader(id uint64, reader ethdb.AncientReader, keyRa } keyStart := int(keyRange.start) keyLimit := int(keyRange.limit) - if keyLimit == math.MaxUint32 { + if keyRange.limit == math.MaxUint32 { keyLimit = len(keyData) } if len(keyData) < keyStart || len(keyData) < keyLimit { diff --git a/version/version.go b/version/version.go index db4e5394b9..ead2d04f2a 100644 --- a/version/version.go +++ b/version/version.go @@ -19,6 +19,6 @@ package version const ( Major = 1 // Major version component of the current release Minor = 16 // Minor version component of the current release - Patch = 5 // Patch version component of the current release + Patch = 6 // Patch version component of the current release Meta = "unstable" // Version metadata to append to the version string )