From edc864f9ba186fd307d9c98c42136db6c9411cf9 Mon Sep 17 00:00:00 2001 From: alex <152680487+bodhi-crypo@users.noreply.github.com> Date: Mon, 18 Dec 2023 16:35:12 +0800 Subject: [PATCH 01/18] all: fix typos in comments (#28682) chore(core,eth):fix a couple of typos --- cmd/clef/pythonsigner.py | 2 +- core/txpool/blobpool/blobpool.go | 4 ++-- core/txpool/blobpool/blobpool_test.go | 2 +- core/txpool/blobpool/metrics.go | 2 +- core/vm/runtime/runtime_test.go | 2 +- eth/downloader/downloader.go | 2 +- eth/downloader/resultstore.go | 2 +- ethclient/gethclient/gethclient_test.go | 2 +- p2p/rlpx/rlpx_test.go | 2 +- p2p/simulations/network_test.go | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/clef/pythonsigner.py b/cmd/clef/pythonsigner.py index b9ea1e406a..5d0eb18dcc 100644 --- a/cmd/clef/pythonsigner.py +++ b/cmd/clef/pythonsigner.py @@ -91,7 +91,7 @@ class StdIOHandler: {"jsonrpc":"2.0","id":20,"method":"ui_approveTx","params":[{"transaction":{"from":"0xDEADbEeF000000000000000000000000DeaDbeEf","to":"0xDEADbEeF000000000000000000000000DeaDbeEf","gas":"0x3e8","gasPrice":"0x5","maxFeePerGas":null,"maxPriorityFeePerGas":null,"value":"0x6","nonce":"0x1","data":"0x"},"call_info":null,"meta":{"remote":"clef binary","local":"main","scheme":"in-proc","User-Agent":"","Origin":""}}]} :param transaction: transaction info - :param call_info: info abou the call, e.g. if ABI info could not be + :param call_info: info about the call, e.g. if ABI info could not be :param meta: metadata about the request, e.g. where the call comes from :return: """ # noqa: E501 diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 32c6c0e8fe..195697a8f6 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -738,7 +738,7 @@ func (p *BlobPool) offload(addr common.Address, nonce uint64, id uint64, inclusi } // Reset implements txpool.SubPool, allowing the blob pool's internal state to be -// kept in sync with the main transacion pool's internal state. +// kept in sync with the main transaction pool's internal state. func (p *BlobPool) Reset(oldHead, newHead *types.Header) { waitStart := time.Now() p.lock.Lock() @@ -972,7 +972,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error { } // SetGasTip implements txpool.SubPool, allowing the blob pool's gas requirements -// to be kept in sync with the main transacion pool's gas requirements. +// to be kept in sync with the main transaction pool's gas requirements. func (p *BlobPool) SetGasTip(tip *big.Int) { p.lock.Lock() defer p.lock.Unlock() diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index fa3e8edc90..b709ad0e58 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -594,7 +594,7 @@ func TestOpenDrops(t *testing.T) { verifyPoolInternals(t, pool) } -// Tests that transactions loaded from disk are indexed corrently. +// Tests that transactions loaded from disk are indexed correctly. // // - 1. Transactions must be groupped by sender, sorted by nonce // - 2. Eviction thresholds are calculated correctly for the sequences diff --git a/core/txpool/blobpool/metrics.go b/core/txpool/blobpool/metrics.go index 070cc5ca47..587804cc61 100644 --- a/core/txpool/blobpool/metrics.go +++ b/core/txpool/blobpool/metrics.go @@ -65,7 +65,7 @@ var ( pooltipGauge = metrics.NewRegisteredGauge("blobpool/pooltip", nil) // addwait/time, resetwait/time and getwait/time track the rough health of - // the pool and wether or not it's capable of keeping up with the load from + // the pool and whether or not it's capable of keeping up with the load from // the network. addwaitHist = metrics.NewRegisteredHistogram("blobpool/addwait", nil, metrics.NewExpDecaySample(1028, 0.015)) addtimeHist = metrics.NewRegisteredHistogram("blobpool/addtime", nil, metrics.NewExpDecaySample(1028, 0.015)) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 796d3b4434..e71760bb23 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -671,7 +671,7 @@ func TestColdAccountAccessCost(t *testing.T) { for ii, op := range tracer.StructLogs() { t.Logf("%d: %v %d", ii, op.OpName(), op.GasCost) } - t.Fatalf("tescase %d, gas report wrong, step %d, have %d want %d", i, tc.step, have, want) + t.Fatalf("testcase %d, gas report wrong, step %d, have %d want %d", i, tc.step, have, want) } } } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 2ca7e328c6..f1cfa92d5d 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -576,7 +576,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td, ttd * // For non-merged networks, if there is a checkpoint available, then calculate // the ancientLimit through that. Otherwise calculate the ancient limit through // the advertised height of the remote peer. This most is mostly a fallback for - // legacy networks, but should eventually be droppped. TODO(karalabe). + // legacy networks, but should eventually be dropped. TODO(karalabe). if beaconMode { // Beacon sync, use the latest finalized block as the ancient limit // or a reasonable height if no finalized block is yet announced. diff --git a/eth/downloader/resultstore.go b/eth/downloader/resultstore.go index 7f7f5a89e2..e4323c04eb 100644 --- a/eth/downloader/resultstore.go +++ b/eth/downloader/resultstore.go @@ -142,7 +142,7 @@ func (r *resultStore) HasCompletedItems() bool { // countCompleted returns the number of items ready for delivery, stopping at // the first non-complete item. // -// The mthod assumes (at least) rlock is held. +// The method assumes (at least) rlock is held. func (r *resultStore) countCompleted() int { // We iterate from the already known complete point, and see // if any more has completed since last count diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index a718246bd0..fdd94a7d73 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -450,7 +450,7 @@ func testCallContract(t *testing.T, client *rpc.Client) { func TestOverrideAccountMarshal(t *testing.T) { om := map[common.Address]OverrideAccount{ {0x11}: { - // Zero-valued nonce is not overriddden, but simply dropped by the encoder. + // Zero-valued nonce is not overridden, but simply dropped by the encoder. Nonce: 0, }, {0xaa}: { diff --git a/p2p/rlpx/rlpx_test.go b/p2p/rlpx/rlpx_test.go index 28759f2b49..136cb1b5bf 100644 --- a/p2p/rlpx/rlpx_test.go +++ b/p2p/rlpx/rlpx_test.go @@ -421,7 +421,7 @@ func BenchmarkThroughput(b *testing.B) { } conn2.SetSnappy(true) if err := <-handshakeDone; err != nil { - b.Fatal("server hanshake error:", err) + b.Fatal("server handshake error:", err) } // Read N messages. diff --git a/p2p/simulations/network_test.go b/p2p/simulations/network_test.go index ab8cf19462..4ed1e4e6c3 100644 --- a/p2p/simulations/network_test.go +++ b/p2p/simulations/network_test.go @@ -683,7 +683,7 @@ func triggerChecks(ctx context.Context, ids []enode.ID, trigger chan enode.ID, i } } -// \todo: refactor to implement shapshots +// \todo: refactor to implement snapshots // and connect configuration methods once these are moved from // swarm/network/simulations/connect.go func BenchmarkMinimalService(b *testing.B) { From 5b22a472d6aaaa17daf0543b5914ca1f2f5518a7 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 18 Dec 2023 10:47:21 +0100 Subject: [PATCH 02/18] p2p/discover: add liveness check in collectTableNodes (#28686) * p2p/discover: add liveness check in collectTableNodes * p2p/discover: fix test * p2p/discover: rename to appendLiveNodes * p2p/discover: add dedup logic back * p2p/discover: simplify * p2p/discover: fix issue found by test --- p2p/discover/table.go | 20 ++++++++++++++++++++ p2p/discover/table_test.go | 2 +- p2p/discover/table_util_test.go | 5 ++++- p2p/discover/v4_lookup_test.go | 6 +++--- p2p/discover/v4_udp_test.go | 2 +- p2p/discover/v5_udp.go | 17 ++++------------- p2p/discover/v5_udp_test.go | 8 ++++---- 7 files changed, 37 insertions(+), 23 deletions(-) diff --git a/p2p/discover/table.go b/p2p/discover/table.go index e6dafb0dca..2b7a28708b 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -459,6 +459,26 @@ func (tab *Table) findnodeByID(target enode.ID, nresults int, preferLive bool) * return nodes } +// appendLiveNodes adds nodes at the given distance to the result slice. +func (tab *Table) appendLiveNodes(dist uint, result []*enode.Node) []*enode.Node { + if dist > 256 { + return result + } + if dist == 0 { + return append(result, tab.self()) + } + + tab.mutex.Lock() + defer tab.mutex.Unlock() + for _, n := range tab.bucketAtDistance(int(dist)).entries { + if n.livenessChecks >= 1 { + node := n.Node // avoid handing out pointer to struct field + result = append(result, &node) + } + } + return result +} + // len returns the number of nodes in the table. func (tab *Table) len() (n int) { tab.mutex.Lock() diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index 2781dd4225..3ba3422251 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -199,7 +199,7 @@ func TestTable_findnodeByID(t *testing.T) { tab, db := newTestTable(transport) defer db.Close() defer tab.close() - fillTable(tab, test.All) + fillTable(tab, test.All, true) // check that closest(Target, N) returns nodes result := tab.findnodeByID(test.Target, test.N, false).entries diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 8f3813bdcf..d6309dfd6c 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -109,8 +109,11 @@ func fillBucket(tab *Table, n *node) (last *node) { // fillTable adds nodes the table to the end of their corresponding bucket // if the bucket is not full. The caller must not hold tab.mutex. -func fillTable(tab *Table, nodes []*node) { +func fillTable(tab *Table, nodes []*node, setLive bool) { for _, n := range nodes { + if setLive { + n.livenessChecks = 1 + } tab.addSeenNode(n) } } diff --git a/p2p/discover/v4_lookup_test.go b/p2p/discover/v4_lookup_test.go index 1f9ad69d0a..8867a5a8ac 100644 --- a/p2p/discover/v4_lookup_test.go +++ b/p2p/discover/v4_lookup_test.go @@ -40,7 +40,7 @@ func TestUDPv4_Lookup(t *testing.T) { } // Seed table with initial node. - fillTable(test.table, []*node{wrapNode(lookupTestnet.node(256, 0))}) + fillTable(test.table, []*node{wrapNode(lookupTestnet.node(256, 0))}, true) // Start the lookup. resultC := make(chan []*enode.Node, 1) @@ -74,7 +74,7 @@ func TestUDPv4_LookupIterator(t *testing.T) { for i := range lookupTestnet.dists[256] { bootnodes[i] = wrapNode(lookupTestnet.node(256, i)) } - fillTable(test.table, bootnodes) + fillTable(test.table, bootnodes, true) go serveTestnet(test, lookupTestnet) // Create the iterator and collect the nodes it yields. @@ -109,7 +109,7 @@ func TestUDPv4_LookupIteratorClose(t *testing.T) { for i := range lookupTestnet.dists[256] { bootnodes[i] = wrapNode(lookupTestnet.node(256, i)) } - fillTable(test.table, bootnodes) + fillTable(test.table, bootnodes, true) go serveTestnet(test, lookupTestnet) it := test.udp.RandomNodes() diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index 53ecb1bc6e..361e379626 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -269,7 +269,7 @@ func TestUDPv4_findnode(t *testing.T) { } nodes.push(n, numCandidates) } - fillTable(test.table, nodes.entries) + fillTable(test.table, nodes.entries, false) // ensure there's a bond with the test node, // findnode won't be accepted otherwise. diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 6ba7a90618..8b3e33d37c 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -851,6 +851,7 @@ func (t *UDPv5) handleFindnode(p *v5wire.Findnode, fromID enode.ID, fromAddr *ne // collectTableNodes creates a FINDNODE result set for the given distances. func (t *UDPv5) collectTableNodes(rip net.IP, distances []uint, limit int) []*enode.Node { + var bn []*enode.Node var nodes []*enode.Node var processed = make(map[uint]struct{}) for _, dist := range distances { @@ -859,21 +860,11 @@ func (t *UDPv5) collectTableNodes(rip net.IP, distances []uint, limit int) []*en if seen || dist > 256 { continue } - - // Get the nodes. - var bn []*enode.Node - if dist == 0 { - bn = []*enode.Node{t.Self()} - } else if dist <= 256 { - t.tab.mutex.Lock() - bn = unwrapNodes(t.tab.bucketAtDistance(int(dist)).entries) - t.tab.mutex.Unlock() - } processed[dist] = struct{}{} - // Apply some pre-checks to avoid sending invalid nodes. - for _, n := range bn { - // TODO livenessChecks > 1 + for _, n := range t.tab.appendLiveNodes(dist, bn[:0]) { + // Apply some pre-checks to avoid sending invalid nodes. + // Note liveness is checked by appendLiveNodes. if netutil.CheckRelayIP(rip, n.IP()) != nil { continue } diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 18d8aeac6d..eaa969ea8b 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -159,9 +159,9 @@ func TestUDPv5_findnodeHandling(t *testing.T) { nodes253 := nodesAtDistance(test.table.self().ID(), 253, 16) nodes249 := nodesAtDistance(test.table.self().ID(), 249, 4) nodes248 := nodesAtDistance(test.table.self().ID(), 248, 10) - fillTable(test.table, wrapNodes(nodes253)) - fillTable(test.table, wrapNodes(nodes249)) - fillTable(test.table, wrapNodes(nodes248)) + fillTable(test.table, wrapNodes(nodes253), true) + fillTable(test.table, wrapNodes(nodes249), true) + fillTable(test.table, wrapNodes(nodes248), true) // Requesting with distance zero should return the node's own record. test.packetIn(&v5wire.Findnode{ReqID: []byte{0}, Distances: []uint{0}}) @@ -589,7 +589,7 @@ func TestUDPv5_lookup(t *testing.T) { // Seed table with initial node. initialNode := lookupTestnet.node(256, 0) - fillTable(test.table, []*node{wrapNode(initialNode)}) + fillTable(test.table, []*node{wrapNode(initialNode)}, true) // Start the lookup. resultC := make(chan []*enode.Node, 1) From 02766d349a14171b781b0afe083f6d898cf58c3b Mon Sep 17 00:00:00 2001 From: Sebastian Stammler Date: Mon, 18 Dec 2023 13:28:41 +0100 Subject: [PATCH 03/18] internal/flags: add missing flag types for auto-env-var generation (#28692) Certain flags, such as `--rpc.txfeecap` currently do not have an env-var auto-generated for them. This change adds three missing cli flag types to the auto env-var helper function to fix this. --- internal/flags/helpers.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index d4b8e373cc..d9d1f79036 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -105,7 +105,7 @@ func MigrateGlobalFlags(ctx *cli.Context) { func doMigrateFlags(ctx *cli.Context) { // Figure out if there are any aliases of commands. If there are, we want // to ignore them when iterating over the flags. - var aliases = make(map[string]bool) + aliases := make(map[string]bool) for _, fl := range ctx.Command.Flags { for _, alias := range fl.Names()[1:] { aliases[alias] = true @@ -239,15 +239,24 @@ func AutoEnvVars(flags []cli.Flag, prefix string) { case *cli.StringFlag: flag.EnvVars = append(flag.EnvVars, envvar) + case *cli.StringSliceFlag: + flag.EnvVars = append(flag.EnvVars, envvar) + case *cli.BoolFlag: flag.EnvVars = append(flag.EnvVars, envvar) case *cli.IntFlag: flag.EnvVars = append(flag.EnvVars, envvar) + case *cli.Int64Flag: + flag.EnvVars = append(flag.EnvVars, envvar) + case *cli.Uint64Flag: flag.EnvVars = append(flag.EnvVars, envvar) + case *cli.Float64Flag: + flag.EnvVars = append(flag.EnvVars, envvar) + case *cli.DurationFlag: flag.EnvVars = append(flag.EnvVars, envvar) From 05bbc56677129c759a28330a22e1e6dc3b8ce8f5 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Mon, 18 Dec 2023 20:56:27 +0800 Subject: [PATCH 04/18] cmd/evm: default to mirror mainnet forks enabled (#28691) cmd/evm: default to using dev chain config (all mainnet HFs activated at block/timestamp 0 --- cmd/evm/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index c9a870022a..f3ffb3ed9f 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -144,7 +144,7 @@ func runCmd(ctx *cli.Context) error { initialGas = genesisConfig.GasLimit } } else { - genesisConfig.Config = params.AllEthashProtocolChanges + genesisConfig.Config = params.AllDevChainProtocolChanges } db := rawdb.NewMemoryDatabase() From 553bafc12720d2a3eef396cfea20f0637fb41cc4 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Mon, 18 Dec 2023 14:11:27 +0100 Subject: [PATCH 05/18] cmd/evm, cmd/clef, cmd/bootnode: fix / unify logging (#28696) This change fixes a problem with our non-core binaries: evm, clef, bootnode. First of all, they failed to convert from legacy loglevels 1 to 5, to the new slog loglevels -4 to 4. Secondly, the logging was actually setup in the init phase, and then overridden in the main. This is not needed for evm, since it used the same flag name as the main geth verbosity. Better to let the flags/internal handle the logging init. --- cmd/bootnode/main.go | 6 +++--- cmd/clef/main.go | 4 ++-- cmd/evm/internal/t8ntool/block.go | 7 ------- cmd/evm/internal/t8ntool/transaction.go | 7 ------- cmd/evm/internal/t8ntool/transition.go | 7 ------- cmd/evm/main.go | 3 --- 6 files changed, 5 insertions(+), 29 deletions(-) diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 1660b43b74..350b85df1e 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" - "golang.org/x/exp/slog" ) func main() { @@ -45,7 +44,7 @@ func main() { natdesc = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|pmp:|extip:)") netrestrict = flag.String("netrestrict", "", "restrict network communication to the given IP networks (CIDR masks)") runv5 = flag.Bool("v5", false, "run a v5 topic discovery bootnode") - verbosity = flag.Int("verbosity", int(log.LvlInfo), "log verbosity (0-5)") + verbosity = flag.Int("verbosity", 3, "log verbosity (0-5)") vmodule = flag.String("vmodule", "", "log verbosity pattern") nodeKey *ecdsa.PrivateKey @@ -54,7 +53,8 @@ func main() { flag.Parse() glogger := log.NewGlogHandler(log.NewTerminalHandler(os.Stderr, false)) - glogger.Verbosity(slog.Level(*verbosity)) + slogVerbosity := log.FromLegacyLevel(*verbosity) + glogger.Verbosity(slogVerbosity) glogger.Vmodule(*vmodule) log.SetDefault(log.NewLogger(glogger)) diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 27b7b70771..2346991369 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -57,7 +57,6 @@ import ( "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" "github.com/urfave/cli/v2" - "golang.org/x/exp/slog" ) const legalWarning = ` @@ -493,7 +492,8 @@ func initialize(c *cli.Context) error { if usecolor { output = colorable.NewColorable(logOutput) } - log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(output, slog.Level(c.Int(logLevelFlag.Name)), usecolor))) + verbosity := log.FromLegacyLevel(c.Int(logLevelFlag.Name)) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(output, verbosity, usecolor))) return nil } diff --git a/cmd/evm/internal/t8ntool/block.go b/cmd/evm/internal/t8ntool/block.go index 429ae12c54..a2dc473437 100644 --- a/cmd/evm/internal/t8ntool/block.go +++ b/cmd/evm/internal/t8ntool/block.go @@ -30,10 +30,8 @@ import ( "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/urfave/cli/v2" - "golang.org/x/exp/slog" ) //go:generate go run github.com/fjl/gencodec -type header -field-override headerMarshaling -out gen_header.go @@ -216,11 +214,6 @@ func (i *bbInput) sealClique(block *types.Block) (*types.Block, error) { // BuildBlock constructs a block from the given inputs. func BuildBlock(ctx *cli.Context) error { - // Configure the go-ethereum logger - glogger := log.NewGlogHandler(log.NewTerminalHandler(os.Stderr, false)) - glogger.Verbosity(slog.Level(ctx.Int(VerbosityFlag.Name))) - log.SetDefault(log.NewLogger(glogger)) - baseDir, err := createBasedir(ctx) if err != nil { return NewError(ErrorIO, fmt.Errorf("failed creating output basedir: %v", err)) diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index e1c98c7fe2..8533b78637 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -28,12 +28,10 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" - "golang.org/x/exp/slog" ) type result struct { @@ -66,11 +64,6 @@ func (r *result) MarshalJSON() ([]byte, error) { } func Transaction(ctx *cli.Context) error { - // Configure the go-ethereum logger - glogger := log.NewGlogHandler(log.NewTerminalHandler(os.Stderr, false)) - glogger.Verbosity(slog.Level(ctx.Int(VerbosityFlag.Name))) - log.SetDefault(log.NewLogger(glogger)) - var ( err error ) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index a01dfedab9..0a9c555cff 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -24,8 +24,6 @@ import ( "os" "path" - "golang.org/x/exp/slog" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" @@ -82,11 +80,6 @@ type input struct { } func Transition(ctx *cli.Context) error { - // Configure the go-ethereum logger - glogger := log.NewGlogHandler(log.NewTerminalHandler(os.Stderr, false)) - glogger.Verbosity(slog.Level(ctx.Int(VerbosityFlag.Name))) - log.SetDefault(log.NewLogger(glogger)) - var ( err error tracer vm.EVMLogger diff --git a/cmd/evm/main.go b/cmd/evm/main.go index ef5d25418d..f486856805 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -158,7 +158,6 @@ var stateTransitionCommand = &cli.Command{ t8ntool.ForknameFlag, t8ntool.ChainIDFlag, t8ntool.RewardFlag, - t8ntool.VerbosityFlag, }, } @@ -171,7 +170,6 @@ var transactionCommand = &cli.Command{ t8ntool.InputTxsFlag, t8ntool.ChainIDFlag, t8ntool.ForknameFlag, - t8ntool.VerbosityFlag, }, } @@ -188,7 +186,6 @@ var blockBuilderCommand = &cli.Command{ t8ntool.InputWithdrawalsFlag, t8ntool.InputTxsRlpFlag, t8ntool.SealCliqueFlag, - t8ntool.VerbosityFlag, }, } From c18c5c3d9297195a6f6b05076ae7940d45ab6846 Mon Sep 17 00:00:00 2001 From: Delweng Date: Mon, 18 Dec 2023 22:16:25 +0800 Subject: [PATCH 06/18] cmd/evm: t8n support custom tracers (#28557) This change implements ability for the `evm t8n` tool to use custom tracers; either 'native' golang tracers or javascript tracers. --- cmd/evm/internal/t8ntool/execution.go | 2 +- cmd/evm/internal/t8ntool/flags.go | 18 +++--- cmd/evm/internal/t8ntool/tracewriter.go | 81 +++++++++++++++++++++++++ cmd/evm/internal/t8ntool/transition.go | 61 +++++++------------ cmd/evm/main.go | 8 ++- 5 files changed, 119 insertions(+), 51 deletions(-) create mode 100644 cmd/evm/internal/t8ntool/tracewriter.go diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 5cac5f07f8..a4ffd09e4f 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -117,7 +117,7 @@ type rejectedTx struct { // Apply applies a set of transactions to a pre-state func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, txIt txIterator, miningReward int64, - getTracerFn func(txIndex int, txHash common.Hash) (tracer vm.EVMLogger, err error)) (*state.StateDB, *ExecutionResult, []byte, error) { + getTracerFn func(txIndex int, txHash common.Hash) (vm.EVMLogger, error)) (*state.StateDB, *ExecutionResult, []byte, error) { // Capture errors for BLOCKHASH operation, if we haven't been supplied the // required blockhashes var hashError error diff --git a/cmd/evm/internal/t8ntool/flags.go b/cmd/evm/internal/t8ntool/flags.go index de19dbc851..c2eca8cc21 100644 --- a/cmd/evm/internal/t8ntool/flags.go +++ b/cmd/evm/internal/t8ntool/flags.go @@ -28,12 +28,15 @@ import ( var ( TraceFlag = &cli.BoolFlag{ Name: "trace", - Usage: "Output full trace logs to files .jsonl", + Usage: "Configures the use of the JSON opcode tracer. This tracer emits traces to files as trace--.jsonl", } - TraceDisableMemoryFlag = &cli.BoolFlag{ - Name: "trace.nomemory", - Value: true, - Usage: "Disable full memory dump in traces (deprecated)", + TraceTracerFlag = &cli.StringFlag{ + Name: "trace.tracer", + Usage: "Configures the use of a custom tracer, e.g native or js tracers. Examples are callTracer and 4byteTracer. These tracers emit results into files as trace--.json", + } + TraceTracerConfigFlag = &cli.StringFlag{ + Name: "trace.jsonconfig", + Usage: "The configurations for the custom tracer specified by --trace.tracer. If provided, must be in JSON format", } TraceEnableMemoryFlag = &cli.BoolFlag{ Name: "trace.memory", @@ -43,11 +46,6 @@ var ( Name: "trace.nostack", Usage: "Disable stack output in traces", } - TraceDisableReturnDataFlag = &cli.BoolFlag{ - Name: "trace.noreturndata", - Value: true, - Usage: "Disable return data output in traces (deprecated)", - } TraceEnableReturnDataFlag = &cli.BoolFlag{ Name: "trace.returndata", Usage: "Enable return data output in traces", diff --git a/cmd/evm/internal/t8ntool/tracewriter.go b/cmd/evm/internal/t8ntool/tracewriter.go new file mode 100644 index 0000000000..e4efad112f --- /dev/null +++ b/cmd/evm/internal/t8ntool/tracewriter.go @@ -0,0 +1,81 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package t8ntool + +import ( + "encoding/json" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/log" +) + +// traceWriter is an vm.EVMLogger which also holds an inner logger/tracer. +// When the TxEnd event happens, the inner tracer result is written to the file, and +// the file is closed. +type traceWriter struct { + inner vm.EVMLogger + f io.WriteCloser +} + +// Compile-time interface check +var _ = vm.EVMLogger((*traceWriter)(nil)) + +func (t *traceWriter) CaptureTxEnd(restGas uint64) { + t.inner.CaptureTxEnd(restGas) + defer t.f.Close() + + if tracer, ok := t.inner.(tracers.Tracer); ok { + result, err := tracer.GetResult() + if err != nil { + log.Warn("Error in tracer", "err", err) + return + } + err = json.NewEncoder(t.f).Encode(result) + if err != nil { + log.Warn("Error writing tracer output", "err", err) + return + } + } +} + +func (t *traceWriter) CaptureTxStart(gasLimit uint64) { t.inner.CaptureTxStart(gasLimit) } +func (t *traceWriter) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { + t.inner.CaptureStart(env, from, to, create, input, gas, value) +} + +func (t *traceWriter) CaptureEnd(output []byte, gasUsed uint64, err error) { + t.inner.CaptureEnd(output, gasUsed, err) +} + +func (t *traceWriter) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + t.inner.CaptureEnter(typ, from, to, input, gas, value) +} + +func (t *traceWriter) CaptureExit(output []byte, gasUsed uint64, err error) { + t.inner.CaptureExit(output, gasUsed, err) +} + +func (t *traceWriter) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { + t.inner.CaptureState(pc, op, gas, cost, scope, rData, depth, err) +} +func (t *traceWriter) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { + t.inner.CaptureFault(pc, op, gas, cost, scope, depth, err) +} diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 0a9c555cff..c8ba69f40f 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -80,57 +81,43 @@ type input struct { } func Transition(ctx *cli.Context) error { - var ( - err error - tracer vm.EVMLogger - ) - var getTracer func(txIndex int, txHash common.Hash) (vm.EVMLogger, error) + var getTracer = func(txIndex int, txHash common.Hash) (vm.EVMLogger, error) { return nil, nil } baseDir, err := createBasedir(ctx) if err != nil { return NewError(ErrorIO, fmt.Errorf("failed creating output basedir: %v", err)) } - if ctx.Bool(TraceFlag.Name) { - if ctx.IsSet(TraceDisableMemoryFlag.Name) && ctx.IsSet(TraceEnableMemoryFlag.Name) { - return NewError(ErrorConfig, fmt.Errorf("can't use both flags --%s and --%s", TraceDisableMemoryFlag.Name, TraceEnableMemoryFlag.Name)) - } - if ctx.IsSet(TraceDisableReturnDataFlag.Name) && ctx.IsSet(TraceEnableReturnDataFlag.Name) { - return NewError(ErrorConfig, fmt.Errorf("can't use both flags --%s and --%s", TraceDisableReturnDataFlag.Name, TraceEnableReturnDataFlag.Name)) - } - if ctx.IsSet(TraceDisableMemoryFlag.Name) { - log.Warn(fmt.Sprintf("--%s has been deprecated in favour of --%s", TraceDisableMemoryFlag.Name, TraceEnableMemoryFlag.Name)) - } - if ctx.IsSet(TraceDisableReturnDataFlag.Name) { - log.Warn(fmt.Sprintf("--%s has been deprecated in favour of --%s", TraceDisableReturnDataFlag.Name, TraceEnableReturnDataFlag.Name)) - } + + if ctx.Bool(TraceFlag.Name) { // JSON opcode tracing // Configure the EVM logger logConfig := &logger.Config{ DisableStack: ctx.Bool(TraceDisableStackFlag.Name), - EnableMemory: !ctx.Bool(TraceDisableMemoryFlag.Name) || ctx.Bool(TraceEnableMemoryFlag.Name), - EnableReturnData: !ctx.Bool(TraceDisableReturnDataFlag.Name) || ctx.Bool(TraceEnableReturnDataFlag.Name), + EnableMemory: ctx.Bool(TraceEnableMemoryFlag.Name), + EnableReturnData: ctx.Bool(TraceEnableReturnDataFlag.Name), Debug: true, } - var prevFile *os.File - // This one closes the last file - defer func() { - if prevFile != nil { - prevFile.Close() - } - }() getTracer = func(txIndex int, txHash common.Hash) (vm.EVMLogger, error) { - if prevFile != nil { - prevFile.Close() - } traceFile, err := os.Create(path.Join(baseDir, fmt.Sprintf("trace-%d-%v.jsonl", txIndex, txHash.String()))) if err != nil { return nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err)) } - prevFile = traceFile - return logger.NewJSONLogger(logConfig, traceFile), nil + return &traceWriter{logger.NewJSONLogger(logConfig, traceFile), traceFile}, nil } - } else { - getTracer = func(txIndex int, txHash common.Hash) (tracer vm.EVMLogger, err error) { - return nil, nil + } else if ctx.IsSet(TraceTracerFlag.Name) { + var config json.RawMessage + if ctx.IsSet(TraceTracerConfigFlag.Name) { + config = []byte(ctx.String(TraceTracerConfigFlag.Name)) + } + getTracer = func(txIndex int, txHash common.Hash) (vm.EVMLogger, error) { + traceFile, err := os.Create(path.Join(baseDir, fmt.Sprintf("trace-%d-%v.json", txIndex, txHash.String()))) + if err != nil { + return nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err)) + } + tracer, err := tracers.DefaultDirectory.New(ctx.String(TraceTracerFlag.Name), nil, config) + if err != nil { + return nil, NewError(ErrorConfig, fmt.Errorf("failed instantiating tracer: %w", err)) + } + return &traceWriter{tracer, traceFile}, nil } } // We need to load three things: alloc, env and transactions. May be either in @@ -169,9 +156,7 @@ func Transition(ctx *cli.Context) error { } prestate.Env = *inputData.Env - vmConfig := vm.Config{ - Tracer: tracer, - } + vmConfig := vm.Config{} // Construct the chainconfig var chainConfig *params.ChainConfig if cConf, extraEips, err := tests.GetChainConfig(ctx.String(ForknameFlag.Name)); err != nil { diff --git a/cmd/evm/main.go b/cmd/evm/main.go index f486856805..c3e6a4af91 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -26,6 +26,10 @@ import ( "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/flags" "github.com/urfave/cli/v2" + + // Force-load the tracer engines to trigger registration + _ "github.com/ethereum/go-ethereum/eth/tracers/js" + _ "github.com/ethereum/go-ethereum/eth/tracers/native" ) var ( @@ -143,10 +147,10 @@ var stateTransitionCommand = &cli.Command{ Action: t8ntool.Transition, Flags: []cli.Flag{ t8ntool.TraceFlag, - t8ntool.TraceDisableMemoryFlag, + t8ntool.TraceTracerFlag, + t8ntool.TraceTracerConfigFlag, t8ntool.TraceEnableMemoryFlag, t8ntool.TraceDisableStackFlag, - t8ntool.TraceDisableReturnDataFlag, t8ntool.TraceEnableReturnDataFlag, t8ntool.OutputBasedir, t8ntool.OutputAllocFlag, From a18b845ecda84968125c09f054deb49773cd8cfe Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 18 Dec 2023 18:53:47 +0100 Subject: [PATCH 07/18] params: release go-ethereum v1.13.6 stable --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index bcab461a43..636f95bad8 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 5 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 13 // Minor version component of the current release + VersionPatch = 6 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 4410c1416abce38925c60550bf2bfb7f7db5c3f5 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 18 Dec 2023 19:10:11 +0100 Subject: [PATCH 08/18] params: begin v1.13.7 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 636f95bad8..ef3c47e7ac 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 6 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 13 // Minor version component of the current release + VersionPatch = 7 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 54a400ee717caf44603fac390314747c5592ee1b Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 19 Dec 2023 03:09:41 +0800 Subject: [PATCH 09/18] internal/ethapi: ethSendTransaction check baseFee (#27834) If the EIP-1559 is activated, reject 0-priced transactions in the rpc level --- internal/ethapi/transaction_args.go | 33 +++++++++++++++++------- internal/ethapi/transaction_args_test.go | 22 ++++++++++++++++ 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index e4cf81a3f4..aaf2c05d89 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -137,20 +137,35 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") } - // If the tx has completely specified a fee mechanism, no default is needed. This allows users - // who are not yet synced past London to get defaults for other tx values. See - // https://github.com/ethereum/go-ethereum/pull/23274 for more information. + // If the tx has completely specified a fee mechanism, no default is needed. + // This allows users who are not yet synced past London to get defaults for + // other tx values. See https://github.com/ethereum/go-ethereum/pull/23274 + // for more information. eip1559ParamsSet := args.MaxFeePerGas != nil && args.MaxPriorityFeePerGas != nil - if (args.GasPrice != nil && !eip1559ParamsSet) || (args.GasPrice == nil && eip1559ParamsSet) { - // Sanity check the EIP-1559 fee parameters if present. - if args.GasPrice == nil && args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + + // Sanity check the EIP-1559 fee parameters if present. + if args.GasPrice == nil && eip1559ParamsSet { + if args.MaxFeePerGas.ToInt().Sign() == 0 { + return errors.New("maxFeePerGas must be non-zero") + } + if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) } - return nil + return nil // No need to set anything, user already set MaxFeePerGas and MaxPriorityFeePerGas } - // Now attempt to fill in default value depending on whether London is active or not. + // Sanity check the non-EIP-1559 fee parameters. head := b.CurrentHeader() - if b.ChainConfig().IsLondon(head.Number) { + isLondon := b.ChainConfig().IsLondon(head.Number) + if args.GasPrice != nil && !eip1559ParamsSet { + // Zero gas-price is not allowed after London fork + if args.GasPrice.ToInt().Sign() == 0 && isLondon { + return errors.New("gasPrice must be non-zero after london fork") + } + return nil // No need to set anything, user already set GasPrice + } + + // Now attempt to fill in default value depending on whether London is active or not. + if isLondon { // London is active, set maxPriorityFeePerGas and maxFeePerGas. if err := args.setLondonFeeDefaults(ctx, head, b); err != nil { return err diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index 9dc58bdeb5..ab7c2f70ed 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -52,6 +52,7 @@ func TestSetFeeDefaults(t *testing.T) { var ( b = newBackendMock() + zero = (*hexutil.Big)(big.NewInt(0)) fortytwo = (*hexutil.Big)(big.NewInt(42)) maxFee = (*hexutil.Big)(new(big.Int).Add(new(big.Int).Mul(b.current.BaseFee, big.NewInt(2)), fortytwo.ToInt())) al = &types.AccessList{types.AccessTuple{Address: common.Address{0xaa}, StorageKeys: []common.Hash{{0x01}}}} @@ -66,6 +67,13 @@ func TestSetFeeDefaults(t *testing.T) { &TransactionArgs{GasPrice: fortytwo}, nil, }, + { + "legacy tx pre-London with zero price", + false, + &TransactionArgs{GasPrice: zero}, + &TransactionArgs{GasPrice: zero}, + nil, + }, { "legacy tx post-London, explicit gas price", true, @@ -73,6 +81,13 @@ func TestSetFeeDefaults(t *testing.T) { &TransactionArgs{GasPrice: fortytwo}, nil, }, + { + "legacy tx post-London with zero price", + true, + &TransactionArgs{GasPrice: zero}, + nil, + errors.New("gasPrice must be non-zero after london fork"), + }, // Access list txs { @@ -161,6 +176,13 @@ func TestSetFeeDefaults(t *testing.T) { nil, errors.New("maxFeePerGas (0x7) < maxPriorityFeePerGas (0x2a)"), }, + { + "dynamic fee tx post-London, explicit gas price", + true, + &TransactionArgs{MaxFeePerGas: zero, MaxPriorityFeePerGas: zero}, + nil, + errors.New("maxFeePerGas must be non-zero"), + }, // Misc { From cd58897f18fdb12c5a1d41f8e73612c0d296211f Mon Sep 17 00:00:00 2001 From: wangyifan Date: Mon, 18 Dec 2023 11:10:54 -0800 Subject: [PATCH 10/18] core/rawdb: implement size reporting for live items in freezer_table (#28525) This is the fix to issue #27483. A new hiddenBytes() is introduced to calculate the byte size of hidden items in the freezer table. When reporting the size of the freezer table, size of the hidden items will be subtracted from the total size. --------- Co-authored-by: Yifan Co-authored-by: Gary Rong --- core/rawdb/freezer_table.go | 39 ++++++++++++++++++++++++-------- core/rawdb/freezer_table_test.go | 33 +++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 61436bf932..4b9d510e82 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -467,6 +467,20 @@ func (t *freezerTable) truncateHead(items uint64) error { return nil } +// sizeHidden returns the total data size of hidden items in the freezer table. +// This function assumes the lock is already held. +func (t *freezerTable) sizeHidden() (uint64, error) { + hidden, offset := t.itemHidden.Load(), t.itemOffset.Load() + if hidden <= offset { + return 0, nil + } + indices, err := t.getIndices(hidden-1, 1) + if err != nil { + return 0, err + } + return uint64(indices[1].offset), nil +} + // truncateTail discards any recent data before the provided threshold number. func (t *freezerTable) truncateTail(items uint64) error { t.lock.Lock() @@ -495,6 +509,12 @@ func (t *freezerTable) truncateTail(items uint64) error { newTail.unmarshalBinary(buffer) newTailId = newTail.filenum } + // Save the old size for metrics tracking. This needs to be done + // before any updates to either itemHidden or itemOffset. + oldSize, err := t.sizeNolock() + if err != nil { + return err + } // Update the virtual tail marker and hidden these entries in table. t.itemHidden.Store(items) if err := writeMetadata(t.meta, newMetadata(items)); err != nil { @@ -509,18 +529,12 @@ func (t *freezerTable) truncateTail(items uint64) error { if t.tailId > newTailId { return fmt.Errorf("invalid index, tail-file %d, item-file %d", t.tailId, newTailId) } - // Hidden items exceed the current tail file, drop the relevant - // data files. We need to truncate, save the old size for metrics - // tracking. - oldSize, err := t.sizeNolock() - if err != nil { - return err - } // Count how many items can be deleted from the file. var ( newDeleted = items deleted = t.itemOffset.Load() ) + // Hidden items exceed the current tail file, drop the relevant data files. for current := items - 1; current >= deleted; current -= 1 { if _, err := t.index.ReadAt(buffer, int64((current-deleted+1)*indexEntrySize)); err != nil { return err @@ -680,6 +694,7 @@ func (t *freezerTable) releaseFilesBefore(num uint32, remove bool) { func (t *freezerTable) getIndices(from, count uint64) ([]*indexEntry, error) { // Apply the table-offset from = from - t.itemOffset.Load() + // For reading N items, we need N+1 indices. buffer := make([]byte, (count+1)*indexEntrySize) if _, err := t.index.ReadAt(buffer, int64(from*indexEntrySize)); err != nil { @@ -870,14 +885,18 @@ func (t *freezerTable) size() (uint64, error) { return t.sizeNolock() } -// sizeNolock returns the total data size in the freezer table without obtaining -// the mutex first. +// sizeNolock returns the total data size in the freezer table. This function +// assumes the lock is already held. func (t *freezerTable) sizeNolock() (uint64, error) { stat, err := t.index.Stat() if err != nil { return 0, err } - total := uint64(t.maxFileSize)*uint64(t.headId-t.tailId) + uint64(t.headBytes) + uint64(stat.Size()) + hidden, err := t.sizeHidden() + if err != nil { + return 0, err + } + total := uint64(t.maxFileSize)*uint64(t.headId-t.tailId) + uint64(t.headBytes) + uint64(stat.Size()) - hidden return total, nil } diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 939d093946..4471463932 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -658,6 +658,13 @@ func TestFreezerOffset(t *testing.T) { } } +func assertTableSize(t *testing.T, f *freezerTable, size int) { + t.Helper() + if got, err := f.size(); got != uint64(size) { + t.Fatalf("expected size of %d bytes, got %d, err: %v", size, got, err) + } +} + func TestTruncateTail(t *testing.T) { t.Parallel() rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() @@ -692,6 +699,9 @@ func TestTruncateTail(t *testing.T) { 5: getChunk(20, 0xaa), 6: getChunk(20, 0x11), }) + // maxFileSize*fileCount + headBytes + indexFileSize - hiddenBytes + expected := 20*7 + 48 - 0 + assertTableSize(t, f, expected) // truncate single element( item 0 ), deletion is only supported at file level f.truncateTail(1) @@ -707,6 +717,8 @@ func TestTruncateTail(t *testing.T) { 5: getChunk(20, 0xaa), 6: getChunk(20, 0x11), }) + expected = 20*7 + 48 - 20 + assertTableSize(t, f, expected) // Reopen the table, the deletion information should be persisted as well f.Close() @@ -739,6 +751,8 @@ func TestTruncateTail(t *testing.T) { 5: getChunk(20, 0xaa), 6: getChunk(20, 0x11), }) + expected = 20*5 + 36 - 0 + assertTableSize(t, f, expected) // Reopen the table, the above testing should still pass f.Close() @@ -760,6 +774,23 @@ func TestTruncateTail(t *testing.T) { 6: getChunk(20, 0x11), }) + // truncate 3 more elements( item 2, 3, 4), the file 1 should be deleted + // file 2 should only contain item 5 + f.truncateTail(5) + checkRetrieveError(t, f, map[uint64]error{ + 0: errOutOfBounds, + 1: errOutOfBounds, + 2: errOutOfBounds, + 3: errOutOfBounds, + 4: errOutOfBounds, + }) + checkRetrieve(t, f, map[uint64][]byte{ + 5: getChunk(20, 0xaa), + 6: getChunk(20, 0x11), + }) + expected = 20*3 + 24 - 20 + assertTableSize(t, f, expected) + // truncate all, the entire freezer should be deleted f.truncateTail(7) checkRetrieveError(t, f, map[uint64]error{ @@ -771,6 +802,8 @@ func TestTruncateTail(t *testing.T) { 5: errOutOfBounds, 6: errOutOfBounds, }) + expected = 12 + assertTableSize(t, f, expected) } func TestTruncateHead(t *testing.T) { From 952b343cb3d319b77076ef3acb60e29e04cd51fd Mon Sep 17 00:00:00 2001 From: Martin HS Date: Tue, 19 Dec 2023 08:55:04 +0100 Subject: [PATCH 11/18] build: make linter emit output (#28704) --- build/ci.go | 2 +- internal/build/util.go | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/build/ci.go b/build/ci.go index afe1c332b8..c272d3f2b9 100644 --- a/build/ci.go +++ b/build/ci.go @@ -366,7 +366,7 @@ func doLint(cmdline []string) { linter := downloadLinter(*cachedir) lflags := []string{"run", "--config", ".golangci.yml"} - build.MustRunCommand(linter, append(lflags, packages...)...) + build.MustRunCommandWithOutput(linter, append(lflags, packages...)...) fmt.Println("You have achieved perfection.") } diff --git a/internal/build/util.go b/internal/build/util.go index 5c77b236dc..17928118a0 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -68,6 +68,25 @@ func MustRunCommand(cmd string, args ...string) { MustRun(exec.Command(cmd, args...)) } +func MustRunCommandWithOutput(cmd string, args ...string) { + var done chan bool + // This is a little loop to generate some output, so CI does not tear down the + // process after 300 seconds. + go func() { + for i := 0; i < 15; i++ { + fmt.Printf("Waiting for command %q\n", cmd) + select { + case <-time.After(time.Minute): + break + case <-done: + return + } + } + }() + MustRun(exec.Command(cmd, args...)) + close(done) +} + var warnedAboutGit bool // RunGit runs a git subcommand and returns its output. From 5a9dda64ce17dda86720ed62d502831e5f616144 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Tue, 19 Dec 2023 09:24:23 +0100 Subject: [PATCH 12/18] .travis: set lower GOGC value (#28705) As documented on https://golangci-lint.run/usage/performance/ , a lower GOGC value causes less peak mem consumption when running the linter. Exceeding 3Gb is a common cause for build failures, according to https://docs.travis-ci.com/user/common-build-problems/#my-build-script-is-killed-without-any-error --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c2bfc3f2bf..40080dafab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ jobs: git: submodules: false # avoid cloning ethereum/tests script: - - go run build/ci.go lint + - GOGC=10 go run build/ci.go lint # These builders create the Docker sub-images for multi-arch push and each # will attempt to push the multi-arch image if they are the last builder From 435bed5da04a386198ca25c5e1264330c7a0da5b Mon Sep 17 00:00:00 2001 From: Martin HS Date: Tue, 19 Dec 2023 10:35:02 +0100 Subject: [PATCH 13/18] ci: disable lint on travis (#28706) --- .travis.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 40080dafab..a55583a703 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,18 +9,6 @@ jobs: - azure-osx include: - # This builder only tests code linters on latest version of Go - - stage: lint - os: linux - dist: bionic - go: 1.21.x - env: - - lint - git: - submodules: false # avoid cloning ethereum/tests - script: - - GOGC=10 go run build/ci.go lint - # These builders create the Docker sub-images for multi-arch push and each # will attempt to push the multi-arch image if they are the last builder - stage: build From 0cc192bd3a89cae6d3c2a787b9265dda631d6529 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 10:50:02 +0100 Subject: [PATCH 14/18] build(deps): bump golang.org/x/crypto from 0.15.0 to 0.17.0 (#28702) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.15.0 to 0.17.0. - [Commits](https://github.com/golang/crypto/compare/v0.15.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 8f99a00754..b4d077fc47 100644 --- a/go.mod +++ b/go.mod @@ -62,10 +62,10 @@ require ( github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v2 v2.25.7 go.uber.org/automaxprocs v1.5.2 - golang.org/x/crypto v0.15.0 + golang.org/x/crypto v0.17.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/sync v0.5.0 - golang.org/x/sys v0.14.0 + golang.org/x/sys v0.15.0 golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.15.0 diff --git a/go.sum b/go.sum index f89adbe571..bab51b1345 100644 --- a/go.sum +++ b/go.sum @@ -614,8 +614,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -774,8 +774,8 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 3fd568855f1e6d1370e61a30d10a4055ab682851 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 19 Dec 2023 13:25:03 +0100 Subject: [PATCH 15/18] params: go-ethereum v1.13.7 stable --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index ef3c47e7ac..5908849d9c 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 13 // Minor version component of the current release + VersionPatch = 7 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 9258a44b8f455d74f1c344bb82af39accb6c65aa Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 19 Dec 2023 13:32:25 +0100 Subject: [PATCH 16/18] params: begin go-ethereum v1.13.8 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 5908849d9c..a9192845bc 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 13 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 7124057bad16694d2b1f15dfe68a6109961b34ab Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 20 Dec 2023 07:56:41 +0100 Subject: [PATCH 17/18] internal/build: fix crash in MustRunCommandWithOutput (#28709) --- internal/build/util.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/internal/build/util.go b/internal/build/util.go index 17928118a0..82f9ba51a1 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -68,23 +68,18 @@ func MustRunCommand(cmd string, args ...string) { MustRun(exec.Command(cmd, args...)) } +// MustRunCommandWithOutput runs the given command, and ensures that some output will be +// printed while it runs. This is useful for CI builds where the process will be stopped +// when there is no output. func MustRunCommandWithOutput(cmd string, args ...string) { - var done chan bool - // This is a little loop to generate some output, so CI does not tear down the - // process after 300 seconds. + interval := time.NewTicker(time.Minute) + defer interval.Stop() go func() { - for i := 0; i < 15; i++ { + for range interval.C { fmt.Printf("Waiting for command %q\n", cmd) - select { - case <-time.After(time.Minute): - break - case <-done: - return - } } }() MustRun(exec.Command(cmd, args...)) - close(done) } var warnedAboutGit bool From d3452a22cc871306c62de52d19295914141863c0 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Wed, 20 Dec 2023 13:41:40 +0100 Subject: [PATCH 18/18] accounts: properly close managed wallets when closing manager (#28710) --- accounts/manager.go | 3 +++ accounts/usbwallet/wallet.go | 4 ++++ cmd/clef/main.go | 1 + 3 files changed, 8 insertions(+) diff --git a/accounts/manager.go b/accounts/manager.go index a0b5c329cd..cbe4f7c79d 100644 --- a/accounts/manager.go +++ b/accounts/manager.go @@ -98,6 +98,9 @@ func NewManager(config *Config, backends ...Backend) *Manager { // Close terminates the account manager's internal notification processes. func (am *Manager) Close() error { + for _, w := range am.wallets { + w.Close() + } errc := make(chan error) am.quit <- errc return <-errc diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 05add081ab..69083dc893 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -483,6 +483,10 @@ func (w *wallet) Derive(path accounts.DerivationPath, pin bool) (accounts.Accoun w.stateLock.Lock() defer w.stateLock.Unlock() + if w.device == nil { + return accounts.Account{}, accounts.ErrWalletClosed + } + if _, ok := w.paths[address]; !ok { w.accounts = append(w.accounts, account) w.paths[address] = make(accounts.DerivationPath, len(path)) diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 2346991369..f9b00e4a12 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -704,6 +704,7 @@ func signer(c *cli.Context) error { log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc, "light-kdf", lightKdf, "advanced", advanced) am := core.StartClefAccountManager(ksLoc, nousb, lightKdf, scpath) + defer am.Close() apiImpl := core.NewSignerAPI(am, chainId, nousb, ui, db, advanced, pwStorage) // Establish the bidirectional communication, by creating a new UI backend and registering