From 804d45cc2eddd92ee4b9940875e4a6914ec5256f Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 13 Dec 2024 08:46:12 -0300 Subject: [PATCH 1/5] p2p: DNS resolution for static nodes (#30822) Closes #23210 # Context When deploying Geth in Kubernetes with ReplicaSets, we encountered two DNS-related issues affecting node connectivity. First, during startup, Geth tries to resolve DNS names for static nodes too early in the config unmarshaling phase. If peer nodes aren't ready yet (which is common in Kubernetes rolling deployments), this causes an immediate failure: ``` INFO [11-26|10:03:42.816] Starting Geth on Ethereum mainnet... INFO [11-26|10:03:42.817] Bumping default cache on mainnet provided=1024 updated=4096 Fatal: config.toml, line 81: (p2p.Config.StaticNodes) lookup idontexist.geth.node: no such host ``` The second issue comes up when pods get rescheduled to different nodes - their IPs change but peers keep using the initially resolved IP, never updating the DNS mapping. This PR adds proper DNS support for enode:// URLs by deferring resolution to connection time. It also handles DNS failures gracefully instead of failing fatally during startup, making it work better in container environments where IPs are dynamic and peers come and go during rollouts. --------- Co-authored-by: Felix Lange --- p2p/dial.go | 143 ++++++++++++++++++++++++++++++++-------- p2p/dial_test.go | 29 ++++++++ p2p/enode/node.go | 22 +++++++ p2p/enode/node_test.go | 19 ++++++ p2p/enode/urlv4.go | 38 ++++++----- p2p/enode/urlv4_test.go | 23 +++---- p2p/peer_test.go | 30 ++++----- 7 files changed, 228 insertions(+), 76 deletions(-) diff --git a/p2p/dial.go b/p2p/dial.go index 24d4dc2e89..225709427c 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -24,6 +24,7 @@ import ( "fmt" mrand "math/rand" "net" + "net/netip" "sync" "sync/atomic" "time" @@ -31,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/netutil" ) @@ -77,6 +79,7 @@ var ( errRecentlyDialed = errors.New("recently dialed") errNetRestrict = errors.New("not contained in netrestrict list") errNoPort = errors.New("node does not provide TCP port") + errNoResolvedIP = errors.New("node does not provide a resolved IP") ) // dialer creates outbound connections and submits them into Server. @@ -90,16 +93,17 @@ var ( // to create peer connections to nodes arriving through the iterator. type dialScheduler struct { dialConfig - setupFunc dialSetupFunc - wg sync.WaitGroup - cancel context.CancelFunc - ctx context.Context - nodesIn chan *enode.Node - doneCh chan *dialTask - addStaticCh chan *enode.Node - remStaticCh chan *enode.Node - addPeerCh chan *conn - remPeerCh chan *conn + setupFunc dialSetupFunc + dnsLookupFunc func(ctx context.Context, network string, name string) ([]netip.Addr, error) + wg sync.WaitGroup + cancel context.CancelFunc + ctx context.Context + nodesIn chan *enode.Node + doneCh chan *dialTask + addStaticCh chan *enode.Node + remStaticCh chan *enode.Node + addPeerCh chan *conn + remPeerCh chan *conn // Everything below here belongs to loop and // should only be accessed by code on the loop goroutine. @@ -159,18 +163,19 @@ func (cfg dialConfig) withDefaults() dialConfig { func newDialScheduler(config dialConfig, it enode.Iterator, setupFunc dialSetupFunc) *dialScheduler { cfg := config.withDefaults() d := &dialScheduler{ - dialConfig: cfg, - historyTimer: mclock.NewAlarm(cfg.clock), - setupFunc: setupFunc, - dialing: make(map[enode.ID]*dialTask), - static: make(map[enode.ID]*dialTask), - peers: make(map[enode.ID]struct{}), - doneCh: make(chan *dialTask), - nodesIn: make(chan *enode.Node), - addStaticCh: make(chan *enode.Node), - remStaticCh: make(chan *enode.Node), - addPeerCh: make(chan *conn), - remPeerCh: make(chan *conn), + dialConfig: cfg, + historyTimer: mclock.NewAlarm(cfg.clock), + setupFunc: setupFunc, + dnsLookupFunc: net.DefaultResolver.LookupNetIP, + dialing: make(map[enode.ID]*dialTask), + static: make(map[enode.ID]*dialTask), + peers: make(map[enode.ID]struct{}), + doneCh: make(chan *dialTask), + nodesIn: make(chan *enode.Node), + addStaticCh: make(chan *enode.Node), + remStaticCh: make(chan *enode.Node), + addPeerCh: make(chan *conn), + remPeerCh: make(chan *conn), } d.lastStatsLog = d.clock.Now() d.ctx, d.cancel = context.WithCancel(context.Background()) @@ -274,7 +279,7 @@ loop: case node := <-d.addStaticCh: id := node.ID() _, exists := d.static[id] - d.log.Trace("Adding static node", "id", id, "ip", node.IPAddr(), "added", !exists) + d.log.Trace("Adding static node", "id", id, "endpoint", nodeEndpointForLog(node), "added", !exists) if exists { continue loop } @@ -433,10 +438,68 @@ func (d *dialScheduler) removeFromStaticPool(idx int) { task.staticPoolIndex = -1 } +// dnsResolveHostname updates the given node from its DNS hostname. +// This is used to resolve static dial targets. +func (d *dialScheduler) dnsResolveHostname(n *enode.Node) (*enode.Node, error) { + if n.Hostname() == "" { + return n, nil + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + foundIPs, err := d.dnsLookupFunc(ctx, "ip", n.Hostname()) + if err != nil { + return n, err + } + + // Check for IP updates. + var ( + nodeIP4, nodeIP6 netip.Addr + foundIP4, foundIP6 netip.Addr + ) + n.Load((*enr.IPv4Addr)(&nodeIP4)) + n.Load((*enr.IPv6Addr)(&nodeIP6)) + for _, ip := range foundIPs { + if ip.Is4() && !foundIP4.IsValid() { + foundIP4 = ip + } + if ip.Is6() && !foundIP6.IsValid() { + foundIP6 = ip + } + } + + if !foundIP4.IsValid() && !foundIP6.IsValid() { + // Lookup failed. + return n, errNoResolvedIP + } + if foundIP4 == nodeIP4 && foundIP6 == nodeIP6 { + // No updates necessary. + d.log.Trace("Node DNS lookup had no update", "id", n.ID(), "name", n.Hostname(), "ip", foundIP4, "ip6", foundIP6) + return n, nil + } + + // Update the node. Note this invalidates the ENR signature, because we use SignNull + // to create a modified copy. But this should be OK, since we just use the node as a + // dial target. And nodes will usually only have a DNS hostname if they came from a + // enode:// URL, which has no signature anyway. If it ever becomes a problem, the + // resolved IP could also be stored into dialTask instead of the node. + rec := n.Record() + if foundIP4.IsValid() { + rec.Set(enr.IPv4Addr(foundIP4)) + } + if foundIP6.IsValid() { + rec.Set(enr.IPv6Addr(foundIP6)) + } + rec.SetSeq(n.Seq()) // ensure seq not bumped by update + newNode := enode.SignNull(rec, n.ID()).WithHostname(n.Hostname()) + d.log.Debug("Node updated from DNS lookup", "id", n.ID(), "name", n.Hostname(), "ip", newNode.IP()) + return newNode, nil +} + // startDial runs the given dial task in a separate goroutine. func (d *dialScheduler) startDial(task *dialTask) { node := task.dest() - d.log.Trace("Starting p2p dial", "id", node.ID(), "ip", node.IPAddr(), "flag", task.flags) + d.log.Trace("Starting p2p dial", "id", node.ID(), "endpoint", nodeEndpointForLog(node), "flag", task.flags) hkey := string(node.ID().Bytes()) d.history.add(hkey, d.clock.Now().Add(dialHistoryExpiration)) d.dialing[node.ID()] = task @@ -473,14 +536,29 @@ func (t *dialTask) dest() *enode.Node { } func (t *dialTask) run(d *dialScheduler) { - if t.needResolve() && !t.resolve(d) { - return + if t.isStatic() { + // Resolve DNS. + if n := t.dest(); n.Hostname() != "" { + resolved, err := d.dnsResolveHostname(n) + if err != nil { + d.log.Warn("DNS lookup of static node failed", "id", n.ID(), "name", n.Hostname(), "err", err) + } else { + t.destPtr.Store(resolved) + } + } + // Try resolving node ID through the DHT if there is no IP address. + if !t.dest().IPAddr().IsValid() { + if !t.resolve(d) { + return // DHT resolve failed, skip dial. + } + } } err := t.dial(d, t.dest()) if err != nil { // For static nodes, resolve one more time if dialing fails. - if _, ok := err.(*dialError); ok && t.flags&staticDialedConn != 0 { + var dialErr *dialError + if errors.As(err, &dialErr) && t.isStatic() { if t.resolve(d) { t.dial(d, t.dest()) } @@ -488,8 +566,8 @@ func (t *dialTask) run(d *dialScheduler) { } } -func (t *dialTask) needResolve() bool { - return t.flags&staticDialedConn != 0 && !t.dest().IPAddr().IsValid() +func (t *dialTask) isStatic() bool { + return t.flags&staticDialedConn != 0 } // resolve attempts to find the current endpoint for the destination @@ -553,3 +631,10 @@ func cleanupDialErr(err error) error { } return err } + +func nodeEndpointForLog(n *enode.Node) string { + if n.Hostname() != "" { + return n.Hostname() + } + return n.IPAddr().String() +} diff --git a/p2p/dial_test.go b/p2p/dial_test.go index 13908f11ea..f18dacce2a 100644 --- a/p2p/dial_test.go +++ b/p2p/dial_test.go @@ -22,6 +22,7 @@ import ( "fmt" "math/rand" "net" + "net/netip" "reflect" "sync" "testing" @@ -394,6 +395,34 @@ func TestDialSchedResolve(t *testing.T) { }) } +func TestDialSchedDNSHostname(t *testing.T) { + t.Parallel() + + config := dialConfig{ + maxActiveDials: 1, + maxDialPeers: 1, + } + node := newNode(uintID(0x01), ":30303").WithHostname("node-hostname") + resolved := newNode(uintID(0x01), "1.2.3.4:30303").WithHostname("node-hostname") + runDialTest(t, config, []dialTestRound{ + { + update: func(d *dialScheduler) { + d.dnsLookupFunc = func(ctx context.Context, network string, name string) ([]netip.Addr, error) { + if name != "node-hostname" { + t.Error("wrong hostname in DNS lookup:", name) + } + result := []netip.Addr{netip.MustParseAddr("1.2.3.4")} + return result, nil + } + d.addStatic(node) + }, + wantNewDials: []*enode.Node{ + resolved, + }, + }, + }) +} + // ------- // Code below here is the framework for the tests above. diff --git a/p2p/enode/node.go b/p2p/enode/node.go index 4d93d3f6be..d6f2ac7ff5 100644 --- a/p2p/enode/node.go +++ b/p2p/enode/node.go @@ -37,6 +37,10 @@ var errMissingPrefix = errors.New("missing 'enr:' prefix for base64-encoded reco type Node struct { r enr.Record id ID + + // hostname tracks the DNS name of the node. + hostname string + // endpoint information ip netip.Addr udp uint16 @@ -77,6 +81,8 @@ func newNodeWithID(r *enr.Record, id ID) *Node { n.setIP4(ip4) case valid6: n.setIP6(ip6) + default: + n.setIPv4Ports() } return n } @@ -103,6 +109,10 @@ func localityScore(ip netip.Addr) int { func (n *Node) setIP4(ip netip.Addr) { n.ip = ip + n.setIPv4Ports() +} + +func (n *Node) setIPv4Ports() { n.Load((*enr.UDP)(&n.udp)) n.Load((*enr.TCP)(&n.tcp)) } @@ -184,6 +194,18 @@ func (n *Node) TCP() int { return int(n.tcp) } +// WithHostname adds a DNS hostname to the node. +func (n *Node) WithHostname(hostname string) *Node { + cpy := *n + cpy.hostname = hostname + return &cpy +} + +// Hostname returns the DNS name assigned by WithHostname. +func (n *Node) Hostname() string { + return n.hostname +} + // UDPEndpoint returns the announced UDP endpoint. func (n *Node) UDPEndpoint() (netip.AddrPort, bool) { if !n.ip.IsValid() || n.ip.IsUnspecified() || n.udp == 0 { diff --git a/p2p/enode/node_test.go b/p2p/enode/node_test.go index f38c77415e..e9fe631f34 100644 --- a/p2p/enode/node_test.go +++ b/p2p/enode/node_test.go @@ -74,6 +74,7 @@ func TestNodeEndpoints(t *testing.T) { wantUDP int wantTCP int wantQUIC int + wantDNS string } tests := []endpointTest{ { @@ -90,6 +91,7 @@ func TestNodeEndpoints(t *testing.T) { r.Set(enr.UDP(9000)) return SignNull(&r, id) }(), + wantUDP: 9000, }, { name: "tcp-only", @@ -98,6 +100,7 @@ func TestNodeEndpoints(t *testing.T) { r.Set(enr.TCP(9000)) return SignNull(&r, id) }(), + wantTCP: 9000, }, { name: "quic-only", @@ -268,6 +271,19 @@ func TestNodeEndpoints(t *testing.T) { wantIP: netip.MustParseAddr("2001::ff00:0042:8329"), wantQUIC: 9001, }, + { + name: "dns-only", + node: func() *Node { + var r enr.Record + r.Set(enr.UDP(30303)) + r.Set(enr.TCP(30303)) + n := SignNull(&r, id).WithHostname("example.com") + return n + }(), + wantTCP: 30303, + wantUDP: 30303, + wantDNS: "example.com", + }, } for _, test := range tests { @@ -284,6 +300,9 @@ func TestNodeEndpoints(t *testing.T) { if quic, _ := test.node.QUICEndpoint(); test.wantQUIC != int(quic.Port()) { t.Errorf("node has wrong QUIC port %d, want %d", quic.Port(), test.wantQUIC) } + if test.wantDNS != test.node.Hostname() { + t.Errorf("node has wrong DNS name %s, want %s", test.node.Hostname(), test.wantDNS) + } }) } } diff --git a/p2p/enode/urlv4.go b/p2p/enode/urlv4.go index a55dfa6632..b455cd4533 100644 --- a/p2p/enode/urlv4.go +++ b/p2p/enode/urlv4.go @@ -33,7 +33,6 @@ import ( var ( incompleteNodeURL = regexp.MustCompile("(?i)^(?:enode://)?([0-9a-f]+)$") - lookupIPFunc = net.LookupIP ) // MustParseV4 parses a node URL. It panics if the URL is not valid. @@ -126,20 +125,9 @@ func parseComplete(rawurl string) (*Node, error) { if id, err = parsePubkey(u.User.String()); err != nil { return nil, fmt.Errorf("invalid public key (%v)", err) } - // Parse the IP address. + + // Parse the IP and ports. ip := net.ParseIP(u.Hostname()) - if ip == nil { - ips, err := lookupIPFunc(u.Hostname()) - if err != nil { - return nil, err - } - ip = ips[0] - } - // Ensure the IP is 4 bytes long for IPv4 addresses. - if ipv4 := ip.To4(); ipv4 != nil { - ip = ipv4 - } - // Parse the port numbers. if tcpPort, err = strconv.ParseUint(u.Port(), 10, 16); err != nil { return nil, errors.New("invalid port") } @@ -151,7 +139,13 @@ func parseComplete(rawurl string) (*Node, error) { return nil, errors.New("invalid discport in query") } } - return NewV4(id, ip, int(tcpPort), int(udpPort)), nil + + // Create the node. + node := NewV4(id, ip, int(tcpPort), int(udpPort)) + if ip == nil && u.Hostname() != "" { + node = node.WithHostname(u.Hostname()) + } + return node, nil } // parsePubkey parses a hex-encoded secp256k1 public key. @@ -181,15 +175,23 @@ func (n *Node) URLv4() string { nodeid = fmt.Sprintf("%s.%x", scheme, n.id[:]) } u := url.URL{Scheme: "enode"} - if !n.ip.IsValid() { - u.Host = nodeid - } else { + if n.Hostname() != "" { + // For nodes with a DNS name: include DNS name, TCP port, and optional UDP port + u.User = url.User(nodeid) + u.Host = fmt.Sprintf("%s:%d", n.Hostname(), n.TCP()) + if n.UDP() != n.TCP() { + u.RawQuery = "discport=" + strconv.Itoa(n.UDP()) + } + } else if n.ip.IsValid() { + // For IP-based nodes: include IP address, TCP port, and optional UDP port addr := net.TCPAddr{IP: n.IP(), Port: n.TCP()} u.User = url.User(nodeid) u.Host = addr.String() if n.UDP() != n.TCP() { u.RawQuery = "discport=" + strconv.Itoa(n.UDP()) } + } else { + u.Host = nodeid } return u.String() } diff --git a/p2p/enode/urlv4_test.go b/p2p/enode/urlv4_test.go index 33de96cc57..f39d5a2deb 100644 --- a/p2p/enode/urlv4_test.go +++ b/p2p/enode/urlv4_test.go @@ -18,7 +18,6 @@ package enode import ( "crypto/ecdsa" - "errors" "net" "reflect" "strings" @@ -28,15 +27,6 @@ import ( "github.com/ethereum/go-ethereum/p2p/enr" ) -func init() { - lookupIPFunc = func(name string) ([]net.IP, error) { - if name == "node.example.org" { - return []net.IP{{33, 44, 55, 66}}, nil - } - return nil, errors.New("no such host") - } -} - var parseNodeTests = []struct { input string wantError string @@ -70,10 +60,6 @@ var parseNodeTests = []struct { wantError: enr.ErrInvalidSig.Error(), }, // Complete node URLs with IP address and ports - { - input: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@invalid.:3", - wantError: `no such host`, - }, { input: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@127.0.0.1:foo", wantError: `invalid port`, @@ -91,6 +77,15 @@ var parseNodeTests = []struct { 52150, ), }, + { + input: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@valid.:3", + wantResult: NewV4( + hexPubkey("1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), + nil, + 3, + 3, + ).WithHostname("valid."), + }, { input: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@[::]:52150", wantResult: NewV4( diff --git a/p2p/peer_test.go b/p2p/peer_test.go index 4308bbd2eb..dea72875fe 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -60,22 +60,22 @@ func uintID(i uint16) enode.ID { // newNode creates a node record with the given address. func newNode(id enode.ID, addr string) *enode.Node { var r enr.Record - if addr != "" { - // Set the port if present. - if strings.Contains(addr, ":") { - hs, ps, err := net.SplitHostPort(addr) - if err != nil { - panic(fmt.Errorf("invalid address %q", addr)) - } - port, err := strconv.Atoi(ps) - if err != nil { - panic(fmt.Errorf("invalid port in %q", addr)) - } - r.Set(enr.TCP(port)) - r.Set(enr.UDP(port)) - addr = hs + // Set the port if present. + if strings.Contains(addr, ":") { + hs, ps, err := net.SplitHostPort(addr) + if err != nil { + panic(fmt.Errorf("invalid address %q", addr)) } - // Set the IP. + port, err := strconv.Atoi(ps) + if err != nil { + panic(fmt.Errorf("invalid port in %q", addr)) + } + r.Set(enr.TCP(port)) + r.Set(enr.UDP(port)) + addr = hs + } + // Set the IP. + if addr != "" { ip := net.ParseIP(addr) if ip == nil { panic(fmt.Errorf("invalid IP %q", addr)) From f808d7357ed4076b224a8c6fe47893ce022f9409 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Mon, 16 Dec 2024 03:29:37 -0700 Subject: [PATCH 2/5] all: implement eip-7702 set code tx (#30078) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR implements EIP-7702: "Set EOA account code". Specification: https://eips.ethereum.org/EIPS/eip-7702 > Add a new transaction type that adds a list of `[chain_id, address, nonce, y_parity, r, s]` authorization tuples. For each tuple, write a delegation designator `(0xef0100 ++ address)` to the signing account’s code. All code reading operations must load the code pointed to by the designator. --------- Co-authored-by: Mario Vega Co-authored-by: Martin Holst Swende Co-authored-by: Felix Lange --- accounts/external/backend.go | 2 +- cmd/evm/eofparse.go | 2 +- cmd/evm/eofparse_test.go | 2 +- cmd/evm/internal/t8ntool/execution.go | 4 +- cmd/evm/internal/t8ntool/gen_execresult.go | 4 +- cmd/evm/internal/t8ntool/transaction.go | 2 +- cmd/evm/t8n_test.go | 8 + cmd/evm/testdata/1/exp.json | 3 +- cmd/evm/testdata/13/exp2.json | 3 +- cmd/evm/testdata/14/exp.json | 3 +- cmd/evm/testdata/14/exp2.json | 3 +- cmd/evm/testdata/14/exp_berlin.json | 3 +- cmd/evm/testdata/19/exp_arrowglacier.json | 3 +- cmd/evm/testdata/19/exp_grayglacier.json | 25 +- cmd/evm/testdata/19/exp_london.json | 3 +- cmd/evm/testdata/23/exp.json | 3 +- cmd/evm/testdata/24/exp.json | 3 +- cmd/evm/testdata/25/exp.json | 3 +- cmd/evm/testdata/26/exp.json | 3 +- cmd/evm/testdata/28/exp.json | 3 +- cmd/evm/testdata/29/exp.json | 3 +- cmd/evm/testdata/3/exp.json | 3 +- cmd/evm/testdata/30/exp.json | 5 +- cmd/evm/testdata/33/README.md | 1 + cmd/evm/testdata/33/alloc.json | 30 +++ cmd/evm/testdata/33/env.json | 14 ++ cmd/evm/testdata/33/exp.json | 62 +++++ cmd/evm/testdata/33/txs.json | 37 +++ cmd/evm/testdata/5/exp.json | 3 +- core/bench_test.go | 2 +- core/blockchain_test.go | 94 ++++++++ core/error.go | 18 ++ core/state/journal.go | 19 +- core/state/state_object.go | 7 +- core/state/statedb.go | 5 +- core/state/statedb_hooked.go | 11 +- core/state_processor_test.go | 25 +- core/state_transition.go | 107 ++++++++- core/txpool/validation.go | 2 +- core/types/gen_authorization.go | 75 ++++++ core/types/receipt.go | 4 +- core/types/transaction.go | 12 + core/types/transaction_marshalling.go | 92 +++++++ core/types/transaction_signing.go | 77 +++++- core/types/tx_setcode.go | 226 ++++++++++++++++++ core/types/tx_setcode_test.go | 70 ++++++ core/verkle_witness_test.go | 4 +- core/vm/eips.go | 68 ++++++ core/vm/eof_validation_test.go | 16 +- core/vm/evm.go | 39 ++- core/vm/interface.go | 4 +- core/vm/interpreter.go | 2 + core/vm/jump_table.go | 17 +- core/vm/operations_acl.go | 68 ++++++ core/vm/runtime/runtime_test.go | 80 +++++++ .../testdata/prestate_tracer/setcode_tx.json | 82 +++++++ eth/tracers/native/prestate.go | 9 + graphql/graphql.go | 4 +- internal/ethapi/api.go | 61 +++-- internal/ethapi/api_test.go | 23 +- internal/ethapi/transaction_args.go | 27 +++ params/protocol_params.go | 9 +- tests/gen_stauthorization.go | 74 ++++++ tests/gen_sttransaction.go | 6 + tests/state_test_util.go | 37 +++ tests/transaction_test_util.go | 2 +- 66 files changed, 1606 insertions(+), 115 deletions(-) create mode 100644 cmd/evm/testdata/33/README.md create mode 100644 cmd/evm/testdata/33/alloc.json create mode 100644 cmd/evm/testdata/33/env.json create mode 100644 cmd/evm/testdata/33/exp.json create mode 100644 cmd/evm/testdata/33/txs.json create mode 100644 core/types/gen_authorization.go create mode 100644 core/types/tx_setcode.go create mode 100644 core/types/tx_setcode_test.go create mode 100644 eth/tracers/internal/tracetest/testdata/prestate_tracer/setcode_tx.json create mode 100644 tests/gen_stauthorization.go diff --git a/accounts/external/backend.go b/accounts/external/backend.go index 62322753da..42eaf661cc 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -215,7 +215,7 @@ func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transactio switch tx.Type() { case types.LegacyTxType, types.AccessListTxType: args.GasPrice = (*hexutil.Big)(tx.GasPrice()) - case types.DynamicFeeTxType, types.BlobTxType: + case types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType: args.MaxFeePerGas = (*hexutil.Big)(tx.GasFeeCap()) args.MaxPriorityFeePerGas = (*hexutil.Big)(tx.GasTipCap()) default: diff --git a/cmd/evm/eofparse.go b/cmd/evm/eofparse.go index df8581146a..92182a53b3 100644 --- a/cmd/evm/eofparse.go +++ b/cmd/evm/eofparse.go @@ -36,7 +36,7 @@ var jt vm.JumpTable const initcode = "INITCODE" func init() { - jt = vm.NewPragueEOFInstructionSetForTesting() + jt = vm.NewEOFInstructionSetForTesting() } var ( diff --git a/cmd/evm/eofparse_test.go b/cmd/evm/eofparse_test.go index 9b17039f5b..cda4b38fc9 100644 --- a/cmd/evm/eofparse_test.go +++ b/cmd/evm/eofparse_test.go @@ -43,7 +43,7 @@ func FuzzEofParsing(f *testing.F) { // And do the fuzzing f.Fuzz(func(t *testing.T, data []byte) { var ( - jt = vm.NewPragueEOFInstructionSetForTesting() + jt = vm.NewEOFInstructionSetForTesting() c vm.Container ) cpy := common.CopyBytes(data) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 7a0de86a11..aef497885e 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -70,11 +70,11 @@ type ExecutionResult struct { CurrentExcessBlobGas *math.HexOrDecimal64 `json:"currentExcessBlobGas,omitempty"` CurrentBlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed,omitempty"` RequestsHash *common.Hash `json:"requestsHash,omitempty"` - Requests [][]byte `json:"requests,omitempty"` + Requests [][]byte `json:"requests"` } type executionResultMarshaling struct { - Requests []hexutil.Bytes `json:"requests,omitempty"` + Requests []hexutil.Bytes `json:"requests"` } type ommer struct { diff --git a/cmd/evm/internal/t8ntool/gen_execresult.go b/cmd/evm/internal/t8ntool/gen_execresult.go index 0da94f5ca2..38310b9f2b 100644 --- a/cmd/evm/internal/t8ntool/gen_execresult.go +++ b/cmd/evm/internal/t8ntool/gen_execresult.go @@ -31,7 +31,7 @@ func (e ExecutionResult) MarshalJSON() ([]byte, error) { CurrentExcessBlobGas *math.HexOrDecimal64 `json:"currentExcessBlobGas,omitempty"` CurrentBlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed,omitempty"` RequestsHash *common.Hash `json:"requestsHash,omitempty"` - Requests []hexutil.Bytes `json:"requests,omitempty"` + Requests []hexutil.Bytes `json:"requests"` } var enc ExecutionResult enc.StateRoot = e.StateRoot @@ -74,7 +74,7 @@ func (e *ExecutionResult) UnmarshalJSON(input []byte) error { CurrentExcessBlobGas *math.HexOrDecimal64 `json:"currentExcessBlobGas,omitempty"` CurrentBlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed,omitempty"` RequestsHash *common.Hash `json:"requestsHash,omitempty"` - Requests []hexutil.Bytes `json:"requests,omitempty"` + Requests []hexutil.Bytes `json:"requests"` } var dec ExecutionResult if err := json.Unmarshal(input, &dec); err != nil { diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 7f66ba4d85..64e21b24fd 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -133,7 +133,7 @@ func Transaction(ctx *cli.Context) error { r.Address = sender } // Check intrinsic gas - if gas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.To() == nil, + if gas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.AuthList(), tx.To() == nil, chainConfig.IsHomestead(new(big.Int)), chainConfig.IsIstanbul(new(big.Int)), chainConfig.IsShanghai(new(big.Int), 0)); err != nil { r.Error = err results = append(results, r) diff --git a/cmd/evm/t8n_test.go b/cmd/evm/t8n_test.go index 85caf010b3..27c6c43164 100644 --- a/cmd/evm/t8n_test.go +++ b/cmd/evm/t8n_test.go @@ -287,6 +287,14 @@ func TestT8n(t *testing.T) { output: t8nOutput{alloc: true, result: true}, expOut: "exp.json", }, + { // Prague test, EIP-7702 transaction + base: "./testdata/33", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "Prague", "", + }, + output: t8nOutput{alloc: true, result: true}, + expOut: "exp.json", + }, } { args := []string{"t8n"} args = append(args, tc.output.get()...) diff --git a/cmd/evm/testdata/1/exp.json b/cmd/evm/testdata/1/exp.json index d1351e5b76..50662f35ea 100644 --- a/cmd/evm/testdata/1/exp.json +++ b/cmd/evm/testdata/1/exp.json @@ -40,6 +40,7 @@ } ], "currentDifficulty": "0x20000", - "gasUsed": "0x5208" + "gasUsed": "0x5208", + "requests": null } } diff --git a/cmd/evm/testdata/13/exp2.json b/cmd/evm/testdata/13/exp2.json index babce35929..6415a4f1f4 100644 --- a/cmd/evm/testdata/13/exp2.json +++ b/cmd/evm/testdata/13/exp2.json @@ -37,6 +37,7 @@ ], "currentDifficulty": "0x20000", "gasUsed": "0x109a0", - "currentBaseFee": "0x36b" + "currentBaseFee": "0x36b", + "requests": null } } diff --git a/cmd/evm/testdata/14/exp.json b/cmd/evm/testdata/14/exp.json index 26d49173ce..300217ee34 100644 --- a/cmd/evm/testdata/14/exp.json +++ b/cmd/evm/testdata/14/exp.json @@ -8,6 +8,7 @@ "currentDifficulty": "0x2000020000000", "receipts": [], "gasUsed": "0x0", - "currentBaseFee": "0x500" + "currentBaseFee": "0x500", + "requests": null } } diff --git a/cmd/evm/testdata/14/exp2.json b/cmd/evm/testdata/14/exp2.json index cd75b47d5a..6d139e371d 100644 --- a/cmd/evm/testdata/14/exp2.json +++ b/cmd/evm/testdata/14/exp2.json @@ -8,6 +8,7 @@ "receipts": [], "currentDifficulty": "0x1ff8020000000", "gasUsed": "0x0", - "currentBaseFee": "0x500" + "currentBaseFee": "0x500", + "requests": null } } diff --git a/cmd/evm/testdata/14/exp_berlin.json b/cmd/evm/testdata/14/exp_berlin.json index 5c00ef130a..c952d0f517 100644 --- a/cmd/evm/testdata/14/exp_berlin.json +++ b/cmd/evm/testdata/14/exp_berlin.json @@ -8,6 +8,7 @@ "receipts": [], "currentDifficulty": "0x1ff9000000000", "gasUsed": "0x0", - "currentBaseFee": "0x500" + "currentBaseFee": "0x500", + "requests": null } } diff --git a/cmd/evm/testdata/19/exp_arrowglacier.json b/cmd/evm/testdata/19/exp_arrowglacier.json index dd49f7d02e..0822fcc290 100644 --- a/cmd/evm/testdata/19/exp_arrowglacier.json +++ b/cmd/evm/testdata/19/exp_arrowglacier.json @@ -8,6 +8,7 @@ "currentDifficulty": "0x2000000200000", "receipts": [], "gasUsed": "0x0", - "currentBaseFee": "0x500" + "currentBaseFee": "0x500", + "requests": null } } diff --git a/cmd/evm/testdata/19/exp_grayglacier.json b/cmd/evm/testdata/19/exp_grayglacier.json index 86fd8e6c13..e80c9eb000 100644 --- a/cmd/evm/testdata/19/exp_grayglacier.json +++ b/cmd/evm/testdata/19/exp_grayglacier.json @@ -1,13 +1,14 @@ { - "result": { - "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", - "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "receipts": [], - "currentDifficulty": "0x2000000004000", - "gasUsed": "0x0", - "currentBaseFee": "0x500" - } -} \ No newline at end of file + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": "0x2000000004000", + "gasUsed": "0x0", + "currentBaseFee": "0x500", + "requests": null + } +} diff --git a/cmd/evm/testdata/19/exp_london.json b/cmd/evm/testdata/19/exp_london.json index 9e9a17da90..a17f684484 100644 --- a/cmd/evm/testdata/19/exp_london.json +++ b/cmd/evm/testdata/19/exp_london.json @@ -8,6 +8,7 @@ "currentDifficulty": "0x2000080000000", "receipts": [], "gasUsed": "0x0", - "currentBaseFee": "0x500" + "currentBaseFee": "0x500", + "requests": null } } diff --git a/cmd/evm/testdata/23/exp.json b/cmd/evm/testdata/23/exp.json index 22dde0a27c..7f36165e35 100644 --- a/cmd/evm/testdata/23/exp.json +++ b/cmd/evm/testdata/23/exp.json @@ -21,6 +21,7 @@ } ], "currentDifficulty": "0x20000", - "gasUsed": "0x520b" + "gasUsed": "0x520b", + "requests": null } } diff --git a/cmd/evm/testdata/24/exp.json b/cmd/evm/testdata/24/exp.json index ac571d149b..8f380c662b 100644 --- a/cmd/evm/testdata/24/exp.json +++ b/cmd/evm/testdata/24/exp.json @@ -51,6 +51,7 @@ ], "currentDifficulty": null, "gasUsed": "0x10306", - "currentBaseFee": "0x500" + "currentBaseFee": "0x500", + "requests": null } } diff --git a/cmd/evm/testdata/25/exp.json b/cmd/evm/testdata/25/exp.json index 1cb521794c..a674633762 100644 --- a/cmd/evm/testdata/25/exp.json +++ b/cmd/evm/testdata/25/exp.json @@ -34,6 +34,7 @@ ], "currentDifficulty": null, "gasUsed": "0x5208", - "currentBaseFee": "0x460" + "currentBaseFee": "0x460", + "requests": null } } diff --git a/cmd/evm/testdata/26/exp.json b/cmd/evm/testdata/26/exp.json index 4815e5cb65..d6275270f0 100644 --- a/cmd/evm/testdata/26/exp.json +++ b/cmd/evm/testdata/26/exp.json @@ -15,6 +15,7 @@ "currentDifficulty": null, "gasUsed": "0x0", "currentBaseFee": "0x500", - "withdrawalsRoot": "0x4921c0162c359755b2ae714a0978a1dad2eb8edce7ff9b38b9b6fc4cbc547eb5" + "withdrawalsRoot": "0x4921c0162c359755b2ae714a0978a1dad2eb8edce7ff9b38b9b6fc4cbc547eb5", + "requests": null } } diff --git a/cmd/evm/testdata/28/exp.json b/cmd/evm/testdata/28/exp.json index 75c715e972..b86c2d8def 100644 --- a/cmd/evm/testdata/28/exp.json +++ b/cmd/evm/testdata/28/exp.json @@ -42,6 +42,7 @@ "currentBaseFee": "0x9", "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "currentExcessBlobGas": "0x0", - "blobGasUsed": "0x20000" + "blobGasUsed": "0x20000", + "requests": null } } diff --git a/cmd/evm/testdata/29/exp.json b/cmd/evm/testdata/29/exp.json index c4c001ec14..7fbdc18283 100644 --- a/cmd/evm/testdata/29/exp.json +++ b/cmd/evm/testdata/29/exp.json @@ -40,6 +40,7 @@ "currentBaseFee": "0x9", "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "currentExcessBlobGas": "0x0", - "blobGasUsed": "0x0" + "blobGasUsed": "0x0", + "requests": null } } diff --git a/cmd/evm/testdata/3/exp.json b/cmd/evm/testdata/3/exp.json index 7230dca2cf..831c078591 100644 --- a/cmd/evm/testdata/3/exp.json +++ b/cmd/evm/testdata/3/exp.json @@ -34,6 +34,7 @@ } ], "currentDifficulty": "0x20000", - "gasUsed": "0x521f" + "gasUsed": "0x521f", + "requests": null } } diff --git a/cmd/evm/testdata/30/exp.json b/cmd/evm/testdata/30/exp.json index f0b19c6b3d..a206c3bbdf 100644 --- a/cmd/evm/testdata/30/exp.json +++ b/cmd/evm/testdata/30/exp.json @@ -59,6 +59,7 @@ "currentBaseFee": "0x7", "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "currentExcessBlobGas": "0x0", - "blobGasUsed": "0x0" + "blobGasUsed": "0x0", + "requests": null } -} \ No newline at end of file +} diff --git a/cmd/evm/testdata/33/README.md b/cmd/evm/testdata/33/README.md new file mode 100644 index 0000000000..24bea566e5 --- /dev/null +++ b/cmd/evm/testdata/33/README.md @@ -0,0 +1 @@ +This test sets some EIP-7702 delegations and calls them. diff --git a/cmd/evm/testdata/33/alloc.json b/cmd/evm/testdata/33/alloc.json new file mode 100644 index 0000000000..6874a6b339 --- /dev/null +++ b/cmd/evm/testdata/33/alloc.json @@ -0,0 +1,30 @@ +{ + "0x8a0a19589531694250d570040a0c4b74576919b8": { + "nonce": "0x00", + "balance": "0x0de0b6b3a7640000", + "code": "0x600060006000600060007310000000000000000000000000000000000000015af1600155600060006000600060007310000000000000000000000000000000000000025af16002553d600060003e600051600355", + "storage": { + "0x01": "0x0100", + "0x02": "0x0100", + "0x03": "0x0100" + } + }, + "0x000000000000000000000000000000000000aaaa": { + "nonce": "0x00", + "balance": "0x4563918244f40000", + "code": "0x58808080600173703c4b2bd70c169f5717101caee543299fc946c75af100", + "storage": {} + }, + "0x000000000000000000000000000000000000bbbb": { + "nonce": "0x00", + "balance": "0x29a2241af62c0000", + "code": "0x6042805500", + "storage": {} + }, + "0x71562b71999873DB5b286dF957af199Ec94617F7": { + "nonce": "0x00", + "balance": "0x6124fee993bc0000", + "code": "0x", + "storage": {} + } +} diff --git a/cmd/evm/testdata/33/env.json b/cmd/evm/testdata/33/env.json new file mode 100644 index 0000000000..70bb7f9812 --- /dev/null +++ b/cmd/evm/testdata/33/env.json @@ -0,0 +1,14 @@ +{ + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentGasLimit": "71794957647893862", + "currentNumber": "1", + "currentTimestamp": "1000", + "currentRandom": "0", + "currentDifficulty": "0", + "blockHashes": {}, + "ommers": [], + "currentBaseFee": "7", + "parentUncleHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "withdrawals": [], + "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000" +} diff --git a/cmd/evm/testdata/33/exp.json b/cmd/evm/testdata/33/exp.json new file mode 100644 index 0000000000..ae82ef3efa --- /dev/null +++ b/cmd/evm/testdata/33/exp.json @@ -0,0 +1,62 @@ +{ + "alloc": { + "0x000000000000000000000000000000000000aaaa": { + "code": "0x58808080600173703c4b2bd70c169f5717101caee543299fc946c75af100", + "balance": "0x4563918244f40000" + }, + "0x000000000000000000000000000000000000bbbb": { + "code": "0x6042805500", + "balance": "0x29a2241af62c0000" + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x2bf52" + }, + "0x703c4b2bd70c169f5717101caee543299fc946c7": { + "code": "0xef0100000000000000000000000000000000000000bbbb", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000042": "0x0000000000000000000000000000000000000000000000000000000000000042" + }, + "balance": "0x1", + "nonce": "0x1" + }, + "0x71562b71999873db5b286df957af199ec94617f7": { + "code": "0xef0100000000000000000000000000000000000000aaaa", + "balance": "0x6124fee993afa30e", + "nonce": "0x2" + }, + "0x8a0a19589531694250d570040a0c4b74576919b8": { + "code": "0x600060006000600060007310000000000000000000000000000000000000015af1600155600060006000600060007310000000000000000000000000000000000000025af16002553d600060003e600051600355", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x0000000000000000000000000000000000000000000000000000000000000100", + "0x0000000000000000000000000000000000000000000000000000000000000002": "0x0000000000000000000000000000000000000000000000000000000000000100", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x0000000000000000000000000000000000000000000000000000000000000100" + }, + "balance": "0xde0b6b3a7640000" + } + }, + "result": { + "stateRoot": "0x9fdcacd4510e93c4488e537dc51578b5c6d505771db64a2610036eeb4be7b26f", + "txRoot": "0x5d13a0b074e80388dc754da92b22922313a63417b3e25a10f324935e09697a53", + "receiptsRoot": "0x504c5d86c34391f70d210e6c482615b391db4bdb9f43479366399d9c5599850a", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","receipts": [{ + "type": "0x4", + "root": "0x", + "status": "0x1", + "cumulativeGasUsed": "0x15fa9", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","logs": null,"transactionHash": "0x0417aab7c1d8a3989190c3167c132876ce9b8afd99262c5a0f9d06802de3d7ef", + "contractAddress": "0x0000000000000000000000000000000000000000", + "gasUsed": "0x15fa9", + "effectiveGasPrice": null, + "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionIndex": "0x0" + } + ], + "currentDifficulty": null, + "gasUsed": "0x15fa9", + "currentBaseFee": "0x7", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "requestsHash": "0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "requests": [] +} +} diff --git a/cmd/evm/testdata/33/txs.json b/cmd/evm/testdata/33/txs.json new file mode 100644 index 0000000000..9c4db138c8 --- /dev/null +++ b/cmd/evm/testdata/33/txs.json @@ -0,0 +1,37 @@ +[ + { + "type": "0x4", + "chainId": "0x1", + "nonce": "0x0", + "to": "0x71562b71999873db5b286df957af199ec94617f7", + "gas": "0x7a120", + "gasPrice": null, + "maxPriorityFeePerGas": "0x2", + "maxFeePerGas": "0x12a05f200", + "value": "0x0", + "input": "0x", + "accessList": [], + "authorizationList": [ + { + "chainId": "0x1", + "address": "0x000000000000000000000000000000000000aaaa", + "nonce": "0x1", + "v": "0x1", + "r": "0xf7e3e597fc097e71ed6c26b14b25e5395bc8510d58b9136af439e12715f2d721", + "s": "0x6cf7c3d7939bfdb784373effc0ebb0bd7549691a513f395e3cdabf8602724987" + }, + { + "chainId": "0x0", + "address": "0x000000000000000000000000000000000000bbbb", + "nonce": "0x0", + "v": "0x1", + "r": "0x5011890f198f0356a887b0779bde5afa1ed04e6acb1e3f37f8f18c7b6f521b98", + "s": "0x56c3fa3456b103f3ef4a0acb4b647b9cab9ec4bc68fbcdf1e10b49fb2bcbcf61" + } + ], + "secretKey": "0xb71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291", + "v": "0x0", + "r": "0x0", + "s": "0x0" + } +] diff --git a/cmd/evm/testdata/5/exp.json b/cmd/evm/testdata/5/exp.json index 7d715672c5..00af8b084d 100644 --- a/cmd/evm/testdata/5/exp.json +++ b/cmd/evm/testdata/5/exp.json @@ -18,6 +18,7 @@ "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "receipts": [], "currentDifficulty": "0x20000", - "gasUsed": "0x0" + "gasUsed": "0x0", + "requests": null } } diff --git a/core/bench_test.go b/core/bench_test.go index 6d518e8d3b..d376830318 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -90,7 +90,7 @@ func genValueTx(nbytes int) func(int, *BlockGen) { data := make([]byte, nbytes) return func(i int, gen *BlockGen) { toaddr := common.Address{} - gas, _ := IntrinsicGas(data, nil, false, false, false, false) + gas, _ := IntrinsicGas(data, nil, nil, false, false, false, false) signer := gen.Signer() gasPrice := big.NewInt(0) if gen.header.BaseFee != nil { diff --git a/core/blockchain_test.go b/core/blockchain_test.go index dc391bb520..a54a907766 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -17,6 +17,7 @@ package core import ( + "bytes" "errors" "fmt" gomath "math" @@ -36,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/vm/program" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/ethdb" @@ -4232,3 +4234,95 @@ func TestPragueRequests(t *testing.T) { t.Fatalf("block %d: failed to insert into chain: %v", n, err) } } + +// TestEIP7702 deploys two delegation designations and calls them. It writes one +// value to storage which is verified after. +func TestEIP7702(t *testing.T) { + var ( + config = *params.MergedTestChainConfig + signer = types.LatestSigner(&config) + engine = beacon.NewFaker() + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + addr2 = crypto.PubkeyToAddress(key2.PublicKey) + aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") + bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb") + funds = new(big.Int).Mul(common.Big1, big.NewInt(params.Ether)) + ) + gspec := &Genesis{ + Config: &config, + Alloc: types.GenesisAlloc{ + addr1: {Balance: funds}, + addr2: {Balance: funds}, + aa: { // The address 0xAAAA calls into addr2 + Code: program.New().Call(nil, addr2, 1, 0, 0, 0, 0).Bytes(), + Nonce: 0, + Balance: big.NewInt(0), + }, + bb: { // The address 0xBBBB sstores 42 into slot 42. + Code: program.New().Sstore(0x42, 0x42).Bytes(), + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + } + + // Sign authorization tuples. + // The way the auths are combined, it becomes + // 1. tx -> addr1 which is delegated to 0xaaaa + // 2. addr1:0xaaaa calls into addr2:0xbbbb + // 3. addr2:0xbbbb writes to storage + auth1, _ := types.SignAuth(types.Authorization{ + ChainID: gspec.Config.ChainID.Uint64(), + Address: aa, + Nonce: 1, + }, key1) + auth2, _ := types.SignAuth(types.Authorization{ + ChainID: 0, + Address: bb, + Nonce: 0, + }, key2) + + _, blocks, _ := GenerateChainWithGenesis(gspec, engine, 1, func(i int, b *BlockGen) { + b.SetCoinbase(aa) + txdata := &types.SetCodeTx{ + ChainID: gspec.Config.ChainID.Uint64(), + Nonce: 0, + To: addr1, + Gas: 500000, + GasFeeCap: uint256.MustFromBig(newGwei(5)), + GasTipCap: uint256.NewInt(2), + AuthList: []types.Authorization{auth1, auth2}, + } + tx := types.MustSignNewTx(key1, signer, txdata) + b.AddTx(tx) + }) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + defer chain.Stop() + if n, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", n, err) + } + + // Verify delegation designations were deployed. + state, _ := chain.State() + code, want := state.GetCode(addr1), types.AddressToDelegation(auth1.Address) + if !bytes.Equal(code, want) { + t.Fatalf("addr1 code incorrect: got %s, want %s", common.Bytes2Hex(code), common.Bytes2Hex(want)) + } + code, want = state.GetCode(addr2), types.AddressToDelegation(auth2.Address) + if !bytes.Equal(code, want) { + t.Fatalf("addr2 code incorrect: got %s, want %s", common.Bytes2Hex(code), common.Bytes2Hex(want)) + } + // Verify delegation executed the correct code. + var ( + fortyTwo = common.BytesToHash([]byte{0x42}) + actual = state.GetState(addr2, fortyTwo) + ) + if actual.Cmp(fortyTwo) != 0 { + t.Fatalf("addr2 storage wrong: expected %d, got %d", fortyTwo, actual) + } +} diff --git a/core/error.go b/core/error.go index 161538fe43..82f7ddcf5d 100644 --- a/core/error.go +++ b/core/error.go @@ -103,6 +103,8 @@ var ( // ErrSenderNoEOA is returned if the sender of a transaction is a contract. ErrSenderNoEOA = errors.New("sender not an eoa") + // -- EIP-4844 errors -- + // ErrBlobFeeCapTooLow is returned if the transaction fee cap is less than the // blob gas fee of the block. ErrBlobFeeCapTooLow = errors.New("max fee per blob gas less than block blob gas fee") @@ -112,4 +114,20 @@ var ( // ErrBlobTxCreate is returned if a blob transaction has no explicit to field. ErrBlobTxCreate = errors.New("blob transaction of type create") + + // -- EIP-7702 errors -- + + // Message validation errors: + ErrEmptyAuthList = errors.New("EIP-7702 transaction with empty auth list") + ErrSetCodeTxCreate = errors.New("EIP-7702 transaction cannot be used to create contract") +) + +// EIP-7702 state transition errors. +// Note these are just informational, and do not cause tx execution abort. +var ( + ErrAuthorizationWrongChainID = errors.New("EIP-7702 authorization chain ID mismatch") + ErrAuthorizationNonceOverflow = errors.New("EIP-7702 authorization nonce > 64 bit") + ErrAuthorizationInvalidSignature = errors.New("EIP-7702 authorization has invalid signature") + ErrAuthorizationDestinationHasCode = errors.New("EIP-7702 authorization destination is a contract") + ErrAuthorizationNonceMismatch = errors.New("EIP-7702 authorization nonce does not match current account nonce") ) diff --git a/core/state/journal.go b/core/state/journal.go index a2fea6b6ec..13332dbd79 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -23,7 +23,7 @@ import ( "sort" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/holiman/uint256" ) @@ -192,8 +192,11 @@ func (j *journal) balanceChange(addr common.Address, previous *uint256.Int) { }) } -func (j *journal) setCode(address common.Address) { - j.append(codeChange{account: address}) +func (j *journal) setCode(address common.Address, prevCode []byte) { + j.append(codeChange{ + account: address, + prevCode: prevCode, + }) } func (j *journal) nonceChange(address common.Address, prev uint64) { @@ -256,7 +259,8 @@ type ( origvalue common.Hash } codeChange struct { - account common.Address + account common.Address + prevCode []byte } // Changes to other state values. @@ -377,7 +381,7 @@ func (ch nonceChange) copy() journalEntry { } func (ch codeChange) revert(s *StateDB) { - s.getStateObject(ch.account).setCode(types.EmptyCodeHash, nil) + s.getStateObject(ch.account).setCode(crypto.Keccak256Hash(ch.prevCode), ch.prevCode) } func (ch codeChange) dirtied() *common.Address { @@ -385,7 +389,10 @@ func (ch codeChange) dirtied() *common.Address { } func (ch codeChange) copy() journalEntry { - return codeChange{account: ch.account} + return codeChange{ + account: ch.account, + prevCode: ch.prevCode, + } } func (ch storageChange) revert(s *StateDB) { diff --git a/core/state/state_object.go b/core/state/state_object.go index 2d542e5005..76a3aba92c 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -20,6 +20,7 @@ import ( "bytes" "fmt" "maps" + "slices" "time" "github.com/ethereum/go-ethereum/common" @@ -541,9 +542,11 @@ func (s *stateObject) CodeSize() int { return size } -func (s *stateObject) SetCode(codeHash common.Hash, code []byte) { - s.db.journal.setCode(s.address) +func (s *stateObject) SetCode(codeHash common.Hash, code []byte) (prev []byte) { + prev = slices.Clone(s.code) + s.db.journal.setCode(s.address, prev) s.setCode(codeHash, code) + return prev } func (s *stateObject) setCode(codeHash common.Hash, code []byte) { diff --git a/core/state/statedb.go b/core/state/statedb.go index b0603db7f0..d279ccfdfe 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -439,11 +439,12 @@ func (s *StateDB) SetNonce(addr common.Address, nonce uint64) { } } -func (s *StateDB) SetCode(addr common.Address, code []byte) { +func (s *StateDB) SetCode(addr common.Address, code []byte) (prev []byte) { stateObject := s.getOrNewStateObject(addr) if stateObject != nil { - stateObject.SetCode(crypto.Keccak256Hash(code), code) + return stateObject.SetCode(crypto.Keccak256Hash(code), code) } + return nil } func (s *StateDB) SetState(addr common.Address, key, value common.Hash) common.Hash { diff --git a/core/state/statedb_hooked.go b/core/state/statedb_hooked.go index 26d0217099..31bdd06b46 100644 --- a/core/state/statedb_hooked.go +++ b/core/state/statedb_hooked.go @@ -182,11 +182,16 @@ func (s *hookedStateDB) SetNonce(address common.Address, nonce uint64) { } } -func (s *hookedStateDB) SetCode(address common.Address, code []byte) { - s.inner.SetCode(address, code) +func (s *hookedStateDB) SetCode(address common.Address, code []byte) []byte { + prev := s.inner.SetCode(address, code) if s.hooks.OnCodeChange != nil { - s.hooks.OnCodeChange(address, types.EmptyCodeHash, nil, crypto.Keccak256Hash(code), code) + prevHash := types.EmptyCodeHash + if len(prev) != 0 { + prevHash = crypto.Keccak256Hash(prev) + } + s.hooks.OnCodeChange(address, prevHash, prev, crypto.Keccak256Hash(code), code) } + return prev } func (s *hookedStateDB) SetState(address common.Address, key common.Hash, value common.Hash) common.Hash { diff --git a/core/state_processor_test.go b/core/state_processor_test.go index f3d2304690..e8d8c2ca2e 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -63,6 +63,7 @@ func TestStateProcessorErrors(t *testing.T) { TerminalTotalDifficulty: big.NewInt(0), ShanghaiTime: new(uint64), CancunTime: new(uint64), + PragueTime: new(uint64), } signer = types.LatestSigner(config) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -110,6 +111,21 @@ func TestStateProcessorErrors(t *testing.T) { } return tx } + var mkSetCodeTx = func(nonce uint64, to common.Address, gasLimit uint64, gasTipCap, gasFeeCap *big.Int, authlist []types.Authorization) *types.Transaction { + tx, err := types.SignTx(types.NewTx(&types.SetCodeTx{ + Nonce: nonce, + GasTipCap: uint256.MustFromBig(gasTipCap), + GasFeeCap: uint256.MustFromBig(gasFeeCap), + Gas: gasLimit, + To: to, + Value: new(uint256.Int), + AuthList: authlist, + }), signer, key1) + if err != nil { + t.Fatal(err) + } + return tx + } { // Tests against a 'recent' chain definition var ( @@ -251,6 +267,13 @@ func TestStateProcessorErrors(t *testing.T) { }, want: "could not apply tx 0 [0x6c11015985ce82db691d7b2d017acda296db88b811c3c60dc71449c76256c716]: max fee per gas less than block base fee: address 0x71562b71999873DB5b286dF957af199Ec94617F7, maxFeePerGas: 1, baseFee: 875000000", }, + { // ErrEmptyAuthList + txs: []*types.Transaction{ + mkSetCodeTx(0, common.Address{}, params.TxGas, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), nil), + }, + want: "could not apply tx 0 [0xc18d10f4c809dbdfa1a074c3300de9bc4b7f16a20f0ec667f6f67312b71b956a]: EIP-7702 transaction with empty auth list (sender 0x71562b71999873DB5b286dF957af199Ec94617F7)", + }, + // ErrSetCodeTxCreate cannot be tested: it is impossible to create a SetCode-tx with nil `to`. } { block := GenerateBadBlock(gspec.ToBlock(), beacon.New(ethash.NewFaker()), tt.txs, gspec.Config, false) _, err := blockchain.InsertChain(types.Blocks{block}) @@ -337,7 +360,7 @@ func TestStateProcessorErrors(t *testing.T) { txs: []*types.Transaction{ mkDynamicTx(0, common.Address{}, params.TxGas-1000, big.NewInt(0), big.NewInt(0)), }, - want: "could not apply tx 0 [0x88626ac0d53cb65308f2416103c62bb1f18b805573d4f96a3640bbbfff13c14f]: sender not an eoa: address 0x71562b71999873DB5b286dF957af199Ec94617F7, codehash: 0x9280914443471259d4570a8661015ae4a5b80186dbc619658fb494bebc3da3d1", + want: "could not apply tx 0 [0x88626ac0d53cb65308f2416103c62bb1f18b805573d4f96a3640bbbfff13c14f]: sender not an eoa: address 0x71562b71999873DB5b286dF957af199Ec94617F7, len(code): 4", }, } { block := GenerateBadBlock(gspec.ToBlock(), beacon.New(ethash.NewFaker()), tt.txs, gspec.Config, false) diff --git a/core/state_transition.go b/core/state_transition.go index ea7e3df2ff..58728e470e 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -67,7 +67,7 @@ func (result *ExecutionResult) Revert() []byte { } // IntrinsicGas computes the 'intrinsic gas' for a message with the given data. -func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation, isHomestead, isEIP2028, isEIP3860 bool) (uint64, error) { +func IntrinsicGas(data []byte, accessList types.AccessList, authList []types.Authorization, isContractCreation, isHomestead, isEIP2028, isEIP3860 bool) (uint64, error) { // Set the starting gas for the raw transaction var gas uint64 if isContractCreation && isHomestead { @@ -113,6 +113,9 @@ func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation, gas += uint64(len(accessList)) * params.TxAccessListAddressGas gas += uint64(accessList.StorageKeys()) * params.TxAccessListStorageKeyGas } + if authList != nil { + gas += uint64(len(authList)) * params.CallNewAccountGas + } return gas, nil } @@ -140,6 +143,7 @@ type Message struct { AccessList types.AccessList BlobGasFeeCap *big.Int BlobHashes []common.Hash + AuthList []types.Authorization // When SkipNonceChecks is true, the message nonce is not checked against the // account nonce in state. @@ -162,6 +166,7 @@ func TransactionToMessage(tx *types.Transaction, s types.Signer, baseFee *big.In Value: tx.Value(), Data: tx.Data(), AccessList: tx.AccessList(), + AuthList: tx.AuthList(), SkipNonceChecks: false, SkipFromEOACheck: false, BlobHashes: tx.BlobHashes(), @@ -303,10 +308,10 @@ func (st *stateTransition) preCheck() error { } if !msg.SkipFromEOACheck { // Make sure the sender is an EOA - codeHash := st.state.GetCodeHash(msg.From) - if codeHash != (common.Hash{}) && codeHash != types.EmptyCodeHash { - return fmt.Errorf("%w: address %v, codehash: %s", ErrSenderNoEOA, - msg.From.Hex(), codeHash) + code := st.state.GetCode(msg.From) + _, delegated := types.ParseDelegation(code) + if len(code) > 0 && !delegated { + return fmt.Errorf("%w: address %v, len(code): %d", ErrSenderNoEOA, msg.From.Hex(), len(code)) } } // Make sure that transaction gasFeeCap is greater than the baseFee (post london) @@ -366,6 +371,15 @@ func (st *stateTransition) preCheck() error { } } } + // Check that EIP-7702 authorization list signatures are well formed. + if msg.AuthList != nil { + if msg.To == nil { + return fmt.Errorf("%w (sender %v)", ErrSetCodeTxCreate, msg.From) + } + if len(msg.AuthList) == 0 { + return fmt.Errorf("%w (sender %v)", ErrEmptyAuthList, msg.From) + } + } return st.buyGas() } @@ -403,7 +417,7 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { ) // Check clauses 4-5, subtract intrinsic gas if everything is correct - gas, err := IntrinsicGas(msg.Data, msg.AccessList, contractCreation, rules.IsHomestead, rules.IsIstanbul, rules.IsShanghai) + gas, err := IntrinsicGas(msg.Data, msg.AccessList, msg.AuthList, contractCreation, rules.IsHomestead, rules.IsIstanbul, rules.IsShanghai) if err != nil { return nil, err } @@ -449,8 +463,27 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { if contractCreation { ret, _, st.gasRemaining, vmerr = st.evm.Create(sender, msg.Data, st.gasRemaining, value) } else { - // Increment the nonce for the next transaction - st.state.SetNonce(msg.From, st.state.GetNonce(sender.Address())+1) + // Increment the nonce for the next transaction. + st.state.SetNonce(msg.From, st.state.GetNonce(msg.From)+1) + + // Apply EIP-7702 authorizations. + if msg.AuthList != nil { + for _, auth := range msg.AuthList { + // Note errors are ignored, we simply skip invalid authorizations here. + st.applyAuthorization(msg, &auth) + } + } + + // Perform convenience warming of sender's delegation target. Although the + // sender is already warmed in Prepare(..), it's possible a delegation to + // the account was deployed during this transaction. To handle correctly, + // simply wait until the final state of delegations is determined before + // performing the resolution and warming. + if addr, ok := types.ParseDelegation(st.state.GetCode(*msg.To)); ok { + st.state.AddAddressToAccessList(addr) + } + + // Execute the transaction's call. ret, st.gasRemaining, vmerr = st.evm.Call(sender, st.to(), msg.Data, st.gasRemaining, value) } @@ -494,6 +527,64 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { }, nil } +// validateAuthorization validates an EIP-7702 authorization against the state. +func (st *stateTransition) validateAuthorization(auth *types.Authorization) (authority common.Address, err error) { + // Verify chain ID is 0 or equal to current chain ID. + if auth.ChainID != 0 && st.evm.ChainConfig().ChainID.Uint64() != auth.ChainID { + return authority, ErrAuthorizationWrongChainID + } + // Limit nonce to 2^64-1 per EIP-2681. + if auth.Nonce+1 < auth.Nonce { + return authority, ErrAuthorizationNonceOverflow + } + // Validate signature values and recover authority. + authority, err = auth.Authority() + if err != nil { + return authority, fmt.Errorf("%w: %v", ErrAuthorizationInvalidSignature, err) + } + // Check the authority account + // 1) doesn't have code or has exisiting delegation + // 2) matches the auth's nonce + // + // Note it is added to the access list even if the authorization is invalid. + st.state.AddAddressToAccessList(authority) + code := st.state.GetCode(authority) + if _, ok := types.ParseDelegation(code); len(code) != 0 && !ok { + return authority, ErrAuthorizationDestinationHasCode + } + if have := st.state.GetNonce(authority); have != auth.Nonce { + return authority, ErrAuthorizationNonceMismatch + } + return authority, nil +} + +// applyAuthorization applies an EIP-7702 code delegation to the state. +func (st *stateTransition) applyAuthorization(msg *Message, auth *types.Authorization) error { + authority, err := st.validateAuthorization(auth) + if err != nil { + return err + } + + // If the account already exists in state, refund the new account cost + // charged in the intrinsic calculation. + if st.state.Exist(authority) { + st.state.AddRefund(params.CallNewAccountGas - params.TxAuthTupleGas) + } + + // Update nonce and account code. + st.state.SetNonce(authority, auth.Nonce+1) + if auth.Address == (common.Address{}) { + // Delegation to zero address means clear. + st.state.SetCode(authority, nil) + return nil + } + + // Otherwise install delegation to auth.Address. + st.state.SetCode(authority, types.AddressToDelegation(auth.Address)) + + return nil +} + func (st *stateTransition) refundGas(refundQuotient uint64) uint64 { // Apply refund counter, capped to a refund quotient refund := st.gasUsed() / refundQuotient diff --git a/core/txpool/validation.go b/core/txpool/validation.go index 33b383d5cf..908945471d 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -108,7 +108,7 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types } // Ensure the transaction has more gas than the bare minimum needed to cover // the transaction metadata - intrGas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.To() == nil, true, opts.Config.IsIstanbul(head.Number), opts.Config.IsShanghai(head.Number, head.Time)) + intrGas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.AuthList(), tx.To() == nil, true, opts.Config.IsIstanbul(head.Number), opts.Config.IsShanghai(head.Number, head.Time)) if err != nil { return err } diff --git a/core/types/gen_authorization.go b/core/types/gen_authorization.go new file mode 100644 index 0000000000..b598b64ff7 --- /dev/null +++ b/core/types/gen_authorization.go @@ -0,0 +1,75 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package types + +import ( + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/holiman/uint256" +) + +var _ = (*authorizationMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (a Authorization) MarshalJSON() ([]byte, error) { + type Authorization struct { + ChainID hexutil.Uint64 `json:"chainId" gencodec:"required"` + Address common.Address `json:"address" gencodec:"required"` + Nonce hexutil.Uint64 `json:"nonce" gencodec:"required"` + V hexutil.Uint64 `json:"v" gencodec:"required"` + R uint256.Int `json:"r" gencodec:"required"` + S uint256.Int `json:"s" gencodec:"required"` + } + var enc Authorization + enc.ChainID = hexutil.Uint64(a.ChainID) + enc.Address = a.Address + enc.Nonce = hexutil.Uint64(a.Nonce) + enc.V = hexutil.Uint64(a.V) + enc.R = a.R + enc.S = a.S + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (a *Authorization) UnmarshalJSON(input []byte) error { + type Authorization struct { + ChainID *hexutil.Uint64 `json:"chainId" gencodec:"required"` + Address *common.Address `json:"address" gencodec:"required"` + Nonce *hexutil.Uint64 `json:"nonce" gencodec:"required"` + V *hexutil.Uint64 `json:"v" gencodec:"required"` + R *uint256.Int `json:"r" gencodec:"required"` + S *uint256.Int `json:"s" gencodec:"required"` + } + var dec Authorization + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.ChainID == nil { + return errors.New("missing required field 'chainId' for Authorization") + } + a.ChainID = uint64(*dec.ChainID) + if dec.Address == nil { + return errors.New("missing required field 'address' for Authorization") + } + a.Address = *dec.Address + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' for Authorization") + } + a.Nonce = uint64(*dec.Nonce) + if dec.V == nil { + return errors.New("missing required field 'v' for Authorization") + } + a.V = uint8(*dec.V) + if dec.R == nil { + return errors.New("missing required field 'r' for Authorization") + } + a.R = *dec.R + if dec.S == nil { + return errors.New("missing required field 's' for Authorization") + } + a.S = *dec.S + return nil +} diff --git a/core/types/receipt.go b/core/types/receipt.go index 4f96fde59c..47f16d950a 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -204,7 +204,7 @@ func (r *Receipt) decodeTyped(b []byte) error { return errShortTypedReceipt } switch b[0] { - case DynamicFeeTxType, AccessListTxType, BlobTxType: + case DynamicFeeTxType, AccessListTxType, BlobTxType, SetCodeTxType: var data receiptRLP err := rlp.DecodeBytes(b[1:], &data) if err != nil { @@ -312,7 +312,7 @@ func (rs Receipts) EncodeIndex(i int, w *bytes.Buffer) { } w.WriteByte(r.Type) switch r.Type { - case AccessListTxType, DynamicFeeTxType, BlobTxType: + case AccessListTxType, DynamicFeeTxType, BlobTxType, SetCodeTxType: rlp.Encode(w, data) default: // For unsupported types, write nothing. Since this is for diff --git a/core/types/transaction.go b/core/types/transaction.go index ea6f5ad6b9..b5fb3e2db2 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -48,6 +48,7 @@ const ( AccessListTxType = 0x01 DynamicFeeTxType = 0x02 BlobTxType = 0x03 + SetCodeTxType = 0x04 ) // Transaction is an Ethereum transaction. @@ -205,6 +206,8 @@ func (tx *Transaction) decodeTyped(b []byte) (TxData, error) { inner = new(DynamicFeeTx) case BlobTxType: inner = new(BlobTx) + case SetCodeTxType: + inner = new(SetCodeTx) default: return nil, ErrTxTypeNotSupported } @@ -471,6 +474,15 @@ func (tx *Transaction) WithBlobTxSidecar(sideCar *BlobTxSidecar) *Transaction { return cpy } +// AuthList returns the authorizations list of the transaction. +func (tx *Transaction) AuthList() []Authorization { + setcodetx, ok := tx.inner.(*SetCodeTx) + if !ok { + return nil + } + return setcodetx.AuthList +} + // SetTime sets the decoding time of a transaction. This is used by tests to set // arbitrary times and by persistent transaction pools when loading old txs from // disk. diff --git a/core/types/transaction_marshalling.go b/core/types/transaction_marshalling.go index 4d5b2bcdd4..4176a8220c 100644 --- a/core/types/transaction_marshalling.go +++ b/core/types/transaction_marshalling.go @@ -43,6 +43,7 @@ type txJSON struct { Input *hexutil.Bytes `json:"input"` AccessList *AccessList `json:"accessList,omitempty"` BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` + AuthorizationList []Authorization `json:"authorizationList,omitempty"` V *hexutil.Big `json:"v"` R *hexutil.Big `json:"r"` S *hexutil.Big `json:"s"` @@ -153,6 +154,22 @@ func (tx *Transaction) MarshalJSON() ([]byte, error) { enc.Commitments = itx.Sidecar.Commitments enc.Proofs = itx.Sidecar.Proofs } + case *SetCodeTx: + enc.ChainID = (*hexutil.Big)(new(big.Int).SetUint64(itx.ChainID)) + enc.Nonce = (*hexutil.Uint64)(&itx.Nonce) + enc.To = tx.To() + enc.Gas = (*hexutil.Uint64)(&itx.Gas) + enc.MaxFeePerGas = (*hexutil.Big)(itx.GasFeeCap.ToBig()) + enc.MaxPriorityFeePerGas = (*hexutil.Big)(itx.GasTipCap.ToBig()) + enc.Value = (*hexutil.Big)(itx.Value.ToBig()) + enc.Input = (*hexutil.Bytes)(&itx.Data) + enc.AccessList = &itx.AccessList + enc.AuthorizationList = itx.AuthList + enc.V = (*hexutil.Big)(itx.V.ToBig()) + enc.R = (*hexutil.Big)(itx.R.ToBig()) + enc.S = (*hexutil.Big)(itx.S.ToBig()) + yparity := itx.V.Uint64() + enc.YParity = (*hexutil.Uint64)(&yparity) } return json.Marshal(&enc) } @@ -409,6 +426,81 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error { } } + case SetCodeTxType: + var itx SetCodeTx + inner = &itx + if dec.ChainID == nil { + return errors.New("missing required field 'chainId' in transaction") + } + itx.ChainID = dec.ChainID.ToInt().Uint64() + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' in transaction") + } + itx.Nonce = uint64(*dec.Nonce) + if dec.To == nil { + return errors.New("missing required field 'to' in transaction") + } + itx.To = *dec.To + if dec.Gas == nil { + return errors.New("missing required field 'gas' for txdata") + } + itx.Gas = uint64(*dec.Gas) + if dec.MaxPriorityFeePerGas == nil { + return errors.New("missing required field 'maxPriorityFeePerGas' for txdata") + } + itx.GasTipCap = uint256.MustFromBig((*big.Int)(dec.MaxPriorityFeePerGas)) + if dec.MaxFeePerGas == nil { + return errors.New("missing required field 'maxFeePerGas' for txdata") + } + itx.GasFeeCap = uint256.MustFromBig((*big.Int)(dec.MaxFeePerGas)) + if dec.Value == nil { + return errors.New("missing required field 'value' in transaction") + } + itx.Value = uint256.MustFromBig((*big.Int)(dec.Value)) + if dec.Input == nil { + return errors.New("missing required field 'input' in transaction") + } + itx.Data = *dec.Input + if dec.AccessList != nil { + itx.AccessList = *dec.AccessList + } + if dec.AuthorizationList == nil { + return errors.New("missing required field 'authorizationList' in transaction") + } + itx.AuthList = dec.AuthorizationList + + // signature R + var overflow bool + if dec.R == nil { + return errors.New("missing required field 'r' in transaction") + } + itx.R, overflow = uint256.FromBig((*big.Int)(dec.R)) + if overflow { + return errors.New("'r' value overflows uint256") + } + // signature S + if dec.S == nil { + return errors.New("missing required field 's' in transaction") + } + itx.S, overflow = uint256.FromBig((*big.Int)(dec.S)) + if overflow { + return errors.New("'s' value overflows uint256") + } + // signature V + vbig, err := dec.yParityValue() + if err != nil { + return err + } + itx.V, overflow = uint256.FromBig(vbig) + if overflow { + return errors.New("'v' value overflows uint256") + } + if itx.V.Sign() != 0 || itx.R.Sign() != 0 || itx.S.Sign() != 0 { + if err := sanityCheckSignature(vbig, itx.R.ToBig(), itx.S.ToBig(), false); err != nil { + return err + } + } + default: return ErrTxTypeNotSupported } diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 73011e238b..78fa2fc8ef 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -40,6 +40,8 @@ type sigCache struct { func MakeSigner(config *params.ChainConfig, blockNumber *big.Int, blockTime uint64) Signer { var signer Signer switch { + case config.IsPrague(blockNumber, blockTime): + signer = NewPragueSigner(config.ChainID) case config.IsCancun(blockNumber, blockTime): signer = NewCancunSigner(config.ChainID) case config.IsLondon(blockNumber): @@ -67,6 +69,8 @@ func LatestSigner(config *params.ChainConfig) Signer { var signer Signer if config.ChainID != nil { switch { + case config.PragueTime != nil: + signer = NewPragueSigner(config.ChainID) case config.CancunTime != nil: signer = NewCancunSigner(config.ChainID) case config.LondonBlock != nil: @@ -94,7 +98,7 @@ func LatestSigner(config *params.ChainConfig) Signer { func LatestSignerForChainID(chainID *big.Int) Signer { var signer Signer if chainID != nil { - signer = NewCancunSigner(chainID) + signer = NewPragueSigner(chainID) } else { signer = HomesteadSigner{} } @@ -174,6 +178,77 @@ type Signer interface { Equal(Signer) bool } +type pragueSigner struct{ cancunSigner } + +// NewPragueSigner returns a signer that accepts +// - EIP-7702 set code transactions +// - EIP-4844 blob transactions +// - EIP-1559 dynamic fee transactions +// - EIP-2930 access list transactions, +// - EIP-155 replay protected transactions, and +// - legacy Homestead transactions. +func NewPragueSigner(chainId *big.Int) Signer { + signer, _ := NewCancunSigner(chainId).(cancunSigner) + return pragueSigner{signer} +} + +func (s pragueSigner) Sender(tx *Transaction) (common.Address, error) { + if tx.Type() != SetCodeTxType { + return s.cancunSigner.Sender(tx) + } + V, R, S := tx.RawSignatureValues() + + // Set code txs are defined to use 0 and 1 as their recovery + // id, add 27 to become equivalent to unprotected Homestead signatures. + V = new(big.Int).Add(V, big.NewInt(27)) + if tx.ChainId().Cmp(s.chainId) != 0 { + return common.Address{}, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, tx.ChainId(), s.chainId) + } + return recoverPlain(s.Hash(tx), R, S, V, true) +} + +func (s pragueSigner) Equal(s2 Signer) bool { + x, ok := s2.(pragueSigner) + return ok && x.chainId.Cmp(s.chainId) == 0 +} + +func (s pragueSigner) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) { + txdata, ok := tx.inner.(*SetCodeTx) + if !ok { + return s.cancunSigner.SignatureValues(tx, sig) + } + // Check that chain ID of tx matches the signer. We also accept ID zero here, + // because it indicates that the chain ID was not specified in the tx. + if txdata.ChainID != 0 && new(big.Int).SetUint64(txdata.ChainID).Cmp(s.chainId) != 0 { + return nil, nil, nil, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, txdata.ChainID, s.chainId) + } + R, S, _ = decodeSignature(sig) + V = big.NewInt(int64(sig[64])) + return R, S, V, nil +} + +// Hash returns the hash to be signed by the sender. +// It does not uniquely identify the transaction. +func (s pragueSigner) Hash(tx *Transaction) common.Hash { + if tx.Type() != SetCodeTxType { + return s.cancunSigner.Hash(tx) + } + return prefixedRlpHash( + tx.Type(), + []interface{}{ + s.chainId, + tx.Nonce(), + tx.GasTipCap(), + tx.GasFeeCap(), + tx.Gas(), + tx.To(), + tx.Value(), + tx.Data(), + tx.AccessList(), + tx.AuthList(), + }) +} + type cancunSigner struct{ londonSigner } // NewCancunSigner returns a signer that accepts diff --git a/core/types/tx_setcode.go b/core/types/tx_setcode.go new file mode 100644 index 0000000000..0fea9a63a7 --- /dev/null +++ b/core/types/tx_setcode.go @@ -0,0 +1,226 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "bytes" + "crypto/ecdsa" + "errors" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/holiman/uint256" +) + +// DelegationPrefix is used by code to denote the account is delegating to +// another account. +var DelegationPrefix = []byte{0xef, 0x01, 0x00} + +// ParseDelegation tries to parse the address from a delegation slice. +func ParseDelegation(b []byte) (common.Address, bool) { + if len(b) != 23 || !bytes.HasPrefix(b, DelegationPrefix) { + return common.Address{}, false + } + return common.BytesToAddress(b[len(DelegationPrefix):]), true +} + +// AddressToDelegation adds the delegation prefix to the specified address. +func AddressToDelegation(addr common.Address) []byte { + return append(DelegationPrefix, addr.Bytes()...) +} + +// SetCodeTx implements the EIP-7702 transaction type which temporarily installs +// the code at the signer's address. +type SetCodeTx struct { + ChainID uint64 + Nonce uint64 + GasTipCap *uint256.Int // a.k.a. maxPriorityFeePerGas + GasFeeCap *uint256.Int // a.k.a. maxFeePerGas + Gas uint64 + To common.Address + Value *uint256.Int + Data []byte + AccessList AccessList + AuthList []Authorization + + // Signature values + V *uint256.Int `json:"v" gencodec:"required"` + R *uint256.Int `json:"r" gencodec:"required"` + S *uint256.Int `json:"s" gencodec:"required"` +} + +//go:generate go run github.com/fjl/gencodec -type Authorization -field-override authorizationMarshaling -out gen_authorization.go + +// Authorization is an authorization from an account to deploy code at its address. +type Authorization struct { + ChainID uint64 `json:"chainId" gencodec:"required"` + Address common.Address `json:"address" gencodec:"required"` + Nonce uint64 `json:"nonce" gencodec:"required"` + V uint8 `json:"v" gencodec:"required"` + R uint256.Int `json:"r" gencodec:"required"` + S uint256.Int `json:"s" gencodec:"required"` +} + +// field type overrides for gencodec +type authorizationMarshaling struct { + ChainID hexutil.Uint64 + Nonce hexutil.Uint64 + V hexutil.Uint64 +} + +// SignAuth signs the provided authorization. +func SignAuth(auth Authorization, prv *ecdsa.PrivateKey) (Authorization, error) { + sighash := auth.sigHash() + sig, err := crypto.Sign(sighash[:], prv) + if err != nil { + return Authorization{}, err + } + return auth.withSignature(sig), nil +} + +// withSignature updates the signature of an Authorization to be equal the +// decoded signature provided in sig. +func (a *Authorization) withSignature(sig []byte) Authorization { + r, s, _ := decodeSignature(sig) + return Authorization{ + ChainID: a.ChainID, + Address: a.Address, + Nonce: a.Nonce, + V: sig[64], + R: *uint256.MustFromBig(r), + S: *uint256.MustFromBig(s), + } +} + +func (a *Authorization) sigHash() common.Hash { + return prefixedRlpHash(0x05, []any{ + a.ChainID, + a.Address, + a.Nonce, + }) +} + +// Authority recovers the the authorizing account of an authorization. +func (a *Authorization) Authority() (common.Address, error) { + sighash := a.sigHash() + if !crypto.ValidateSignatureValues(a.V, a.R.ToBig(), a.S.ToBig(), true) { + return common.Address{}, ErrInvalidSig + } + // encode the signature in uncompressed format + var sig [crypto.SignatureLength]byte + a.R.WriteToSlice(sig[:32]) + a.S.WriteToSlice(sig[32:64]) + sig[64] = a.V + // recover the public key from the signature + pub, err := crypto.Ecrecover(sighash[:], sig[:]) + if err != nil { + return common.Address{}, err + } + if len(pub) == 0 || pub[0] != 4 { + return common.Address{}, errors.New("invalid public key") + } + var addr common.Address + copy(addr[:], crypto.Keccak256(pub[1:])[12:]) + return addr, nil +} + +// copy creates a deep copy of the transaction data and initializes all fields. +func (tx *SetCodeTx) copy() TxData { + cpy := &SetCodeTx{ + Nonce: tx.Nonce, + To: tx.To, + Data: common.CopyBytes(tx.Data), + Gas: tx.Gas, + // These are copied below. + AccessList: make(AccessList, len(tx.AccessList)), + AuthList: make([]Authorization, len(tx.AuthList)), + Value: new(uint256.Int), + ChainID: tx.ChainID, + GasTipCap: new(uint256.Int), + GasFeeCap: new(uint256.Int), + V: new(uint256.Int), + R: new(uint256.Int), + S: new(uint256.Int), + } + copy(cpy.AccessList, tx.AccessList) + copy(cpy.AuthList, tx.AuthList) + if tx.Value != nil { + cpy.Value.Set(tx.Value) + } + if tx.GasTipCap != nil { + cpy.GasTipCap.Set(tx.GasTipCap) + } + if tx.GasFeeCap != nil { + cpy.GasFeeCap.Set(tx.GasFeeCap) + } + if tx.V != nil { + cpy.V.Set(tx.V) + } + if tx.R != nil { + cpy.R.Set(tx.R) + } + if tx.S != nil { + cpy.S.Set(tx.S) + } + return cpy +} + +// accessors for innerTx. +func (tx *SetCodeTx) txType() byte { return SetCodeTxType } +func (tx *SetCodeTx) chainID() *big.Int { return big.NewInt(int64(tx.ChainID)) } +func (tx *SetCodeTx) accessList() AccessList { return tx.AccessList } +func (tx *SetCodeTx) data() []byte { return tx.Data } +func (tx *SetCodeTx) gas() uint64 { return tx.Gas } +func (tx *SetCodeTx) gasFeeCap() *big.Int { return tx.GasFeeCap.ToBig() } +func (tx *SetCodeTx) gasTipCap() *big.Int { return tx.GasTipCap.ToBig() } +func (tx *SetCodeTx) gasPrice() *big.Int { return tx.GasFeeCap.ToBig() } +func (tx *SetCodeTx) value() *big.Int { return tx.Value.ToBig() } +func (tx *SetCodeTx) nonce() uint64 { return tx.Nonce } +func (tx *SetCodeTx) to() *common.Address { tmp := tx.To; return &tmp } + +func (tx *SetCodeTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { + if baseFee == nil { + return dst.Set(tx.GasFeeCap.ToBig()) + } + tip := dst.Sub(tx.GasFeeCap.ToBig(), baseFee) + if tip.Cmp(tx.GasTipCap.ToBig()) > 0 { + tip.Set(tx.GasTipCap.ToBig()) + } + return tip.Add(tip, baseFee) +} + +func (tx *SetCodeTx) rawSignatureValues() (v, r, s *big.Int) { + return tx.V.ToBig(), tx.R.ToBig(), tx.S.ToBig() +} + +func (tx *SetCodeTx) setSignatureValues(chainID, v, r, s *big.Int) { + tx.ChainID = chainID.Uint64() + tx.V.SetFromBig(v) + tx.R.SetFromBig(r) + tx.S.SetFromBig(s) +} + +func (tx *SetCodeTx) encode(b *bytes.Buffer) error { + return rlp.Encode(b, tx) +} + +func (tx *SetCodeTx) decode(input []byte) error { + return rlp.DecodeBytes(input, tx) +} diff --git a/core/types/tx_setcode_test.go b/core/types/tx_setcode_test.go new file mode 100644 index 0000000000..d0544573cf --- /dev/null +++ b/core/types/tx_setcode_test.go @@ -0,0 +1,70 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +// TestParseDelegation tests a few possible delegation designator values and +// ensures they are parsed correctly. +func TestParseDelegation(t *testing.T) { + addr := common.Address{0x42} + for _, tt := range []struct { + val []byte + want *common.Address + }{ + { // simple correct delegation + val: append(DelegationPrefix, addr.Bytes()...), + want: &addr, + }, + { // wrong address size + val: append(DelegationPrefix, addr.Bytes()[0:19]...), + }, + { // short address + val: append(DelegationPrefix, 0x42), + }, + { // long address + val: append(append(DelegationPrefix, addr.Bytes()...), 0x42), + }, + { // wrong prefix size + val: append(DelegationPrefix[:2], addr.Bytes()...), + }, + { // wrong prefix + val: append([]byte{0xef, 0x01, 0x01}, addr.Bytes()...), + }, + { // wrong prefix + val: append([]byte{0xef, 0x00, 0x00}, addr.Bytes()...), + }, + { // no prefix + val: addr.Bytes(), + }, + { // no address + val: DelegationPrefix, + }, + } { + got, ok := ParseDelegation(tt.val) + if ok && tt.want == nil { + t.Fatalf("expected fail, got %s", got.Hex()) + } + if !ok && tt.want != nil { + t.Fatalf("failed to parse, want %s", tt.want.Hex()) + } + } +} diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index 45b317d3c0..5088231207 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -83,12 +83,12 @@ var ( func TestProcessVerkle(t *testing.T) { var ( code = common.FromHex(`6060604052600a8060106000396000f360606040526008565b00`) - intrinsicContractCreationGas, _ = IntrinsicGas(code, nil, true, true, true, true) + intrinsicContractCreationGas, _ = IntrinsicGas(code, nil, nil, true, true, true, true) // A contract creation that calls EXTCODECOPY in the constructor. Used to ensure that the witness // will not contain that copied data. // Source: https://gist.github.com/gballet/a23db1e1cb4ed105616b5920feb75985 codeWithExtCodeCopy = common.FromHex(`0x60806040526040516100109061017b565b604051809103906000f08015801561002c573d6000803e3d6000fd5b506000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555034801561007857600080fd5b5060008067ffffffffffffffff8111156100955761009461024a565b5b6040519080825280601f01601f1916602001820160405280156100c75781602001600182028036833780820191505090505b50905060008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690506020600083833c81610101906101e3565b60405161010d90610187565b61011791906101a3565b604051809103906000f080158015610133573d6000803e3d6000fd5b50600160006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550505061029b565b60d58061046783390190565b6102068061053c83390190565b61019d816101d9565b82525050565b60006020820190506101b86000830184610194565b92915050565b6000819050602082019050919050565b600081519050919050565b6000819050919050565b60006101ee826101ce565b826101f8846101be565b905061020381610279565b925060208210156102435761023e7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8360200360080261028e565b831692505b5050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600061028582516101d9565b80915050919050565b600082821b905092915050565b6101bd806102aa6000396000f3fe608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f566852414610030575b600080fd5b61003861004e565b6040516100459190610146565b60405180910390f35b6000600160009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff166381ca91d36040518163ffffffff1660e01b815260040160206040518083038186803b1580156100b857600080fd5b505afa1580156100cc573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906100f0919061010a565b905090565b60008151905061010481610170565b92915050565b6000602082840312156101205761011f61016b565b5b600061012e848285016100f5565b91505092915050565b61014081610161565b82525050565b600060208201905061015b6000830184610137565b92915050565b6000819050919050565b600080fd5b61017981610161565b811461018457600080fd5b5056fea2646970667358221220a6a0e11af79f176f9c421b7b12f441356b25f6489b83d38cc828a701720b41f164736f6c63430008070033608060405234801561001057600080fd5b5060b68061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063ab5ed15014602d575b600080fd5b60336047565b604051603e9190605d565b60405180910390f35b60006001905090565b6057816076565b82525050565b6000602082019050607060008301846050565b92915050565b600081905091905056fea26469706673582212203a14eb0d5cd07c277d3e24912f110ddda3e553245a99afc4eeefb2fbae5327aa64736f6c63430008070033608060405234801561001057600080fd5b5060405161020638038061020683398181016040528101906100329190610063565b60018160001c6100429190610090565b60008190555050610145565b60008151905061005d8161012e565b92915050565b60006020828403121561007957610078610129565b5b60006100878482850161004e565b91505092915050565b600061009b826100f0565b91506100a6836100f0565b9250827fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff038211156100db576100da6100fa565b5b828201905092915050565b6000819050919050565b6000819050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b600080fd5b610137816100e6565b811461014257600080fd5b50565b60b3806101536000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806381ca91d314602d575b600080fd5b60336047565b604051603e9190605a565b60405180910390f35b60005481565b6054816073565b82525050565b6000602082019050606d6000830184604d565b92915050565b600081905091905056fea26469706673582212209bff7098a2f526de1ad499866f27d6d0d6f17b74a413036d6063ca6a0998ca4264736f6c63430008070033`) - intrinsicCodeWithExtCodeCopyGas, _ = IntrinsicGas(codeWithExtCodeCopy, nil, true, true, true, true) + intrinsicCodeWithExtCodeCopyGas, _ = IntrinsicGas(codeWithExtCodeCopy, nil, nil, true, true, true, true) signer = types.LatestSigner(testVerkleChainConfig) testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") bcdb = rawdb.NewMemoryDatabase() // Database for the blockchain diff --git a/core/vm/eips.go b/core/vm/eips.go index 71d51f81ef..a51a18dc60 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -23,6 +23,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" ) @@ -40,6 +42,7 @@ var activators = map[int]func(*JumpTable){ 1344: enable1344, 1153: enable1153, 4762: enable4762, + 7702: enable7702, } // EnableEIP enables the given EIP on the config. @@ -703,3 +706,68 @@ func enableEOF(jt *JumpTable) { memorySize: memoryExtCall, } } + +// opExtCodeCopyEIP7702 implements the EIP-7702 variation of opExtCodeCopy. +func opExtCodeCopyEIP7702(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + var ( + stack = scope.Stack + a = stack.pop() + memOffset = stack.pop() + codeOffset = stack.pop() + length = stack.pop() + ) + uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow() + if overflow { + uint64CodeOffset = math.MaxUint64 + } + code := interpreter.evm.StateDB.GetCode(common.Address(a.Bytes20())) + if _, ok := types.ParseDelegation(code); ok { + code = types.DelegationPrefix[:2] + } + codeCopy := getData(code, uint64CodeOffset, length.Uint64()) + scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) + + return nil, nil +} + +// opExtCodeSizeEIP7702 implements the EIP-7702 variation of opExtCodeSize. +func opExtCodeSizeEIP7702(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + slot := scope.Stack.peek() + code := interpreter.evm.StateDB.GetCode(common.Address(slot.Bytes20())) + if _, ok := types.ParseDelegation(code); ok { + code = types.DelegationPrefix[:2] + } + slot.SetUint64(uint64(len(code))) + return nil, nil +} + +// opExtCodeHashEIP7702 implements the EIP-7702 variation of opExtCodeHash. +func opExtCodeHashEIP7702(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + slot := scope.Stack.peek() + addr := common.Address(slot.Bytes20()) + if interpreter.evm.StateDB.Empty(addr) { + slot.Clear() + return nil, nil + } + code := interpreter.evm.StateDB.GetCode(addr) + if _, ok := types.ParseDelegation(code); ok { + // If the code is a delegation, return the prefix without version. + slot.SetBytes(crypto.Keccak256(types.DelegationPrefix[:2])) + } else { + // Otherwise, return normal code hash. + slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(addr).Bytes()) + } + return nil, nil +} + +// enable7702 the EIP-7702 changes to support delegation designators. +func enable7702(jt *JumpTable) { + jt[EXTCODECOPY].execute = opExtCodeCopyEIP7702 + jt[EXTCODESIZE].execute = opExtCodeSizeEIP7702 + jt[EXTCODEHASH].execute = opExtCodeHashEIP7702 + + jt[CALL].dynamicGas = gasCallEIP7702 + jt[CALLCODE].dynamicGas = gasCallCodeEIP7702 + jt[STATICCALL].dynamicGas = gasStaticCallEIP7702 + jt[DELEGATECALL].dynamicGas = gasDelegateCallEIP7702 +} diff --git a/core/vm/eof_validation_test.go b/core/vm/eof_validation_test.go index 6680ca3a5d..afb856a32c 100644 --- a/core/vm/eof_validation_test.go +++ b/core/vm/eof_validation_test.go @@ -251,7 +251,7 @@ func TestValidateCode(t *testing.T) { data: make([]byte, 0), subContainers: make([]*Container, 0), } - _, err := validateCode(test.code, test.section, container, &pragueEOFInstructionSet, false) + _, err := validateCode(test.code, test.section, container, &eofInstructionSet, false) if !errors.Is(err, test.err) { t.Errorf("test %d (%s): unexpected error (want: %v, got: %v)", i, common.Bytes2Hex(test.code), test.err, err) } @@ -277,7 +277,7 @@ func BenchmarkRJUMPI(b *testing.B) { } b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := validateCode(code, 0, container, &pragueEOFInstructionSet, false) + _, err := validateCode(code, 0, container, &eofInstructionSet, false) if err != nil { b.Fatal(err) } @@ -309,7 +309,7 @@ func BenchmarkRJUMPV(b *testing.B) { } b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := validateCode(code, 0, container, &pragueEOFInstructionSet, false) + _, err := validateCode(code, 0, container, &pragueInstructionSet, false) if err != nil { b.Fatal(err) } @@ -357,7 +357,7 @@ func BenchmarkEOFValidation(b *testing.B) { if err := container2.UnmarshalBinary(bin, true); err != nil { b.Fatal(err) } - if err := container2.ValidateCode(&pragueEOFInstructionSet, false); err != nil { + if err := container2.ValidateCode(&pragueInstructionSet, false); err != nil { b.Fatal(err) } } @@ -412,7 +412,7 @@ func BenchmarkEOFValidation2(b *testing.B) { if err := container2.UnmarshalBinary(bin, true); err != nil { b.Fatal(err) } - if err := container2.ValidateCode(&pragueEOFInstructionSet, false); err != nil { + if err := container2.ValidateCode(&pragueInstructionSet, false); err != nil { b.Fatal(err) } } @@ -468,7 +468,7 @@ func BenchmarkEOFValidation3(b *testing.B) { if err := container2.UnmarshalBinary(bin, true); err != nil { b.Fatal(err) } - if err := container2.ValidateCode(&pragueEOFInstructionSet, false); err != nil { + if err := container2.ValidateCode(&pragueInstructionSet, false); err != nil { b.Fatal(err) } } @@ -494,7 +494,7 @@ func BenchmarkRJUMPI_2(b *testing.B) { } b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := validateCode(code, 0, container, &pragueEOFInstructionSet, false) + _, err := validateCode(code, 0, container, &pragueInstructionSet, false) if err != nil { b.Fatal(err) } @@ -512,6 +512,6 @@ func FuzzValidate(f *testing.F) { f.Fuzz(func(_ *testing.T, code []byte, maxStack uint16) { var container Container container.types = append(container.types, &functionMetadata{inputs: 0, outputs: 0x80, maxStackHeight: maxStack}) - validateCode(code, 0, &container, &pragueEOFInstructionSet, true) + validateCode(code, 0, &container, &pragueInstructionSet, true) }) } diff --git a/core/vm/evm.go b/core/vm/evm.go index 07e4a272fa..1a0215459c 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -217,7 +217,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas } else { // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. - code := evm.StateDB.GetCode(addr) + code := evm.resolveCode(addr) if len(code) == 0 { ret, err = nil, nil // gas is unchanged } else { @@ -225,7 +225,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas // If the account has no code, we can abort here // The depth-check is already done, and precompiles handled above contract := NewContract(caller, AccountRef(addrCopy), value, gas) - contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), code) + contract.SetCallCode(&addrCopy, evm.resolveCodeHash(addrCopy), code) ret, err = evm.interpreter.Run(contract, input, false) gas = contract.Gas } @@ -285,7 +285,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. contract := NewContract(caller, AccountRef(caller.Address()), value, gas) - contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy)) + contract.SetCallCode(&addrCopy, evm.resolveCodeHash(addrCopy), evm.resolveCode(addrCopy)) ret, err = evm.interpreter.Run(contract, input, false) gas = contract.Gas } @@ -332,7 +332,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by addrCopy := addr // Initialise a new contract and make initialise the delegate values contract := NewContract(caller, AccountRef(caller.Address()), nil, gas).AsDelegate() - contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy)) + contract.SetCallCode(&addrCopy, evm.resolveCodeHash(addrCopy), evm.resolveCode(addrCopy)) ret, err = evm.interpreter.Run(contract, input, false) gas = contract.Gas } @@ -387,7 +387,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. contract := NewContract(caller, AccountRef(addrCopy), new(uint256.Int), gas) - contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy)) + contract.SetCallCode(&addrCopy, evm.resolveCodeHash(addrCopy), evm.resolveCode(addrCopy)) // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally // when we're in Homestead this also counts for code storage gas errors. @@ -567,6 +567,35 @@ func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment * return evm.create(caller, codeAndHash, gas, endowment, contractAddr, CREATE2) } +// resolveCode returns the code associated with the provided account. After +// Prague, it can also resolve code pointed to by a delegation designator. +func (evm *EVM) resolveCode(addr common.Address) []byte { + code := evm.StateDB.GetCode(addr) + if !evm.chainRules.IsPrague { + return code + } + if target, ok := types.ParseDelegation(code); ok { + // Note we only follow one level of delegation. + return evm.StateDB.GetCode(target) + } + return code +} + +// resolveCodeHash returns the code hash associated with the provided address. +// After Prague, it can also resolve code hash of the account pointed to by a +// delegation designator. Although this is not accessible in the EVM it is used +// internally to associate jumpdest analysis to code. +func (evm *EVM) resolveCodeHash(addr common.Address) common.Hash { + if evm.chainRules.IsPrague { + code := evm.StateDB.GetCode(addr) + if target, ok := types.ParseDelegation(code); ok { + // Note we only follow one level of delegation. + return evm.StateDB.GetCodeHash(target) + } + } + return evm.StateDB.GetCodeHash(addr) +} + // ChainConfig returns the environment's chain configuration func (evm *EVM) ChainConfig() *params.ChainConfig { return evm.chainConfig } diff --git a/core/vm/interface.go b/core/vm/interface.go index 9229f4d2cd..011541dde3 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -42,7 +42,9 @@ type StateDB interface { GetCodeHash(common.Address) common.Hash GetCode(common.Address) []byte - SetCode(common.Address, []byte) + + // SetCode sets the new code for the address, and returns the previous code, if any. + SetCode(common.Address, []byte) []byte GetCodeSize(common.Address) int AddRefund(uint64) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index c408994401..996ed6e56a 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -109,6 +109,8 @@ func NewEVMInterpreter(evm *EVM) *EVMInterpreter { case evm.chainRules.IsVerkle: // TODO replace with proper instruction set when fork is specified table = &verkleInstructionSet + case evm.chainRules.IsPrague: + table = &pragueInstructionSet case evm.chainRules.IsCancun: table = &cancunInstructionSet case evm.chainRules.IsShanghai: diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index 658014f24c..6610fa7f9a 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -61,7 +61,8 @@ var ( shanghaiInstructionSet = newShanghaiInstructionSet() cancunInstructionSet = newCancunInstructionSet() verkleInstructionSet = newVerkleInstructionSet() - pragueEOFInstructionSet = newPragueEOFInstructionSet() + pragueInstructionSet = newPragueInstructionSet() + eofInstructionSet = newEOFInstructionSetForTesting() ) // JumpTable contains the EVM opcodes supported at a given fork. @@ -91,16 +92,22 @@ func newVerkleInstructionSet() JumpTable { return validate(instructionSet) } -func NewPragueEOFInstructionSetForTesting() JumpTable { - return newPragueEOFInstructionSet() +func NewEOFInstructionSetForTesting() JumpTable { + return newEOFInstructionSetForTesting() } -func newPragueEOFInstructionSet() JumpTable { - instructionSet := newCancunInstructionSet() +func newEOFInstructionSetForTesting() JumpTable { + instructionSet := newPragueInstructionSet() enableEOF(&instructionSet) return validate(instructionSet) } +func newPragueInstructionSet() JumpTable { + instructionSet := newCancunInstructionSet() + enable7702(&instructionSet) // EIP-7702 Setcode transaction type + return validate(instructionSet) +} + func newCancunInstructionSet() JumpTable { instructionSet := newShanghaiInstructionSet() enable4844(&instructionSet) // EIP-4844 (BLOBHASH opcode) diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index b993b651ff..ff3875868f 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" ) @@ -242,3 +243,70 @@ func makeSelfdestructGasFn(refundsEnabled bool) gasFunc { } return gasFunc } + +var ( + gasCallEIP7702 = makeCallVariantGasCallEIP7702(gasCall) + gasDelegateCallEIP7702 = makeCallVariantGasCallEIP7702(gasDelegateCall) + gasStaticCallEIP7702 = makeCallVariantGasCallEIP7702(gasStaticCall) + gasCallCodeEIP7702 = makeCallVariantGasCallEIP7702(gasCallCode) +) + +func makeCallVariantGasCallEIP7702(oldCalculator gasFunc) gasFunc { + return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + var ( + total uint64 // total dynamic gas used + addr = common.Address(stack.Back(1).Bytes20()) + ) + + // Check slot presence in the access list + if !evm.StateDB.AddressInAccessList(addr) { + evm.StateDB.AddAddressToAccessList(addr) + // The WarmStorageReadCostEIP2929 (100) is already deducted in the form of a constant cost, so + // the cost to charge for cold access, if any, is Cold - Warm + coldCost := params.ColdAccountAccessCostEIP2929 - params.WarmStorageReadCostEIP2929 + // Charge the remaining difference here already, to correctly calculate available + // gas for call + if !contract.UseGas(coldCost, evm.Config.Tracer, tracing.GasChangeCallStorageColdAccess) { + return 0, ErrOutOfGas + } + total += coldCost + } + + // Check if code is a delegation and if so, charge for resolution. + if target, ok := types.ParseDelegation(evm.StateDB.GetCode(addr)); ok { + var cost uint64 + if evm.StateDB.AddressInAccessList(target) { + cost = params.WarmStorageReadCostEIP2929 + } else { + evm.StateDB.AddAddressToAccessList(target) + cost = params.ColdAccountAccessCostEIP2929 + } + if !contract.UseGas(cost, evm.Config.Tracer, tracing.GasChangeCallStorageColdAccess) { + return 0, ErrOutOfGas + } + total += cost + } + + // Now call the old calculator, which takes into account + // - create new account + // - transfer value + // - memory expansion + // - 63/64ths rule + old, err := oldCalculator(evm, contract, stack, mem, memorySize) + if err != nil { + return old, err + } + + // Temporarily add the gas charge back to the contract and return value. By + // adding it to the return, it will be charged outside of this function, as + // part of the dynamic gas. This will ensure it is correctly reported to + // tracers. + contract.Gas += total + + var overflow bool + if total, overflow = math.SafeAdd(old, total); overflow { + return 0, ErrGasUintOverflow + } + return total, nil + } +} diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 0e774a01c2..6074e9a096 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -866,3 +866,83 @@ func BenchmarkTracerStepVsCallFrame(b *testing.B) { benchmarkNonModifyingCode(10000000, code, "tracer-step-10M", stepTracer, b) benchmarkNonModifyingCode(10000000, code, "tracer-call-frame-10M", callFrameTracer, b) } + +// TestDelegatedAccountAccessCost tests that calling an account with an EIP-7702 +// delegation designator incurs the correct amount of gas based on the tracer. +func TestDelegatedAccountAccessCost(t *testing.T) { + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) + statedb.SetCode(common.HexToAddress("0xff"), types.AddressToDelegation(common.HexToAddress("0xaa"))) + statedb.SetCode(common.HexToAddress("0xaa"), program.New().Return(0, 0).Bytes()) + + for i, tc := range []struct { + code []byte + step int + want uint64 + }{ + { // CALL(0xff) + code: []byte{ + byte(vm.PUSH1), 0x0, + byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xff, byte(vm.DUP1), byte(vm.CALL), byte(vm.POP), + }, + step: 7, + want: 5455, + }, + { // CALLCODE(0xff) + code: []byte{ + byte(vm.PUSH1), 0x0, + byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xff, byte(vm.DUP1), byte(vm.CALLCODE), byte(vm.POP), + }, + step: 7, + want: 5455, + }, + { // DELEGATECALL(0xff) + code: []byte{ + byte(vm.PUSH1), 0x0, + byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xff, byte(vm.DUP1), byte(vm.DELEGATECALL), byte(vm.POP), + }, + step: 6, + want: 5455, + }, + { // STATICCALL(0xff) + code: []byte{ + byte(vm.PUSH1), 0x0, + byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xff, byte(vm.DUP1), byte(vm.STATICCALL), byte(vm.POP), + }, + step: 6, + want: 5455, + }, + { // SELFDESTRUCT(0xff): should not be affected by resolution + code: []byte{ + byte(vm.PUSH1), 0xff, byte(vm.SELFDESTRUCT), + }, + step: 1, + want: 7600, + }, + } { + var step = 0 + var have = uint64(0) + Execute(tc.code, nil, &Config{ + ChainConfig: params.MergedTestChainConfig, + State: statedb, + EVMConfig: vm.Config{ + Tracer: &tracing.Hooks{ + OnOpcode: func(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { + // Uncomment to investigate failures: + t.Logf("%d: %v %d", step, vm.OpCode(op).String(), cost) + if step == tc.step { + have = cost + } + step++ + }, + }, + }, + }) + if want := tc.want; have != want { + t.Fatalf("testcase %d, gas report wrong, step %d, have %d want %d", i, tc.step, have, want) + } + } +} diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer/setcode_tx.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer/setcode_tx.json new file mode 100644 index 0000000000..b7d5ee1c54 --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer/setcode_tx.json @@ -0,0 +1,82 @@ +{ + "genesis": { + "baseFeePerGas": "7", + "blobGasUsed": "0", + "difficulty": "0", + "excessBlobGas": "36306944", + "extraData": "0xd983010e00846765746888676f312e32312e308664617277696e", + "gasLimit": "15639172", + "hash": "0xc682259fda061bb9ce8ccb491d5b2d436cb73daf04e1025dd116d045ce4ad28c", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0xae1a5ba939a4c9ac38aabeff361169fb55a6fc2c9511457e0be6eff9514faec0", + "nonce": "0x0000000000000000", + "number": "315", + "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "stateRoot": "0x577f42ab21ccfd946511c57869ace0bdf7c217c36f02b7cd3459df0ed1cffc1a", + "timestamp": "1709626771", + "withdrawals": [], + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "alloc": { + "0x0000000000000000000000000000000000000000": { + "balance": "0x272e0528" + }, + "0x000000000000000000000000000000000000aaaa": { + "code": "0x6000600060006000600173703c4b2bd70c169f5717101caee543299fc946c75af1", + "balance": "0x0" + }, + "0x000000000000000000000000000000000000bbbb": { + "code": "0x6042604255", + "balance": "0x0" + }, + "0x703c4b2bd70c169f5717101caee543299fc946c7": { + "balance": "0xde0b6b3a7640000" + }, + "0x71562b71999873db5b286df957af199ec94617f7": { + "balance": "0xde0b6b3a7640000" + } + }, + "config": { + "chainId": 1337, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "shanghaiTime": 0, + "cancunTime": 0, + "pragueTime": 0, + "terminalTotalDifficulty": 0 + } + }, + "context": { + "number": "316", + "difficulty": "0", + "timestamp": "1709626785", + "gasLimit": "15654443", + "miner": "0x0000000000000000000000000000000000000000", + "baseFeePerGas": "7" + }, + "input": "04f90126820539800285012a05f2008307a1209471562b71999873db5b286df957af199ec94617f78080c0f8baf85c82053994000000000000000000000000000000000000aaaa0101a07ed17af7d2d2b9ba7d797a202125bf505b9a0f962a67b3b61b56783d8faf7461a001b73b6e586edc706dce6c074eaec28692fa6359fb3446a2442f36777e1c0669f85a8094000000000000000000000000000000000000bbbb8001a05011890f198f0356a887b0779bde5afa1ed04e6acb1e3f37f8f18c7b6f521b98a056c3fa3456b103f3ef4a0acb4b647b9cab9ec4bc68fbcdf1e10b49fb2bcbcf6101a0167b0ecfc343a497095c22ee4270d3cc3b971cc3599fc73bbff727e0d2ed432da01c003c72306807492bf1150e39b2f79da23b49a4e83eb6e9209ae30d3572368f", + "result": { + "0x0000000000000000000000000000000000000000": { + "balance": "0x272e0528" + }, + "0x703c4b2bd70c169f5717101caee543299fc946c7": { + "balance": "0xde0b6b3a7640000", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000042": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "0x71562b71999873db5b286df957af199ec94617f7": { + "balance": "0xde0b6b3a7640000" + } + } +} diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index 9706eb43f6..5776275c2b 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -159,6 +159,15 @@ func (t *prestateTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction t.lookupAccount(from) t.lookupAccount(t.to) t.lookupAccount(env.Coinbase) + + // Add accounts with authorizations to the prestate before they get applied. + for _, auth := range tx.AuthList() { + addr, err := auth.Authority() + if err != nil { + continue + } + t.lookupAccount(addr) + } } func (t *prestateTracer) OnTxEnd(receipt *types.Receipt, err error) { diff --git a/graphql/graphql.go b/graphql/graphql.go index 6e364de6d9..7af1adbb4a 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -318,7 +318,7 @@ func (t *Transaction) MaxFeePerGas(ctx context.Context) *hexutil.Big { return nil } switch tx.Type() { - case types.DynamicFeeTxType, types.BlobTxType: + case types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType: return (*hexutil.Big)(tx.GasFeeCap()) default: return nil @@ -331,7 +331,7 @@ func (t *Transaction) MaxPriorityFeePerGas(ctx context.Context) *hexutil.Big { return nil } switch tx.Type() { - case types.DynamicFeeTxType, types.BlobTxType: + case types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType: return (*hexutil.Big)(tx.GasTipCap()) default: return nil diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index de75c697ef..3f9ce46ddb 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -937,28 +937,29 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, config *param // RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction type RPCTransaction struct { - BlockHash *common.Hash `json:"blockHash"` - BlockNumber *hexutil.Big `json:"blockNumber"` - From common.Address `json:"from"` - Gas hexutil.Uint64 `json:"gas"` - GasPrice *hexutil.Big `json:"gasPrice"` - GasFeeCap *hexutil.Big `json:"maxFeePerGas,omitempty"` - GasTipCap *hexutil.Big `json:"maxPriorityFeePerGas,omitempty"` - MaxFeePerBlobGas *hexutil.Big `json:"maxFeePerBlobGas,omitempty"` - Hash common.Hash `json:"hash"` - Input hexutil.Bytes `json:"input"` - Nonce hexutil.Uint64 `json:"nonce"` - To *common.Address `json:"to"` - TransactionIndex *hexutil.Uint64 `json:"transactionIndex"` - Value *hexutil.Big `json:"value"` - Type hexutil.Uint64 `json:"type"` - Accesses *types.AccessList `json:"accessList,omitempty"` - ChainID *hexutil.Big `json:"chainId,omitempty"` - BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` - V *hexutil.Big `json:"v"` - R *hexutil.Big `json:"r"` - S *hexutil.Big `json:"s"` - YParity *hexutil.Uint64 `json:"yParity,omitempty"` + BlockHash *common.Hash `json:"blockHash"` + BlockNumber *hexutil.Big `json:"blockNumber"` + From common.Address `json:"from"` + Gas hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + GasFeeCap *hexutil.Big `json:"maxFeePerGas,omitempty"` + GasTipCap *hexutil.Big `json:"maxPriorityFeePerGas,omitempty"` + MaxFeePerBlobGas *hexutil.Big `json:"maxFeePerBlobGas,omitempty"` + Hash common.Hash `json:"hash"` + Input hexutil.Bytes `json:"input"` + Nonce hexutil.Uint64 `json:"nonce"` + To *common.Address `json:"to"` + TransactionIndex *hexutil.Uint64 `json:"transactionIndex"` + Value *hexutil.Big `json:"value"` + Type hexutil.Uint64 `json:"type"` + Accesses *types.AccessList `json:"accessList,omitempty"` + ChainID *hexutil.Big `json:"chainId,omitempty"` + BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` + AuthorizationList []types.Authorization `json:"authorizationList,omitempty"` + V *hexutil.Big `json:"v"` + R *hexutil.Big `json:"r"` + S *hexutil.Big `json:"s"` + YParity *hexutil.Uint64 `json:"yParity,omitempty"` } // newRPCTransaction returns a transaction that will serialize to the RPC @@ -1033,6 +1034,22 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber } result.MaxFeePerBlobGas = (*hexutil.Big)(tx.BlobGasFeeCap()) result.BlobVersionedHashes = tx.BlobHashes() + + case types.SetCodeTxType: + al := tx.AccessList() + yparity := hexutil.Uint64(v.Sign()) + result.Accesses = &al + result.ChainID = (*hexutil.Big)(tx.ChainId()) + result.YParity = &yparity + result.GasFeeCap = (*hexutil.Big)(tx.GasFeeCap()) + result.GasTipCap = (*hexutil.Big)(tx.GasTipCap()) + // if the transaction has been mined, compute the effective gas price + if baseFee != nil && blockHash != (common.Hash{}) { + result.GasPrice = (*hexutil.Big)(effectiveGasPrice(tx, baseFee)) + } else { + result.GasPrice = (*hexutil.Big)(tx.GasFeeCap()) + } + result.AuthorizationList = tx.AuthList() } return result } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index ae2ec9f0f0..0303a0a6ea 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -629,12 +629,13 @@ func TestEstimateGas(t *testing.T) { t.Parallel() // Initialize test accounts var ( - accounts = newAccounts(2) + accounts = newAccounts(4) genesis = &core.Genesis{ Config: params.MergedTestChainConfig, Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + accounts[2].addr: {Balance: big.NewInt(params.Ether), Code: append(types.DelegationPrefix, accounts[3].addr.Bytes()...)}, }, } genBlocks = 10 @@ -819,6 +820,26 @@ func TestEstimateGas(t *testing.T) { blockOverrides: override.BlockOverrides{Number: (*hexutil.Big)(big.NewInt(11))}, expectErr: newRevertError(packRevert("block 11")), }, + // Should be able to send to an EIP-7702 delegated account. + { + blockNumber: rpc.LatestBlockNumber, + call: TransactionArgs{ + From: &accounts[0].addr, + To: &accounts[2].addr, + Value: (*hexutil.Big)(big.NewInt(1)), + }, + want: 21000, + }, + // Should be able to send as EIP-7702 delegated account. + { + blockNumber: rpc.LatestBlockNumber, + call: TransactionArgs{ + From: &accounts[2].addr, + To: &accounts[1].addr, + Value: (*hexutil.Big)(big.NewInt(1)), + }, + want: 21000, + }, } for i, tc := range testSuite { result, err := api.EstimateGas(context.Background(), tc.call, &rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, &tc.overrides, &tc.blockOverrides) diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 2a0508b147..3942540b04 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -72,6 +72,9 @@ type TransactionArgs struct { Commitments []kzg4844.Commitment `json:"commitments"` Proofs []kzg4844.Proof `json:"proofs"` + // For SetCodeTxType + AuthorizationList []types.Authorization `json:"authorizationList"` + // This configures whether blobs are allowed to be passed. blobSidecarAllowed bool } @@ -473,6 +476,8 @@ func (args *TransactionArgs) ToMessage(baseFee *big.Int, skipNonceCheck, skipEoA func (args *TransactionArgs) ToTransaction(defaultType int) *types.Transaction { usedType := types.LegacyTxType switch { + case args.AuthorizationList != nil || defaultType == types.SetCodeTxType: + usedType = types.SetCodeTxType case args.BlobHashes != nil || defaultType == types.BlobTxType: usedType = types.BlobTxType case args.MaxFeePerGas != nil || defaultType == types.DynamicFeeTxType: @@ -486,6 +491,28 @@ func (args *TransactionArgs) ToTransaction(defaultType int) *types.Transaction { } var data types.TxData switch usedType { + case types.SetCodeTxType: + al := types.AccessList{} + if args.AccessList != nil { + al = *args.AccessList + } + authList := []types.Authorization{} + if args.AuthorizationList != nil { + authList = args.AuthorizationList + } + data = &types.SetCodeTx{ + To: *args.To, + ChainID: args.ChainID.ToInt().Uint64(), + Nonce: uint64(*args.Nonce), + Gas: uint64(*args.Gas), + GasFeeCap: uint256.MustFromBig((*big.Int)(args.MaxFeePerGas)), + GasTipCap: uint256.MustFromBig((*big.Int)(args.MaxPriorityFeePerGas)), + Value: uint256.MustFromBig((*big.Int)(args.Value)), + Data: args.data(), + AccessList: al, + AuthList: authList, + } + case types.BlobTxType: al := types.AccessList{} if args.AccessList != nil { diff --git a/params/protocol_params.go b/params/protocol_params.go index 90e7487cff..4d2baf8054 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -90,10 +90,11 @@ const ( SelfdestructRefundGas uint64 = 24000 // Refunded following a selfdestruct operation. MemoryGas uint64 = 3 // Times the address of the (highest referenced byte in memory + 1). NOTE: referencing happens on read, write and in instructions such as RETURN and CALL. - TxDataNonZeroGasFrontier uint64 = 68 // Per byte of data attached to a transaction that is not equal to zero. NOTE: Not payable on data of calls between transactions. - TxDataNonZeroGasEIP2028 uint64 = 16 // Per byte of non zero data attached to a transaction after EIP 2028 (part in Istanbul) - TxAccessListAddressGas uint64 = 2400 // Per address specified in EIP 2930 access list - TxAccessListStorageKeyGas uint64 = 1900 // Per storage key specified in EIP 2930 access list + TxDataNonZeroGasFrontier uint64 = 68 // Per byte of data attached to a transaction that is not equal to zero. NOTE: Not payable on data of calls between transactions. + TxDataNonZeroGasEIP2028 uint64 = 16 // Per byte of non zero data attached to a transaction after EIP 2028 (part in Istanbul) + TxAccessListAddressGas uint64 = 2400 // Per address specified in EIP 2930 access list + TxAccessListStorageKeyGas uint64 = 1900 // Per storage key specified in EIP 2930 access list + TxAuthTupleGas uint64 = 12500 // Per auth tuple code specified in EIP-7702 // These have been changed during the course of the chain CallGasFrontier uint64 = 40 // Once per CALL operation & message call transaction. diff --git a/tests/gen_stauthorization.go b/tests/gen_stauthorization.go new file mode 100644 index 0000000000..fbafd6fdea --- /dev/null +++ b/tests/gen_stauthorization.go @@ -0,0 +1,74 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package tests + +import ( + "encoding/json" + "errors" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" +) + +var _ = (*stAuthorizationMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (s stAuthorization) MarshalJSON() ([]byte, error) { + type stAuthorization struct { + ChainID math.HexOrDecimal64 + Address common.Address `json:"address" gencodec:"required"` + Nonce math.HexOrDecimal64 `json:"nonce" gencodec:"required"` + V math.HexOrDecimal64 `json:"v" gencodec:"required"` + R *math.HexOrDecimal256 `json:"r" gencodec:"required"` + S *math.HexOrDecimal256 `json:"s" gencodec:"required"` + } + var enc stAuthorization + enc.ChainID = math.HexOrDecimal64(s.ChainID) + enc.Address = s.Address + enc.Nonce = math.HexOrDecimal64(s.Nonce) + enc.V = math.HexOrDecimal64(s.V) + enc.R = (*math.HexOrDecimal256)(s.R) + enc.S = (*math.HexOrDecimal256)(s.S) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (s *stAuthorization) UnmarshalJSON(input []byte) error { + type stAuthorization struct { + ChainID *math.HexOrDecimal64 + Address *common.Address `json:"address" gencodec:"required"` + Nonce *math.HexOrDecimal64 `json:"nonce" gencodec:"required"` + V *math.HexOrDecimal64 `json:"v" gencodec:"required"` + R *math.HexOrDecimal256 `json:"r" gencodec:"required"` + S *math.HexOrDecimal256 `json:"s" gencodec:"required"` + } + var dec stAuthorization + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.ChainID != nil { + s.ChainID = uint64(*dec.ChainID) + } + if dec.Address == nil { + return errors.New("missing required field 'address' for stAuthorization") + } + s.Address = *dec.Address + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' for stAuthorization") + } + s.Nonce = uint64(*dec.Nonce) + if dec.V == nil { + return errors.New("missing required field 'v' for stAuthorization") + } + s.V = uint8(*dec.V) + if dec.R == nil { + return errors.New("missing required field 'r' for stAuthorization") + } + s.R = (*big.Int)(dec.R) + if dec.S == nil { + return errors.New("missing required field 's' for stAuthorization") + } + s.S = (*big.Int)(dec.S) + return nil +} diff --git a/tests/gen_sttransaction.go b/tests/gen_sttransaction.go index 9b5aecbfe6..b25ce76166 100644 --- a/tests/gen_sttransaction.go +++ b/tests/gen_sttransaction.go @@ -30,6 +30,7 @@ func (s stTransaction) MarshalJSON() ([]byte, error) { Sender *common.Address `json:"sender"` BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` BlobGasFeeCap *math.HexOrDecimal256 `json:"maxFeePerBlobGas,omitempty"` + AuthorizationList []*stAuthorization `json:"authorizationList,omitempty"` } var enc stTransaction enc.GasPrice = (*math.HexOrDecimal256)(s.GasPrice) @@ -50,6 +51,7 @@ func (s stTransaction) MarshalJSON() ([]byte, error) { enc.Sender = s.Sender enc.BlobVersionedHashes = s.BlobVersionedHashes enc.BlobGasFeeCap = (*math.HexOrDecimal256)(s.BlobGasFeeCap) + enc.AuthorizationList = s.AuthorizationList return json.Marshal(&enc) } @@ -69,6 +71,7 @@ func (s *stTransaction) UnmarshalJSON(input []byte) error { Sender *common.Address `json:"sender"` BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` BlobGasFeeCap *math.HexOrDecimal256 `json:"maxFeePerBlobGas,omitempty"` + AuthorizationList []*stAuthorization `json:"authorizationList,omitempty"` } var dec stTransaction if err := json.Unmarshal(input, &dec); err != nil { @@ -116,5 +119,8 @@ func (s *stTransaction) UnmarshalJSON(input []byte) error { if dec.BlobGasFeeCap != nil { s.BlobGasFeeCap = (*big.Int)(dec.BlobGasFeeCap) } + if dec.AuthorizationList != nil { + s.AuthorizationList = dec.AuthorizationList + } return nil } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 6884ae7ed5..e735ce2fb8 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -123,6 +123,7 @@ type stTransaction struct { Sender *common.Address `json:"sender"` BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` BlobGasFeeCap *big.Int `json:"maxFeePerBlobGas,omitempty"` + AuthorizationList []*stAuthorization `json:"authorizationList,omitempty"` } type stTransactionMarshaling struct { @@ -135,6 +136,27 @@ type stTransactionMarshaling struct { BlobGasFeeCap *math.HexOrDecimal256 } +//go:generate go run github.com/fjl/gencodec -type stAuthorization -field-override stAuthorizationMarshaling -out gen_stauthorization.go + +// Authorization is an authorization from an account to deploy code at it's address. +type stAuthorization struct { + ChainID uint64 + Address common.Address `json:"address" gencodec:"required"` + Nonce uint64 `json:"nonce" gencodec:"required"` + V uint8 `json:"v" gencodec:"required"` + R *big.Int `json:"r" gencodec:"required"` + S *big.Int `json:"s" gencodec:"required"` +} + +// field type overrides for gencodec +type stAuthorizationMarshaling struct { + ChainID math.HexOrDecimal64 + Nonce math.HexOrDecimal64 + V math.HexOrDecimal64 + R *math.HexOrDecimal256 + S *math.HexOrDecimal256 +} + // GetChainConfig takes a fork definition and returns a chain config. // The fork definition can be // - a plain forkname, e.g. `Byzantium`, @@ -419,6 +441,20 @@ func (tx *stTransaction) toMessage(ps stPostState, baseFee *big.Int) (*core.Mess if gasPrice == nil { return nil, errors.New("no gas price provided") } + var authList []types.Authorization + if tx.AuthorizationList != nil { + authList = make([]types.Authorization, len(tx.AuthorizationList)) + for i, auth := range tx.AuthorizationList { + authList[i] = types.Authorization{ + ChainID: auth.ChainID, + Address: auth.Address, + Nonce: auth.Nonce, + V: auth.V, + R: *uint256.MustFromBig(auth.R), + S: *uint256.MustFromBig(auth.S), + } + } + } msg := &core.Message{ From: from, @@ -433,6 +469,7 @@ func (tx *stTransaction) toMessage(ps stPostState, baseFee *big.Int) (*core.Mess AccessList: accessList, BlobHashes: tx.BlobVersionedHashes, BlobGasFeeCap: tx.BlobGasFeeCap, + AuthList: authList, } return msg, nil } diff --git a/tests/transaction_test_util.go b/tests/transaction_test_util.go index d3dbbd5db2..4da27ff943 100644 --- a/tests/transaction_test_util.go +++ b/tests/transaction_test_util.go @@ -59,7 +59,7 @@ func (tt *TransactionTest) Run(config *params.ChainConfig) error { return nil, nil, err } // Intrinsic gas - requiredGas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.To() == nil, isHomestead, isIstanbul, false) + requiredGas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.AuthList(), tx.To() == nil, isHomestead, isIstanbul, false) if err != nil { return nil, nil, err } From bc1ec69008174aa2f5fd3642adeafb0459d2464b Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 16 Dec 2024 21:10:08 +0800 Subject: [PATCH 3/5] trie/pathdb: state iterator (snapshot integration pt 4) (#30654) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In this pull request, the state iterator is implemented. It's mostly a copy-paste from the original state snapshot package, but still has some important changes to highlight here: (a) The iterator for the disk layer consists of a diff iterator and a disk iterator. Originally, the disk layer in the state snapshot was a wrapper around the disk, and its corresponding iterator was also a wrapper around the disk iterator. However, due to structural differences, the disk layer iterator is divided into two parts: - The disk iterator, which traverses the content stored on disk. - The diff iterator, which traverses the aggregated state buffer. Checkout `BinaryIterator` and `FastIterator` for more details. (b) The staleness management is improved in the diffAccountIterator and diffStorageIterator Originally, in the `diffAccountIterator`, the layer’s staleness had to be checked within the Next function to ensure the iterator remained usable. Additionally, a read lock on the associated diff layer was required to first retrieve the account blob. This read lock protection is essential to prevent concurrent map read/write. Afterward, a staleness check was performed to ensure the retrieved data was not outdated. The entire logic can be simplified as follows: a loadAccount callback is provided to retrieve account data. If the corresponding state is immutable (e.g., diff layers in the path database), the staleness check can be skipped, and a single account data retrieval is sufficient. However, if the corresponding state is mutable (e.g., the disk layer in the path database), the callback can operate as follows: ```go func(hash common.Hash) ([]byte, error) { dl.lock.RLock() defer dl.lock.RUnlock() if dl.stale { return nil, errSnapshotStale } return dl.buffer.states.mustAccount(hash) } ``` The callback solution can eliminate the complexity for managing concurrency with the read lock for atomic operation. --- triedb/pathdb/database.go | 12 + triedb/pathdb/holdable_iterator.go | 97 ++ triedb/pathdb/holdable_iterator_test.go | 176 ++++ triedb/pathdb/iterator.go | 369 +++++++ triedb/pathdb/iterator_binary.go | 344 +++++++ triedb/pathdb/iterator_fast.go | 380 ++++++++ triedb/pathdb/iterator_test.go | 1162 +++++++++++++++++++++++ triedb/pathdb/reader.go | 11 + triedb/pathdb/states.go | 28 +- triedb/pathdb/states_test.go | 118 +-- 10 files changed, 2634 insertions(+), 63 deletions(-) create mode 100644 triedb/pathdb/holdable_iterator.go create mode 100644 triedb/pathdb/holdable_iterator_test.go create mode 100644 triedb/pathdb/iterator.go create mode 100644 triedb/pathdb/iterator_binary.go create mode 100644 triedb/pathdb/iterator_fast.go create mode 100644 triedb/pathdb/iterator_test.go diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index cfbdb01c49..914b17de5b 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -555,3 +555,15 @@ func (db *Database) StorageHistory(address common.Address, slot common.Hash, sta func (db *Database) HistoryRange() (uint64, uint64, error) { return historyRange(db.freezer) } + +// AccountIterator creates a new account iterator for the specified root hash and +// seeks to a starting account hash. +func (db *Database) AccountIterator(root common.Hash, seek common.Hash) (AccountIterator, error) { + return newFastAccountIterator(db, root, seek) +} + +// StorageIterator creates a new storage iterator for the specified root hash and +// account. The iterator will be moved to the specific start position. +func (db *Database) StorageIterator(root common.Hash, account common.Hash, seek common.Hash) (StorageIterator, error) { + return newFastStorageIterator(db, root, account, seek) +} diff --git a/triedb/pathdb/holdable_iterator.go b/triedb/pathdb/holdable_iterator.go new file mode 100644 index 0000000000..1f8e6b7068 --- /dev/null +++ b/triedb/pathdb/holdable_iterator.go @@ -0,0 +1,97 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +// holdableIterator is a wrapper of underlying database iterator. It extends +// the basic iterator interface by adding Hold which can hold the element +// locally where the iterator is currently located and serve it up next time. +type holdableIterator struct { + it ethdb.Iterator + key []byte + val []byte + atHeld bool +} + +// newHoldableIterator initializes the holdableIterator with the given iterator. +func newHoldableIterator(it ethdb.Iterator) *holdableIterator { + return &holdableIterator{it: it} +} + +// Hold holds the element locally where the iterator is currently located which +// can be served up next time. +func (it *holdableIterator) Hold() { + if it.it.Key() == nil { + return // nothing to hold + } + it.key = common.CopyBytes(it.it.Key()) + it.val = common.CopyBytes(it.it.Value()) + it.atHeld = false +} + +// Next moves the iterator to the next key/value pair. It returns whether the +// iterator is exhausted. +func (it *holdableIterator) Next() bool { + if !it.atHeld && it.key != nil { + it.atHeld = true + } else if it.atHeld { + it.atHeld = false + it.key = nil + it.val = nil + } + if it.key != nil { + return true // shifted to locally held value + } + return it.it.Next() +} + +// Error returns any accumulated error. Exhausting all the key/value pairs +// is not considered to be an error. +func (it *holdableIterator) Error() error { return it.it.Error() } + +// Release releases associated resources. Release should always succeed and can +// be called multiple times without causing error. +func (it *holdableIterator) Release() { + it.atHeld = false + it.key = nil + it.val = nil + it.it.Release() +} + +// Key returns the key of the current key/value pair, or nil if done. The caller +// should not modify the contents of the returned slice, and its contents may +// change on the next call to Next. +func (it *holdableIterator) Key() []byte { + if it.key != nil { + return it.key + } + return it.it.Key() +} + +// Value returns the value of the current key/value pair, or nil if done. The +// caller should not modify the contents of the returned slice, and its contents +// may change on the next call to Next. +func (it *holdableIterator) Value() []byte { + if it.val != nil { + return it.val + } + return it.it.Value() +} diff --git a/triedb/pathdb/holdable_iterator_test.go b/triedb/pathdb/holdable_iterator_test.go new file mode 100644 index 0000000000..2abc92e154 --- /dev/null +++ b/triedb/pathdb/holdable_iterator_test.go @@ -0,0 +1,176 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" +) + +func TestIteratorHold(t *testing.T) { + // Create the key-value data store + var ( + content = map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"} + order = []string{"k1", "k2", "k3"} + db = rawdb.NewMemoryDatabase() + ) + for key, val := range content { + if err := db.Put([]byte(key), []byte(val)); err != nil { + t.Fatalf("failed to insert item %s:%s into database: %v", key, val, err) + } + } + // Iterate over the database with the given configs and verify the results + it, idx := newHoldableIterator(db.NewIterator(nil, nil)), 0 + + // Nothing should be affected for calling Discard on non-initialized iterator + it.Hold() + + for it.Next() { + if len(content) <= idx { + t.Errorf("more items than expected: checking idx=%d (key %q), expecting len=%d", idx, it.Key(), len(order)) + break + } + if !bytes.Equal(it.Key(), []byte(order[idx])) { + t.Errorf("item %d: key mismatch: have %s, want %s", idx, string(it.Key()), order[idx]) + } + if !bytes.Equal(it.Value(), []byte(content[order[idx]])) { + t.Errorf("item %d: value mismatch: have %s, want %s", idx, string(it.Value()), content[order[idx]]) + } + // Should be safe to call discard multiple times + it.Hold() + it.Hold() + + // Shift iterator to the discarded element + it.Next() + if !bytes.Equal(it.Key(), []byte(order[idx])) { + t.Errorf("item %d: key mismatch: have %s, want %s", idx, string(it.Key()), order[idx]) + } + if !bytes.Equal(it.Value(), []byte(content[order[idx]])) { + t.Errorf("item %d: value mismatch: have %s, want %s", idx, string(it.Value()), content[order[idx]]) + } + + // Discard/Next combo should work always + it.Hold() + it.Next() + if !bytes.Equal(it.Key(), []byte(order[idx])) { + t.Errorf("item %d: key mismatch: have %s, want %s", idx, string(it.Key()), order[idx]) + } + if !bytes.Equal(it.Value(), []byte(content[order[idx]])) { + t.Errorf("item %d: value mismatch: have %s, want %s", idx, string(it.Value()), content[order[idx]]) + } + idx++ + } + if err := it.Error(); err != nil { + t.Errorf("iteration failed: %v", err) + } + if idx != len(order) { + t.Errorf("iteration terminated prematurely: have %d, want %d", idx, len(order)) + } + db.Close() +} + +func TestReopenIterator(t *testing.T) { + var ( + content = map[common.Hash]string{ + common.HexToHash("a1"): "v1", + common.HexToHash("a2"): "v2", + common.HexToHash("a3"): "v3", + common.HexToHash("a4"): "v4", + common.HexToHash("a5"): "v5", + common.HexToHash("a6"): "v6", + } + order = []common.Hash{ + common.HexToHash("a1"), + common.HexToHash("a2"), + common.HexToHash("a3"), + common.HexToHash("a4"), + common.HexToHash("a5"), + common.HexToHash("a6"), + } + db = rawdb.NewMemoryDatabase() + + reopen = func(db ethdb.KeyValueStore, iter *holdableIterator) *holdableIterator { + if !iter.Next() { + iter.Release() + return newHoldableIterator(memorydb.New().NewIterator(nil, nil)) + } + next := iter.Key() + iter.Release() + return newHoldableIterator(db.NewIterator(rawdb.SnapshotAccountPrefix, next[1:])) + } + ) + for key, val := range content { + rawdb.WriteAccountSnapshot(db, key, []byte(val)) + } + checkVal := func(it *holdableIterator, index int) { + if !bytes.Equal(it.Key(), append(rawdb.SnapshotAccountPrefix, order[index].Bytes()...)) { + t.Fatalf("Unexpected data entry key, want %v got %v", order[index], it.Key()) + } + if !bytes.Equal(it.Value(), []byte(content[order[index]])) { + t.Fatalf("Unexpected data entry key, want %v got %v", []byte(content[order[index]]), it.Value()) + } + } + // Iterate over the database with the given configs and verify the results + dbIter := db.NewIterator(rawdb.SnapshotAccountPrefix, nil) + iter, idx := newHoldableIterator(rawdb.NewKeyLengthIterator(dbIter, 1+common.HashLength)), -1 + + idx++ + iter.Next() + checkVal(iter, idx) + + iter = reopen(db, iter) + idx++ + iter.Next() + checkVal(iter, idx) + + // reopen twice + iter = reopen(db, iter) + iter = reopen(db, iter) + idx++ + iter.Next() + checkVal(iter, idx) + + // reopen iterator with held value + iter.Next() + iter.Hold() + iter = reopen(db, iter) + idx++ + iter.Next() + checkVal(iter, idx) + + // reopen twice iterator with held value + iter.Next() + iter.Hold() + iter = reopen(db, iter) + iter = reopen(db, iter) + idx++ + iter.Next() + checkVal(iter, idx) + + // shift to the end and reopen + iter.Next() // the end + iter = reopen(db, iter) + iter.Next() + if iter.Key() != nil { + t.Fatal("Unexpected iterated entry") + } +} diff --git a/triedb/pathdb/iterator.go b/triedb/pathdb/iterator.go new file mode 100644 index 0000000000..980f228cf5 --- /dev/null +++ b/triedb/pathdb/iterator.go @@ -0,0 +1,369 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + "fmt" + "sort" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" +) + +// Iterator is an iterator to step over all the accounts or the specific +// storage in a snapshot which may or may not be composed of multiple layers. +type Iterator interface { + // Next steps the iterator forward one element, returning false if exhausted, + // or an error if iteration failed for some reason (e.g. root being iterated + // becomes stale and garbage collected). + Next() bool + + // Error returns any failure that occurred during iteration, which might have + // caused a premature iteration exit (e.g. layer stack becoming stale). + Error() error + + // Hash returns the hash of the account or storage slot the iterator is + // currently at. + Hash() common.Hash + + // Release releases associated resources. Release should always succeed and + // can be called multiple times without causing error. + Release() +} + +// AccountIterator is an iterator to step over all the accounts in a snapshot, +// which may or may not be composed of multiple layers. +type AccountIterator interface { + Iterator + + // Account returns the RLP encoded slim account the iterator is currently at. + // An error will be returned if the iterator becomes invalid + Account() []byte +} + +// StorageIterator is an iterator to step over the specific storage in a snapshot, +// which may or may not be composed of multiple layers. +type StorageIterator interface { + Iterator + + // Slot returns the storage slot the iterator is currently at. An error will + // be returned if the iterator becomes invalid + Slot() []byte +} + +type ( + // loadAccount is the function to retrieve the account from the associated + // layer. An error will be returned if the associated layer is stale. + loadAccount func(hash common.Hash) ([]byte, error) + + // loadStorage is the function to retrieve the storage slot from the associated + // layer. An error will be returned if the associated layer is stale. + loadStorage func(addrHash common.Hash, slotHash common.Hash) ([]byte, error) +) + +// diffAccountIterator is an account iterator that steps over the accounts (both +// live and deleted) contained within a state set. Higher order iterators will +// use the deleted accounts to skip deeper iterators. +// +// This iterator could be created from the diff layer or the disk layer (the +// aggregated state buffer). +type diffAccountIterator struct { + curHash common.Hash // The current hash the iterator is positioned on + keys []common.Hash // Keys left in the layer to iterate + fail error // Any failures encountered (stale) + loadFn loadAccount // Function to retrieve the account from with supplied hash +} + +// newDiffAccountIterator creates an account iterator over the given state set. +func newDiffAccountIterator(seek common.Hash, states *stateSet, fn loadAccount) AccountIterator { + // Seek out the requested starting account + hashes := states.accountList() + index := sort.Search(len(hashes), func(i int) bool { + return bytes.Compare(seek[:], hashes[i][:]) <= 0 + }) + // Assemble and returned the already seeked iterator + return &diffAccountIterator{ + keys: hashes[index:], + loadFn: fn, + } +} + +// Next steps the iterator forward one element, returning false if exhausted. +func (it *diffAccountIterator) Next() bool { + // If the iterator was already stale, consider it a programmer error. Although + // we could just return false here, triggering this path would probably mean + // somebody forgot to check for Error, so lets blow up instead of undefined + // behavior that's hard to debug. + if it.fail != nil { + panic(fmt.Sprintf("called Next of failed iterator: %v", it.fail)) + } + // Stop iterating if all keys were exhausted + if len(it.keys) == 0 { + return false + } + // Iterator seems to be still alive, retrieve and cache the live hash + it.curHash = it.keys[0] + + // key cached, shift the iterator and notify the user of success + it.keys = it.keys[1:] + return true +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit (e.g. the linked state set becoming stale). +func (it *diffAccountIterator) Error() error { + return it.fail +} + +// Hash returns the hash of the account the iterator is currently at. +func (it *diffAccountIterator) Hash() common.Hash { + return it.curHash +} + +// Account returns the RLP encoded slim account the iterator is currently at. +// This method may fail if the associated state goes stale. An error will +// be set to it.fail just in case. +// +// Note the returned account is not a copy, please don't modify it. +func (it *diffAccountIterator) Account() []byte { + blob, err := it.loadFn(it.curHash) + if err != nil { + it.fail = err + return nil + } + return blob +} + +// Release is a noop for diff account iterators as there are no held resources. +func (it *diffAccountIterator) Release() {} + +// diskAccountIterator is an account iterator that steps over the persistent +// accounts within the database. +// +// To simplify, the staleness of the persistent state is not tracked. The disk +// iterator is not intended to be used alone. It should always be wrapped with +// a diff iterator, as the bottom-most disk layer uses both the in-memory +// aggregated buffer and the persistent disk layer as the data sources. The +// staleness of the diff iterator is sufficient to invalidate the iterator pair. +type diskAccountIterator struct { + it ethdb.Iterator +} + +// newDiskAccountIterator creates an account iterator over the persistent state. +func newDiskAccountIterator(db ethdb.KeyValueStore, seek common.Hash) AccountIterator { + pos := common.TrimRightZeroes(seek[:]) + return &diskAccountIterator{ + it: db.NewIterator(rawdb.SnapshotAccountPrefix, pos), + } +} + +// Next steps the iterator forward one element, returning false if exhausted. +func (it *diskAccountIterator) Next() bool { + // If the iterator was already exhausted, don't bother + if it.it == nil { + return false + } + // Try to advance the iterator and release it if we reached the end + for { + if !it.it.Next() { + it.it.Release() + it.it = nil + return false + } + if len(it.it.Key()) == len(rawdb.SnapshotAccountPrefix)+common.HashLength { + break + } + } + return true +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit. (e.g, any error occurred in the database) +func (it *diskAccountIterator) Error() error { + if it.it == nil { + return nil // Iterator is exhausted and released + } + return it.it.Error() +} + +// Hash returns the hash of the account the iterator is currently at. +func (it *diskAccountIterator) Hash() common.Hash { + return common.BytesToHash(it.it.Key()) // The prefix will be truncated +} + +// Account returns the RLP encoded slim account the iterator is currently at. +func (it *diskAccountIterator) Account() []byte { + return it.it.Value() +} + +// Release releases the database snapshot held during iteration. +func (it *diskAccountIterator) Release() { + // The iterator is auto-released on exhaustion, so make sure it's still alive + if it.it != nil { + it.it.Release() + it.it = nil + } +} + +// diffStorageIterator is a storage iterator that steps over the specific storage +// (both live and deleted) contained within a state set. Higher order iterators +// will use the deleted slot to skip deeper iterators. +// +// This iterator could be created from the diff layer or the disk layer (the +// aggregated state buffer). +type diffStorageIterator struct { + curHash common.Hash // The current slot hash the iterator is positioned on + account common.Hash // The account hash the storage slots belonging to + keys []common.Hash // Keys left in the layer to iterate + fail error // Any failures encountered (stale) + loadFn loadStorage // Function to retrieve the storage slot from with supplied hash +} + +// newDiffStorageIterator creates a storage iterator over a single diff layer. +func newDiffStorageIterator(account common.Hash, seek common.Hash, states *stateSet, fn loadStorage) StorageIterator { + hashes := states.storageList(account) + index := sort.Search(len(hashes), func(i int) bool { + return bytes.Compare(seek[:], hashes[i][:]) <= 0 + }) + // Assemble and returned the already seeked iterator + return &diffStorageIterator{ + account: account, + keys: hashes[index:], + loadFn: fn, + } +} + +// Next steps the iterator forward one element, returning false if exhausted. +func (it *diffStorageIterator) Next() bool { + // If the iterator was already stale, consider it a programmer error. Although + // we could just return false here, triggering this path would probably mean + // somebody forgot to check for Error, so lets blow up instead of undefined + // behavior that's hard to debug. + if it.fail != nil { + panic(fmt.Sprintf("called Next of failed iterator: %v", it.fail)) + } + // Stop iterating if all keys were exhausted + if len(it.keys) == 0 { + return false + } + // Iterator seems to be still alive, retrieve and cache the live hash + it.curHash = it.keys[0] + + // key cached, shift the iterator and notify the user of success + it.keys = it.keys[1:] + return true +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit (e.g. the state set becoming stale). +func (it *diffStorageIterator) Error() error { + return it.fail +} + +// Hash returns the hash of the storage slot the iterator is currently at. +func (it *diffStorageIterator) Hash() common.Hash { + return it.curHash +} + +// Slot returns the raw storage slot value the iterator is currently at. +// This method may fail if the associated state goes stale. An error will +// be set to it.fail just in case. +// +// Note the returned slot is not a copy, please don't modify it. +func (it *diffStorageIterator) Slot() []byte { + storage, err := it.loadFn(it.account, it.curHash) + if err != nil { + it.fail = err + return nil + } + return storage +} + +// Release is a noop for diff account iterators as there are no held resources. +func (it *diffStorageIterator) Release() {} + +// diskStorageIterator is a storage iterator that steps over the persistent +// storage slots contained within the database. +// +// To simplify, the staleness of the persistent state is not tracked. The disk +// iterator is not intended to be used alone. It should always be wrapped with +// a diff iterator, as the bottom-most disk layer uses both the in-memory +// aggregated buffer and the persistent disk layer as the data sources. The +// staleness of the diff iterator is sufficient to invalidate the iterator pair. +type diskStorageIterator struct { + account common.Hash + it ethdb.Iterator +} + +// StorageIterator creates a storage iterator over the persistent state. +func newDiskStorageIterator(db ethdb.KeyValueStore, account common.Hash, seek common.Hash) StorageIterator { + pos := common.TrimRightZeroes(seek[:]) + return &diskStorageIterator{ + account: account, + it: db.NewIterator(append(rawdb.SnapshotStoragePrefix, account.Bytes()...), pos), + } +} + +// Next steps the iterator forward one element, returning false if exhausted. +func (it *diskStorageIterator) Next() bool { + // If the iterator was already exhausted, don't bother + if it.it == nil { + return false + } + // Try to advance the iterator and release it if we reached the end + for { + if !it.it.Next() { + it.it.Release() + it.it = nil + return false + } + if len(it.it.Key()) == len(rawdb.SnapshotStoragePrefix)+common.HashLength+common.HashLength { + break + } + } + return true +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit (e.g. the error occurred in the database). +func (it *diskStorageIterator) Error() error { + if it.it == nil { + return nil // Iterator is exhausted and released + } + return it.it.Error() +} + +// Hash returns the hash of the storage slot the iterator is currently at. +func (it *diskStorageIterator) Hash() common.Hash { + return common.BytesToHash(it.it.Key()) // The prefix will be truncated +} + +// Slot returns the raw storage slot content the iterator is currently at. +func (it *diskStorageIterator) Slot() []byte { + return it.it.Value() +} + +// Release releases the database snapshot held during iteration. +func (it *diskStorageIterator) Release() { + // The iterator is auto-released on exhaustion, so make sure it's still alive + if it.it != nil { + it.it.Release() + it.it = nil + } +} diff --git a/triedb/pathdb/iterator_binary.go b/triedb/pathdb/iterator_binary.go new file mode 100644 index 0000000000..dec31e8f3f --- /dev/null +++ b/triedb/pathdb/iterator_binary.go @@ -0,0 +1,344 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/common" +) + +// binaryIterator is a simplistic iterator to step over the accounts or storage +// in a snapshot, which may or may not be composed of multiple layers. Performance +// wise this iterator is slow, it's meant for cross validating the fast one. +// +// This iterator cannot be used on its own; it should be wrapped with an outer +// iterator, such as accountBinaryIterator or storageBinaryIterator. +// +// This iterator can only traverse the keys of the entries stored in the layers, +// but cannot obtain the corresponding values. Besides, the deleted entry will +// also be traversed, the outer iterator must check the emptiness before returning. +type binaryIterator struct { + a Iterator + b Iterator + aDone bool + bDone bool + k common.Hash + fail error +} + +// initBinaryAccountIterator creates a simplistic iterator to step over all the +// accounts in a slow, but easily verifiable way. Note this function is used +// for initialization, use `newBinaryAccountIterator` as the API. +func (dl *diskLayer) initBinaryAccountIterator(seek common.Hash) *binaryIterator { + // Create two iterators for state buffer and the persistent state in disk + // respectively and combine them as a binary iterator. + l := &binaryIterator{ + // The account loader function is unnecessary; the account key list + // produced by the supplied buffer alone is sufficient for iteration. + // + // The account key list for iteration is deterministic once the iterator + // is constructed, no matter the referenced disk layer is stale or not + // later. + a: newDiffAccountIterator(seek, dl.buffer.states, nil), + b: newDiskAccountIterator(dl.db.diskdb, seek), + } + l.aDone = !l.a.Next() + l.bDone = !l.b.Next() + return l +} + +// initBinaryAccountIterator creates a simplistic iterator to step over all the +// accounts in a slow, but easily verifiable way. Note this function is used +// for initialization, use `newBinaryAccountIterator` as the API. +func (dl *diffLayer) initBinaryAccountIterator(seek common.Hash) *binaryIterator { + parent, ok := dl.parent.(*diffLayer) + if !ok { + l := &binaryIterator{ + // The account loader function is unnecessary; the account key list + // produced by the supplied state set alone is sufficient for iteration. + // + // The account key list for iteration is deterministic once the iterator + // is constructed, no matter the referenced disk layer is stale or not + // later. + a: newDiffAccountIterator(seek, dl.states.stateSet, nil), + b: dl.parent.(*diskLayer).initBinaryAccountIterator(seek), + } + l.aDone = !l.a.Next() + l.bDone = !l.b.Next() + return l + } + l := &binaryIterator{ + // The account loader function is unnecessary; the account key list + // produced by the supplied state set alone is sufficient for iteration. + // + // The account key list for iteration is deterministic once the iterator + // is constructed, no matter the referenced disk layer is stale or not + // later. + a: newDiffAccountIterator(seek, dl.states.stateSet, nil), + b: parent.initBinaryAccountIterator(seek), + } + l.aDone = !l.a.Next() + l.bDone = !l.b.Next() + return l +} + +// initBinaryStorageIterator creates a simplistic iterator to step over all the +// storage slots in a slow, but easily verifiable way. Note this function is used +// for initialization, use `newBinaryStorageIterator` as the API. +func (dl *diskLayer) initBinaryStorageIterator(account common.Hash, seek common.Hash) *binaryIterator { + // Create two iterators for state buffer and the persistent state in disk + // respectively and combine them as a binary iterator. + l := &binaryIterator{ + // The storage loader function is unnecessary; the storage key list + // produced by the supplied buffer alone is sufficient for iteration. + // + // The storage key list for iteration is deterministic once the iterator + // is constructed, no matter the referenced disk layer is stale or not + // later. + a: newDiffStorageIterator(account, seek, dl.buffer.states, nil), + b: newDiskStorageIterator(dl.db.diskdb, account, seek), + } + l.aDone = !l.a.Next() + l.bDone = !l.b.Next() + return l +} + +// initBinaryStorageIterator creates a simplistic iterator to step over all the +// storage slots in a slow, but easily verifiable way. Note this function is used +// for initialization, use `newBinaryStorageIterator` as the API. +func (dl *diffLayer) initBinaryStorageIterator(account common.Hash, seek common.Hash) *binaryIterator { + parent, ok := dl.parent.(*diffLayer) + if !ok { + l := &binaryIterator{ + // The storage loader function is unnecessary; the storage key list + // produced by the supplied state set alone is sufficient for iteration. + // + // The storage key list for iteration is deterministic once the iterator + // is constructed, no matter the referenced disk layer is stale or not + // later. + a: newDiffStorageIterator(account, seek, dl.states.stateSet, nil), + b: dl.parent.(*diskLayer).initBinaryStorageIterator(account, seek), + } + l.aDone = !l.a.Next() + l.bDone = !l.b.Next() + return l + } + l := &binaryIterator{ + // The storage loader function is unnecessary; the storage key list + // produced by the supplied state set alone is sufficient for iteration. + // + // The storage key list for iteration is deterministic once the iterator + // is constructed, no matter the referenced disk layer is stale or not + // later. + a: newDiffStorageIterator(account, seek, dl.states.stateSet, nil), + b: parent.initBinaryStorageIterator(account, seek), + } + l.aDone = !l.a.Next() + l.bDone = !l.b.Next() + return l +} + +// Next advances the iterator by one element, returning false if both iterators +// are exhausted. Note that the entry pointed to by the iterator may be null +// (e.g., when an account is deleted but still accessible for iteration). +// The outer iterator must verify emptiness before terminating the iteration. +// +// There’s no need to check for errors in the two iterators, as we only iterate +// through the entries without retrieving their values. +func (it *binaryIterator) Next() bool { + if it.aDone && it.bDone { + return false + } + for { + if it.aDone { + it.k = it.b.Hash() + it.bDone = !it.b.Next() + return true + } + if it.bDone { + it.k = it.a.Hash() + it.aDone = !it.a.Next() + return true + } + nextA, nextB := it.a.Hash(), it.b.Hash() + if diff := bytes.Compare(nextA[:], nextB[:]); diff < 0 { + it.aDone = !it.a.Next() + it.k = nextA + return true + } else if diff == 0 { + // Now we need to advance one of them + it.aDone = !it.a.Next() + continue + } + it.bDone = !it.b.Next() + it.k = nextB + return true + } +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit (e.g. snapshot stack becoming stale). +func (it *binaryIterator) Error() error { + return it.fail +} + +// Hash returns the hash of the account the iterator is currently at. +func (it *binaryIterator) Hash() common.Hash { + return it.k +} + +// Release recursively releases all the iterators in the stack. +func (it *binaryIterator) Release() { + it.a.Release() + it.b.Release() +} + +// accountBinaryIterator is a wrapper around a binary iterator that adds functionality +// to retrieve account data from the associated layer at the current position. +type accountBinaryIterator struct { + *binaryIterator + layer layer +} + +// newBinaryAccountIterator creates a simplistic account iterator to step over +// all the accounts in a slow, but easily verifiable way. +// +// nolint:all +func (dl *diskLayer) newBinaryAccountIterator(seek common.Hash) AccountIterator { + return &accountBinaryIterator{ + binaryIterator: dl.initBinaryAccountIterator(seek), + layer: dl, + } +} + +// newBinaryAccountIterator creates a simplistic account iterator to step over +// all the accounts in a slow, but easily verifiable way. +func (dl *diffLayer) newBinaryAccountIterator(seek common.Hash) AccountIterator { + return &accountBinaryIterator{ + binaryIterator: dl.initBinaryAccountIterator(seek), + layer: dl, + } +} + +// Next steps the iterator forward one element, returning false if exhausted, +// or an error if iteration failed for some reason (e.g. the linked layer is +// stale during the iteration). +func (it *accountBinaryIterator) Next() bool { + for { + if !it.binaryIterator.Next() { + return false + } + // Retrieve the account data referenced by the current iterator, the + // associated layers might be outdated due to chain progressing, + // the relative error will be set to it.fail just in case. + // + // Skip the null account which was deleted before and move to the + // next account. + if len(it.Account()) != 0 { + return true + } + // it.fail might be set if error occurs by calling it.Account(). + // Stop iteration if so. + if it.fail != nil { + return false + } + } +} + +// Account returns the RLP encoded slim account the iterator is currently at, or +// nil if the iterated snapshot stack became stale (you can check Error after +// to see if it failed or not). +// +// Note the returned account is not a copy, please don't modify it. +func (it *accountBinaryIterator) Account() []byte { + blob, err := it.layer.account(it.k, 0) + if err != nil { + it.fail = err + return nil + } + return blob +} + +// storageBinaryIterator is a wrapper around a binary iterator that adds functionality +// to retrieve storage slot data from the associated layer at the current position. +type storageBinaryIterator struct { + *binaryIterator + account common.Hash + layer layer +} + +// newBinaryStorageIterator creates a simplistic account iterator to step over +// all the storage slots in a slow, but easily verifiable way. +// +// nolint:all +func (dl *diskLayer) newBinaryStorageIterator(account common.Hash, seek common.Hash) StorageIterator { + return &storageBinaryIterator{ + binaryIterator: dl.initBinaryStorageIterator(account, seek), + account: account, + layer: dl, + } +} + +// newBinaryStorageIterator creates a simplistic account iterator to step over +// all the storage slots in a slow, but easily verifiable way. +func (dl *diffLayer) newBinaryStorageIterator(account common.Hash, seek common.Hash) StorageIterator { + return &storageBinaryIterator{ + binaryIterator: dl.initBinaryStorageIterator(account, seek), + account: account, + layer: dl, + } +} + +// Next steps the iterator forward one element, returning false if exhausted, +// or an error if iteration failed for some reason (e.g. the linked layer is +// stale during the iteration). +func (it *storageBinaryIterator) Next() bool { + for { + if !it.binaryIterator.Next() { + return false + } + // Retrieve the storage data referenced by the current iterator, the + // associated layers might be outdated due to chain progressing, + // the relative error will be set to it.fail just in case. + // + // Skip the null storage which was deleted before and move to the + // next account. + if len(it.Slot()) != 0 { + return true + } + // it.fail might be set if error occurs by calling it.Slot(). + // Stop iteration if so. + if it.fail != nil { + return false + } + } +} + +// Slot returns the raw storage slot data the iterator is currently at, or +// nil if the iterated snapshot stack became stale (you can check Error after +// to see if it failed or not). +// +// Note the returned slot is not a copy, please don't modify it. +func (it *storageBinaryIterator) Slot() []byte { + blob, err := it.layer.storage(it.account, it.k, 0) + if err != nil { + it.fail = err + return nil + } + return blob +} diff --git a/triedb/pathdb/iterator_fast.go b/triedb/pathdb/iterator_fast.go new file mode 100644 index 0000000000..217b211fcc --- /dev/null +++ b/triedb/pathdb/iterator_fast.go @@ -0,0 +1,380 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + "fmt" + "slices" + "sort" + + "github.com/ethereum/go-ethereum/common" +) + +// weightedIterator is an iterator with an assigned weight. It is used to prioritise +// which account or storage slot is the correct one if multiple iterators find the +// same one (modified in multiple consecutive blocks). +type weightedIterator struct { + it Iterator + priority int +} + +func (it *weightedIterator) Cmp(other *weightedIterator) int { + // Order the iterators primarily by the account hashes + hashI := it.it.Hash() + hashJ := other.it.Hash() + + switch bytes.Compare(hashI[:], hashJ[:]) { + case -1: + return -1 + case 1: + return 1 + } + // Same account/storage-slot in multiple layers, split by priority + if it.priority < other.priority { + return -1 + } + if it.priority > other.priority { + return 1 + } + return 0 +} + +// fastIterator is a more optimized multi-layer iterator which maintains a +// direct mapping of all iterators leading down to the bottom layer. +type fastIterator struct { + curAccount []byte + curSlot []byte + + iterators []*weightedIterator + initiated bool + account bool + fail error +} + +// newFastIterator creates a new hierarchical account or storage iterator with one +// element per diff layer. The returned combo iterator can be used to walk over +// the entire layer stack simultaneously. +func newFastIterator(db *Database, root common.Hash, account common.Hash, seek common.Hash, accountIterator bool) (*fastIterator, error) { + current := db.tree.get(root) + if current == nil { + return nil, fmt.Errorf("unknown layer: %x", root) + } + fi := &fastIterator{ + account: accountIterator, + } + for depth := 0; current != nil; depth++ { + if accountIterator { + switch dl := current.(type) { + case *diskLayer: + fi.iterators = append(fi.iterators, &weightedIterator{ + // The state set in the disk layer is mutable, and the entire state becomes stale + // if a diff layer above is merged into it. Therefore, staleness must be checked, + // and the storage slot should be retrieved with read lock protection. + it: newDiffAccountIterator(seek, dl.buffer.states, func(hash common.Hash) ([]byte, error) { + dl.lock.RLock() + defer dl.lock.RUnlock() + + if dl.stale { + return nil, errSnapshotStale + } + return dl.buffer.states.mustAccount(hash) + }), + priority: depth, + }) + fi.iterators = append(fi.iterators, &weightedIterator{ + it: newDiskAccountIterator(dl.db.diskdb, seek), + priority: depth + 1, + }) + case *diffLayer: + // The state set in diff layer is immutable and will never be stale, + // so the read lock protection is unnecessary. + fi.iterators = append(fi.iterators, &weightedIterator{ + it: newDiffAccountIterator(seek, dl.states.stateSet, dl.states.mustAccount), + priority: depth, + }) + } + } else { + switch dl := current.(type) { + case *diskLayer: + fi.iterators = append(fi.iterators, &weightedIterator{ + // The state set in the disk layer is mutable, and the entire state becomes stale + // if a diff layer above is merged into it. Therefore, staleness must be checked, + // and the storage slot should be retrieved with read lock protection. + it: newDiffStorageIterator(account, seek, dl.buffer.states, func(addrHash common.Hash, slotHash common.Hash) ([]byte, error) { + dl.lock.RLock() + defer dl.lock.RUnlock() + + if dl.stale { + return nil, errSnapshotStale + } + return dl.buffer.states.mustStorage(addrHash, slotHash) + }), + priority: depth, + }) + fi.iterators = append(fi.iterators, &weightedIterator{ + it: newDiskStorageIterator(dl.db.diskdb, account, seek), + priority: depth + 1, + }) + case *diffLayer: + // The state set in diff layer is immutable and will never be stale, + // so the read lock protection is unnecessary. + fi.iterators = append(fi.iterators, &weightedIterator{ + it: newDiffStorageIterator(account, seek, dl.states.stateSet, dl.states.mustStorage), + priority: depth, + }) + } + } + current = current.parentLayer() + } + fi.init() + return fi, nil +} + +// init walks over all the iterators and resolves any clashes between them, after +// which it prepares the stack for step-by-step iteration. +func (fi *fastIterator) init() { + // Track which account hashes are iterators positioned on + var positioned = make(map[common.Hash]int) + + // Position all iterators and track how many remain live + for i := 0; i < len(fi.iterators); i++ { + // Retrieve the first element and if it clashes with a previous iterator, + // advance either the current one or the old one. Repeat until nothing is + // clashing anymore. + it := fi.iterators[i] + for { + // If the iterator is exhausted, drop it off the end + if !it.it.Next() { + it.it.Release() + last := len(fi.iterators) - 1 + + fi.iterators[i] = fi.iterators[last] + fi.iterators[last] = nil + fi.iterators = fi.iterators[:last] + + i-- + break + } + // The iterator is still alive, check for collisions with previous ones + hash := it.it.Hash() + if other, exist := positioned[hash]; !exist { + positioned[hash] = i + break + } else { + // Iterators collide, one needs to be progressed, use priority to + // determine which. + // + // This whole else-block can be avoided, if we instead + // do an initial priority-sort of the iterators. If we do that, + // then we'll only wind up here if a lower-priority (preferred) iterator + // has the same value, and then we will always just continue. + // However, it costs an extra sort, so it's probably not better + if fi.iterators[other].priority < it.priority { + // The 'it' should be progressed + continue + } else { + // The 'other' should be progressed, swap them + it = fi.iterators[other] + fi.iterators[other], fi.iterators[i] = fi.iterators[i], fi.iterators[other] + continue + } + } + } + } + // Re-sort the entire list + slices.SortFunc(fi.iterators, func(a, b *weightedIterator) int { return a.Cmp(b) }) + fi.initiated = false +} + +// Next steps the iterator forward one element, returning false if exhausted. +func (fi *fastIterator) Next() bool { + if len(fi.iterators) == 0 { + return false + } + if !fi.initiated { + // Don't forward first time -- we had to 'Next' once in order to + // do the sorting already + fi.initiated = true + if fi.account { + fi.curAccount = fi.iterators[0].it.(AccountIterator).Account() + } else { + fi.curSlot = fi.iterators[0].it.(StorageIterator).Slot() + } + if innerErr := fi.iterators[0].it.Error(); innerErr != nil { + fi.fail = innerErr + return false + } + if fi.curAccount != nil || fi.curSlot != nil { + return true + } + // Implicit else: we've hit a nil-account or nil-slot, and need to + // fall through to the loop below to land on something non-nil + } + // If an account or a slot is deleted in one of the layers, the key will + // still be there, but the actual value will be nil. However, the iterator + // should not export nil-values (but instead simply omit the key), so we + // need to loop here until we either + // - get a non-nil value, + // - hit an error, + // - or exhaust the iterator + for { + if !fi.next(0) { + return false // exhausted + } + if fi.account { + fi.curAccount = fi.iterators[0].it.(AccountIterator).Account() + } else { + fi.curSlot = fi.iterators[0].it.(StorageIterator).Slot() + } + if innerErr := fi.iterators[0].it.Error(); innerErr != nil { + fi.fail = innerErr + return false // error + } + if fi.curAccount != nil || fi.curSlot != nil { + break // non-nil value found + } + } + return true +} + +// next handles the next operation internally and should be invoked when we know +// that two elements in the list may have the same value. +// +// For example, if the iterated hashes become [2,3,5,5,8,9,10], then we should +// invoke next(3), which will call Next on elem 3 (the second '5') and will +// cascade along the list, applying the same operation if needed. +func (fi *fastIterator) next(idx int) bool { + // If this particular iterator got exhausted, remove it and return true (the + // next one is surely not exhausted yet, otherwise it would have been removed + // already). + if it := fi.iterators[idx].it; !it.Next() { + it.Release() + + fi.iterators = append(fi.iterators[:idx], fi.iterators[idx+1:]...) + return len(fi.iterators) > 0 + } + // If there's no one left to cascade into, return + if idx == len(fi.iterators)-1 { + return true + } + // We next-ed the iterator at 'idx', now we may have to re-sort that element + var ( + cur, next = fi.iterators[idx], fi.iterators[idx+1] + curHash, nextHash = cur.it.Hash(), next.it.Hash() + ) + if diff := bytes.Compare(curHash[:], nextHash[:]); diff < 0 { + // It is still in correct place + return true + } else if diff == 0 && cur.priority < next.priority { + // So still in correct place, but we need to iterate on the next + fi.next(idx + 1) + return true + } + // At this point, the iterator is in the wrong location, but the remaining + // list is sorted. Find out where to move the item. + clash := -1 + index := sort.Search(len(fi.iterators), func(n int) bool { + // The iterator always advances forward, so anything before the old slot + // is known to be behind us, so just skip them altogether. This actually + // is an important clause since the sort order got invalidated. + if n < idx { + return false + } + if n == len(fi.iterators)-1 { + // Can always place an elem last + return true + } + nextHash := fi.iterators[n+1].it.Hash() + if diff := bytes.Compare(curHash[:], nextHash[:]); diff < 0 { + return true + } else if diff > 0 { + return false + } + // The elem we're placing it next to has the same value, + // so whichever winds up on n+1 will need further iteration + clash = n + 1 + + return cur.priority < fi.iterators[n+1].priority + }) + fi.move(idx, index) + if clash != -1 { + fi.next(clash) + } + return true +} + +// move advances an iterator to another position in the list. +func (fi *fastIterator) move(index, newpos int) { + elem := fi.iterators[index] + copy(fi.iterators[index:], fi.iterators[index+1:newpos+1]) + fi.iterators[newpos] = elem +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit (e.g. snapshot stack becoming stale). +func (fi *fastIterator) Error() error { + return fi.fail +} + +// Hash returns the current key +func (fi *fastIterator) Hash() common.Hash { + return fi.iterators[0].it.Hash() +} + +// Account returns the current account blob. +// Note the returned account is not a copy, please don't modify it. +func (fi *fastIterator) Account() []byte { + return fi.curAccount +} + +// Slot returns the current storage slot. +// Note the returned slot is not a copy, please don't modify it. +func (fi *fastIterator) Slot() []byte { + return fi.curSlot +} + +// Release iterates over all the remaining live layer iterators and releases each +// of them individually. +func (fi *fastIterator) Release() { + for _, it := range fi.iterators { + it.it.Release() + } + fi.iterators = nil +} + +// Debug is a convenience helper during testing +func (fi *fastIterator) Debug() { + for _, it := range fi.iterators { + fmt.Printf("[p=%v v=%v] ", it.priority, it.it.Hash()[0]) + } + fmt.Println() +} + +// newFastAccountIterator creates a new hierarchical account iterator with one +// element per diff layer. The returned combo iterator can be used to walk over +// the entire snapshot diff stack simultaneously. +func newFastAccountIterator(db *Database, root common.Hash, seek common.Hash) (AccountIterator, error) { + return newFastIterator(db, root, common.Hash{}, seek, true) +} + +// newFastStorageIterator creates a new hierarchical storage iterator with one +// element per diff layer. The returned combo iterator can be used to walk over +// the entire snapshot diff stack simultaneously. +func newFastStorageIterator(db *Database, root common.Hash, account common.Hash, seek common.Hash) (StorageIterator, error) { + return newFastIterator(db, root, account, seek, false) +} diff --git a/triedb/pathdb/iterator_test.go b/triedb/pathdb/iterator_test.go new file mode 100644 index 0000000000..48b5870b5b --- /dev/null +++ b/triedb/pathdb/iterator_test.go @@ -0,0 +1,1162 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + "encoding/binary" + "fmt" + "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/testrand" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/holiman/uint256" +) + +type verifyContent int + +const ( + verifyNothing verifyContent = iota + verifyAccount + verifyStorage +) + +func verifyIterator(t *testing.T, expCount int, it Iterator, verify verifyContent) { + t.Helper() + + var ( + count = 0 + last = common.Hash{} + ) + for it.Next() { + hash := it.Hash() + if bytes.Compare(last[:], hash[:]) >= 0 { + t.Errorf("wrong order: %x >= %x", last, hash) + } + count++ + if verify == verifyAccount && len(it.(AccountIterator).Account()) == 0 { + t.Errorf("iterator returned nil-value for hash %x", hash) + } else if verify == verifyStorage && len(it.(StorageIterator).Slot()) == 0 { + t.Errorf("iterator returned nil-value for hash %x", hash) + } + last = hash + } + if count != expCount { + t.Errorf("iterator count mismatch: have %d, want %d", count, expCount) + } + if err := it.Error(); err != nil { + t.Errorf("iterator failed: %v", err) + } +} + +// randomAccount generates a random account and returns it RLP encoded. +func randomAccount() []byte { + a := &types.StateAccount{ + Balance: uint256.NewInt(rand.Uint64()), + Nonce: rand.Uint64(), + Root: testrand.Hash(), + CodeHash: types.EmptyCodeHash[:], + } + data, _ := rlp.EncodeToBytes(a) + return data +} + +// randomAccountSet generates a set of random accounts with the given strings as +// the account address hashes. +func randomAccountSet(hashes ...string) map[common.Hash][]byte { + accounts := make(map[common.Hash][]byte) + for _, hash := range hashes { + accounts[common.HexToHash(hash)] = randomAccount() + } + return accounts +} + +// randomStorageSet generates a set of random slots with the given strings as +// the slot addresses. +func randomStorageSet(accounts []string, hashes [][]string, nilStorage [][]string) map[common.Hash]map[common.Hash][]byte { + storages := make(map[common.Hash]map[common.Hash][]byte) + for index, account := range accounts { + storages[common.HexToHash(account)] = make(map[common.Hash][]byte) + + if index < len(hashes) { + hashes := hashes[index] + for _, hash := range hashes { + storages[common.HexToHash(account)][common.HexToHash(hash)] = testrand.Bytes(32) + } + } + if index < len(nilStorage) { + nils := nilStorage[index] + for _, hash := range nils { + storages[common.HexToHash(account)][common.HexToHash(hash)] = nil + } + } + } + return storages +} + +// TestAccountIteratorBasics tests some simple single-layer(diff and disk) iteration +func TestAccountIteratorBasics(t *testing.T) { + var ( + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + // Fill up a parent + for i := 0; i < 100; i++ { + hash := testrand.Hash() + data := testrand.Bytes(32) + accounts[hash] = data + + if rand.Intn(2) == 0 { + accStorage := make(map[common.Hash][]byte) + accStorage[testrand.Hash()] = testrand.Bytes(32) + storage[hash] = accStorage + } + } + states := newStates(accounts, storage) + it := newDiffAccountIterator(common.Hash{}, states, nil) + verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator + + // TODO reenable these tests once the persistent state iteration + // is implemented. + + //db := rawdb.NewMemoryDatabase() + //batch := db.NewBatch() + //states.write(db, batch, nil, nil) + //batch.Write() + //it = newDiskAccountIterator(db, common.Hash{}) + //verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator +} + +// TestStorageIteratorBasics tests some simple single-layer(diff and disk) iteration for storage +func TestStorageIteratorBasics(t *testing.T) { + var ( + nilStorage = make(map[common.Hash]int) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + // Fill some random data + for i := 0; i < 10; i++ { + hash := testrand.Hash() + accounts[hash] = testrand.Bytes(32) + + accStorage := make(map[common.Hash][]byte) + var nilstorage int + for i := 0; i < 100; i++ { + if rand.Intn(2) == 0 { + accStorage[testrand.Hash()] = testrand.Bytes(32) + } else { + accStorage[testrand.Hash()] = nil // delete slot + nilstorage += 1 + } + } + storage[hash] = accStorage + nilStorage[hash] = nilstorage + } + states := newStates(accounts, storage) + for account := range accounts { + it := newDiffStorageIterator(account, common.Hash{}, states, nil) + verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator + } + + // TODO reenable these tests once the persistent state iteration + // is implemented. + + //db := rawdb.NewMemoryDatabase() + //batch := db.NewBatch() + //states.write(db, batch, nil, nil) + //batch.Write() + //for account := range accounts { + // it := newDiskStorageIterator(db, account, common.Hash{}) + // verifyIterator(t, 100-nilStorage[account], it, verifyNothing) // Nil is allowed for single layer iterator + //} +} + +type testIterator struct { + values []byte +} + +func newTestIterator(values ...byte) *testIterator { + return &testIterator{values} +} + +func (ti *testIterator) Seek(common.Hash) { + panic("implement me") +} + +func (ti *testIterator) Next() bool { + ti.values = ti.values[1:] + return len(ti.values) > 0 +} + +func (ti *testIterator) Error() error { + return nil +} + +func (ti *testIterator) Hash() common.Hash { + return common.BytesToHash([]byte{ti.values[0]}) +} + +func (ti *testIterator) Account() []byte { + return nil +} + +func (ti *testIterator) Slot() []byte { + return nil +} + +func (ti *testIterator) Release() {} + +func TestFastIteratorBasics(t *testing.T) { + type testCase struct { + lists [][]byte + expKeys []byte + } + for i, tc := range []testCase{ + {lists: [][]byte{{0, 1, 8}, {1, 2, 8}, {2, 9}, {4}, + {7, 14, 15}, {9, 13, 15, 16}}, + expKeys: []byte{0, 1, 2, 4, 7, 8, 9, 13, 14, 15, 16}}, + {lists: [][]byte{{0, 8}, {1, 2, 8}, {7, 14, 15}, {8, 9}, + {9, 10}, {10, 13, 15, 16}}, + expKeys: []byte{0, 1, 2, 7, 8, 9, 10, 13, 14, 15, 16}}, + } { + var iterators []*weightedIterator + for i, data := range tc.lists { + it := newTestIterator(data...) + iterators = append(iterators, &weightedIterator{it, i}) + } + fi := &fastIterator{ + iterators: iterators, + initiated: false, + } + count := 0 + for fi.Next() { + if got, exp := fi.Hash()[31], tc.expKeys[count]; exp != got { + t.Errorf("tc %d, [%d]: got %d exp %d", i, count, got, exp) + } + count++ + } + } +} + +// TestAccountIteratorTraversal tests some simple multi-layer iteration. +func TestAccountIteratorTraversal(t *testing.T) { + config := &Config{ + WriteBufferSize: 0, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + // Stack three diff layers on top with various overlaps + db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 0, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil, nil, nil)) + + db.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), 0, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xbb", "0xdd", "0xf0"), nil, nil, nil)) + + db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 0, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xcc", "0xf0", "0xff"), nil, nil, nil)) + + // Verify the single and multi-layer iterators + head := db.tree.get(common.HexToHash("0x04")) + + // singleLayer: 0xcc, 0xf0, 0xff + it := newDiffAccountIterator(common.Hash{}, head.(*diffLayer).states.stateSet, nil) + verifyIterator(t, 3, it, verifyNothing) + + // binaryIterator: 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xf0, 0xff + verifyIterator(t, 7, head.(*diffLayer).newBinaryAccountIterator(common.Hash{}), verifyAccount) + + // fastIterator: 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xf0, 0xff + it, _ = db.AccountIterator(common.HexToHash("0x04"), common.Hash{}) + verifyIterator(t, 7, it, verifyAccount) + it.Release() + + // TODO reenable these tests once the persistent state iteration + // is implemented. + + // Test after persist some bottom-most layers into the disk, + // the functionalities still work. + //db.tree.cap(common.HexToHash("0x04"), 2) + + //head = db.tree.get(common.HexToHash("0x04")) + //verifyIterator(t, 7, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount) + // + //it, _ = db.AccountIterator(common.HexToHash("0x04"), common.Hash{}) + //verifyIterator(t, 7, it, verifyAccount) + //it.Release() +} + +func TestStorageIteratorTraversal(t *testing.T) { + config := &Config{ + WriteBufferSize: 0, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + // Stack three diff layers on top with various overlaps + db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 0, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil), nil, nil)) + + db.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), 0, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x04", "0x05", "0x06"}}, nil), nil, nil)) + + db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 0, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil), nil, nil)) + + // Verify the single and multi-layer iterators + head := db.tree.get(common.HexToHash("0x04")) + + // singleLayer: 0x1, 0x2, 0x3 + diffIter := newDiffStorageIterator(common.HexToHash("0xaa"), common.Hash{}, head.(*diffLayer).states.stateSet, nil) + verifyIterator(t, 3, diffIter, verifyNothing) + + // binaryIterator: 0x1, 0x2, 0x3, 0x4, 0x5, 0x6 + verifyIterator(t, 6, head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}), verifyStorage) + + // fastIterator: 0x1, 0x2, 0x3, 0x4, 0x5, 0x6 + it, _ := db.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 6, it, verifyStorage) + it.Release() + + // TODO reenable these tests once the persistent state iteration + // is implemented. + + // Test after persist some bottom-most layers into the disk, + // the functionalities still work. + //db.tree.cap(common.HexToHash("0x04"), 2) + //verifyIterator(t, 6, head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa")), verifyStorage) + // + //it, _ = db.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) + //verifyIterator(t, 6, it, verifyStorage) + //it.Release() +} + +// TestAccountIteratorTraversalValues tests some multi-layer iteration, where we +// also expect the correct values to show up. +func TestAccountIteratorTraversalValues(t *testing.T) { + config := &Config{ + WriteBufferSize: 0, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + // Create a batch of account sets to seed subsequent layers with + var ( + a = make(map[common.Hash][]byte) + b = make(map[common.Hash][]byte) + c = make(map[common.Hash][]byte) + d = make(map[common.Hash][]byte) + e = make(map[common.Hash][]byte) + f = make(map[common.Hash][]byte) + g = make(map[common.Hash][]byte) + h = make(map[common.Hash][]byte) + ) + for i := byte(2); i < 0xff; i++ { + a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + if i > 20 && i%2 == 0 { + b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + } + if i%4 == 0 { + c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + } + if i%7 == 0 { + d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + } + if i%8 == 0 { + e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + } + if i > 50 || i < 85 { + f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + } + if i%64 == 0 { + g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + } + if i%128 == 0 { + h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + } + } + // Assemble a stack of snapshots from the account layers + db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(a, nil, nil, nil)) + db.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(b, nil, nil, nil)) + db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 4, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(c, nil, nil, nil)) + db.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), 5, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(d, nil, nil, nil)) + db.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), 6, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(e, nil, nil, nil)) + db.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), 7, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(f, nil, nil, nil)) + db.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), 8, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(g, nil, nil, nil)) + db.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), 9, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(h, nil, nil, nil)) + + // binaryIterator + r, _ := db.StateReader(common.HexToHash("0x09")) + head := db.tree.get(common.HexToHash("0x09")) + it := head.(*diffLayer).newBinaryAccountIterator(common.Hash{}) + for it.Next() { + hash := it.Hash() + want, err := r.(*reader).AccountRLP(hash) + if err != nil { + t.Fatalf("failed to retrieve expected account: %v", err) + } + if have := it.Account(); !bytes.Equal(want, have) { + t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want) + } + } + it.Release() + + // fastIterator + it, _ = db.AccountIterator(common.HexToHash("0x09"), common.Hash{}) + for it.Next() { + hash := it.Hash() + want, err := r.(*reader).AccountRLP(hash) + if err != nil { + t.Fatalf("failed to retrieve expected account: %v", err) + } + if have := it.Account(); !bytes.Equal(want, have) { + t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want) + } + } + it.Release() + + // TODO reenable these tests once the persistent state iteration + // is implemented. + + // Test after persist some bottom-most layers into the disk, + // the functionalities still work. + //db.tree.cap(common.HexToHash("0x09"), 2) + // + //it, _ = db.AccountIterator(common.HexToHash("0x09"), common.Hash{}) + //for it.Next() { + // hash := it.Hash() + // account, err := head.Account(hash) + // if err != nil { + // t.Fatalf("failed to retrieve expected account: %v", err) + // } + // want, _ := rlp.EncodeToBytes(account) + // if have := it.Account(); !bytes.Equal(want, have) { + // t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want) + // } + //} + //it.Release() +} + +func TestStorageIteratorTraversalValues(t *testing.T) { + config := &Config{ + WriteBufferSize: 0, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + wrapStorage := func(storage map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte { + return map[common.Hash]map[common.Hash][]byte{ + common.HexToHash("0xaa"): storage, + } + } + // Create a batch of storage sets to seed subsequent layers with + var ( + a = make(map[common.Hash][]byte) + b = make(map[common.Hash][]byte) + c = make(map[common.Hash][]byte) + d = make(map[common.Hash][]byte) + e = make(map[common.Hash][]byte) + f = make(map[common.Hash][]byte) + g = make(map[common.Hash][]byte) + h = make(map[common.Hash][]byte) + ) + for i := byte(2); i < 0xff; i++ { + a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + if i > 20 && i%2 == 0 { + b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + } + if i%4 == 0 { + c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + } + if i%7 == 0 { + d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + } + if i%8 == 0 { + e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + } + if i > 50 || i < 85 { + f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + } + if i%64 == 0 { + g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + } + if i%128 == 0 { + h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + } + } + // Assemble a stack of snapshots from the account layers + db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(randomAccountSet("0xaa"), wrapStorage(a), nil, nil)) + db.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(randomAccountSet("0xaa"), wrapStorage(b), nil, nil)) + db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 4, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(randomAccountSet("0xaa"), wrapStorage(c), nil, nil)) + db.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), 5, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(randomAccountSet("0xaa"), wrapStorage(d), nil, nil)) + db.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), 6, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(randomAccountSet("0xaa"), wrapStorage(e), nil, nil)) + db.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), 7, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(randomAccountSet("0xaa"), wrapStorage(f), nil, nil)) + db.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), 8, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(randomAccountSet("0xaa"), wrapStorage(g), nil, nil)) + db.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), 9, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(randomAccountSet("0xaa"), wrapStorage(h), nil, nil)) + + // binaryIterator + r, _ := db.StateReader(common.HexToHash("0x09")) + head := db.tree.get(common.HexToHash("0x09")) + it := head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}) + for it.Next() { + hash := it.Hash() + want, err := r.Storage(common.HexToHash("0xaa"), hash) + if err != nil { + t.Fatalf("failed to retrieve expected account: %v", err) + } + if have := it.Slot(); !bytes.Equal(want, have) { + t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want) + } + } + it.Release() + + // fastIterator + it, _ = db.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{}) + for it.Next() { + hash := it.Hash() + want, err := r.Storage(common.HexToHash("0xaa"), hash) + if err != nil { + t.Fatalf("failed to retrieve expected storage slot: %v", err) + } + if have := it.Slot(); !bytes.Equal(want, have) { + t.Fatalf("hash %x: slot mismatch: have %x, want %x", hash, have, want) + } + } + it.Release() + + // TODO reenable these tests once the persistent state iteration + // is implemented. + + // Test after persist some bottom-most layers into the disk, + // the functionalities still work. + //db.tree.cap(common.HexToHash("0x09"), 2) + // + //it, _ = db.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{}) + //for it.Next() { + // hash := it.Hash() + // want, err := head.Storage(common.HexToHash("0xaa"), hash) + // if err != nil { + // t.Fatalf("failed to retrieve expected slot: %v", err) + // } + // if have := it.Slot(); !bytes.Equal(want, have) { + // t.Fatalf("hash %x: slot mismatch: have %x, want %x", hash, have, want) + // } + //} + //it.Release() +} + +// This testcase is notorious, all layers contain the exact same 200 accounts. +func TestAccountIteratorLargeTraversal(t *testing.T) { + // Create a custom account factory to recreate the same addresses + makeAccounts := func(num int) map[common.Hash][]byte { + accounts := make(map[common.Hash][]byte) + for i := 0; i < num; i++ { + h := common.Hash{} + binary.BigEndian.PutUint64(h[:], uint64(i+1)) + accounts[h] = randomAccount() + } + return accounts + } + // Build up a large stack of snapshots + config := &Config{ + WriteBufferSize: 0, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + for i := 1; i < 128; i++ { + parent := types.EmptyRootHash + if i == 1 { + parent = common.HexToHash(fmt.Sprintf("0x%02x", i)) + } + db.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), parent, uint64(i), trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(makeAccounts(200), nil, nil, nil)) + } + // Iterate the entire stack and ensure everything is hit only once + head := db.tree.get(common.HexToHash("0x80")) + verifyIterator(t, 200, newDiffAccountIterator(common.Hash{}, head.(*diffLayer).states.stateSet, nil), verifyNothing) + verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator(common.Hash{}), verifyAccount) + + it, _ := db.AccountIterator(common.HexToHash("0x80"), common.Hash{}) + verifyIterator(t, 200, it, verifyAccount) + it.Release() + + // TODO reenable these tests once the persistent state iteration + // is implemented. + + // Test after persist some bottom-most layers into the disk, + // the functionalities still work. + //db.tree.cap(common.HexToHash("0x80"), 2) + // + //verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount) + // + //it, _ = db.AccountIterator(common.HexToHash("0x80"), common.Hash{}) + //verifyIterator(t, 200, it, verifyAccount) + //it.Release() +} + +// TestAccountIteratorFlattening tests what happens when we +// - have a live iterator on child C (parent C1 -> C2 .. CN) +// - flattens C2 all the way into CN +// - continues iterating +func TestAccountIteratorFlattening(t *testing.T) { + config := &Config{ + WriteBufferSize: 10 * 1024, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + // Create a stack of diffs on top + db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil, nil, nil)) + + db.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), 2, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xbb", "0xdd", "0xf0"), nil, nil, nil)) + + db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 3, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xcc", "0xf0", "0xff"), nil, nil, nil)) + + // Create a binary iterator and flatten the data from underneath it + head := db.tree.get(common.HexToHash("0x04")) + bit := head.(*diffLayer).newBinaryAccountIterator(common.Hash{}) + defer bit.Release() + + // Create a fast iterator and flatten the data from underneath it + fit, _ := db.AccountIterator(common.HexToHash("0x04"), common.Hash{}) + defer fit.Release() + + if err := db.tree.cap(common.HexToHash("0x04"), 1); err != nil { + t.Fatalf("failed to flatten snapshot stack: %v", err) + } + verifyIterator(t, 7, bit, verifyAccount) + verifyIterator(t, 7, fit, verifyAccount) +} + +func TestAccountIteratorSeek(t *testing.T) { + config := &Config{ + WriteBufferSize: 0, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil, nil, nil)) + + db.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), 2, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xbb", "0xdd", "0xf0"), nil, nil, nil)) + + db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 3, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xcc", "0xf0", "0xff"), nil, nil, nil)) + + // Account set is now + // 02: aa, ee, f0, ff + // 03: aa, bb, dd, ee, f0 (, f0), ff + // 04: aa, bb, cc, dd, ee, f0 (, f0), ff (, ff) + // Construct various iterators and ensure their traversal is correct + it, _ := db.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xdd")) + defer it.Release() + verifyIterator(t, 3, it, verifyAccount) // expected: ee, f0, ff + + it, _ = db.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xaa")) + defer it.Release() + verifyIterator(t, 4, it, verifyAccount) // expected: aa, ee, f0, ff + + it, _ = db.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xff")) + defer it.Release() + verifyIterator(t, 1, it, verifyAccount) // expected: ff + + it, _ = db.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xff1")) + defer it.Release() + verifyIterator(t, 0, it, verifyAccount) // expected: nothing + + it, _ = db.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xbb")) + defer it.Release() + verifyIterator(t, 6, it, verifyAccount) // expected: bb, cc, dd, ee, f0, ff + + it, _ = db.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xef")) + defer it.Release() + verifyIterator(t, 2, it, verifyAccount) // expected: f0, ff + + it, _ = db.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xf0")) + defer it.Release() + verifyIterator(t, 2, it, verifyAccount) // expected: f0, ff + + it, _ = db.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xff")) + defer it.Release() + verifyIterator(t, 1, it, verifyAccount) // expected: ff + + it, _ = db.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xff1")) + defer it.Release() + verifyIterator(t, 0, it, verifyAccount) // expected: nothing +} + +func TestStorageIteratorSeek(t *testing.T) { + t.Run("fast", func(t *testing.T) { + testStorageIteratorSeek(t, func(db *Database, root, account, seek common.Hash) StorageIterator { + it, _ := db.StorageIterator(root, account, seek) + return it + }) + }) + t.Run("binary", func(t *testing.T) { + testStorageIteratorSeek(t, func(db *Database, root, account, seek common.Hash) StorageIterator { + return db.tree.get(root).(*diffLayer).newBinaryStorageIterator(account, seek) + }) + }) +} + +func testStorageIteratorSeek(t *testing.T, newIterator func(db *Database, root, account, seek common.Hash) StorageIterator) { + config := &Config{ + WriteBufferSize: 0, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + // Stack three diff layers on top with various overlaps + db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil), nil, nil)) + + db.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), 2, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x05", "0x06"}}, nil), nil, nil)) + + db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 3, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x05", "0x08"}}, nil), nil, nil)) + + // Account set is now + // 02: 01, 03, 05 + // 03: 01, 02, 03, 05 (, 05), 06 + // 04: 01(, 01), 02, 03, 05(, 05, 05), 06, 08 + // Construct various iterators and ensure their traversal is correct + it := newIterator(db, common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x01")) + defer it.Release() + verifyIterator(t, 3, it, verifyStorage) // expected: 01, 03, 05 + + it = newIterator(db, common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x02")) + defer it.Release() + verifyIterator(t, 2, it, verifyStorage) // expected: 03, 05 + + it = newIterator(db, common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x5")) + defer it.Release() + verifyIterator(t, 1, it, verifyStorage) // expected: 05 + + it = newIterator(db, common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x6")) + defer it.Release() + verifyIterator(t, 0, it, verifyStorage) // expected: nothing + + it = newIterator(db, common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x01")) + defer it.Release() + verifyIterator(t, 6, it, verifyStorage) // expected: 01, 02, 03, 05, 06, 08 + + it = newIterator(db, common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x05")) + defer it.Release() + verifyIterator(t, 3, it, verifyStorage) // expected: 05, 06, 08 + + it = newIterator(db, common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x08")) + defer it.Release() + verifyIterator(t, 1, it, verifyStorage) // expected: 08 + + it = newIterator(db, common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x09")) + defer it.Release() + verifyIterator(t, 0, it, verifyStorage) // expected: nothing +} + +// TestAccountIteratorDeletions tests that the iterator behaves correct when there are +// deleted accounts (where the Account() value is nil). The iterator +// should not output any accounts or nil-values for those cases. +func TestAccountIteratorDeletions(t *testing.T) { + t.Run("fast", func(t *testing.T) { + testAccountIteratorDeletions(t, func(db *Database, root, seek common.Hash) AccountIterator { + it, _ := db.AccountIterator(root, seek) + return it + }) + }) + t.Run("binary", func(t *testing.T) { + testAccountIteratorDeletions(t, func(db *Database, root, seek common.Hash) AccountIterator { + return db.tree.get(root).(*diffLayer).newBinaryAccountIterator(seek) + }) + }) +} + +func testAccountIteratorDeletions(t *testing.T, newIterator func(db *Database, root, seek common.Hash) AccountIterator) { + config := &Config{ + WriteBufferSize: 0, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + // Stack three diff layers on top with various overlaps + db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0x11", "0x22", "0x33"), nil, nil, nil)) + + deleted := common.HexToHash("0x22") + accounts := randomAccountSet("0x11", "0x33") + accounts[deleted] = nil + db.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), 2, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(accounts, nil, nil, nil)) + + db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 3, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0x33", "0x44", "0x55"), nil, nil, nil)) + + // The output should be 11,33,44,55 + it := newIterator(db, common.HexToHash("0x04"), common.Hash{}) + // Do a quick check + verifyIterator(t, 4, it, verifyAccount) + it.Release() + + // And a more detailed verification that we indeed do not see '0x22' + it = newIterator(db, common.HexToHash("0x04"), common.Hash{}) + defer it.Release() + for it.Next() { + hash := it.Hash() + if it.Account() == nil { + t.Errorf("iterator returned nil-value for hash %x", hash) + } + if hash == deleted { + t.Errorf("expected deleted elem %x to not be returned by iterator", deleted) + } + } +} + +func TestStorageIteratorDeletions(t *testing.T) { + config := &Config{ + WriteBufferSize: 0, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + // Stack three diff layers on top with various overlaps + db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil), nil, nil)) + + db.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), 2, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x04", "0x06"}}, [][]string{{"0x01", "0x03"}}), nil, nil)) + + // The output should be 02,04,05,06 + it, _ := db.StorageIterator(common.HexToHash("0x03"), common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 4, it, verifyStorage) + it.Release() + + // The output should be 04,05,06 + it, _ = db.StorageIterator(common.HexToHash("0x03"), common.HexToHash("0xaa"), common.HexToHash("0x03")) + verifyIterator(t, 3, it, verifyStorage) + it.Release() + + // Destruct the whole storage + accounts := map[common.Hash][]byte{ + common.HexToHash("0xaa"): nil, + } + db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 3, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(accounts, randomStorageSet([]string{"0xaa"}, nil, [][]string{{"0x02", "0x04", "0x05", "0x06"}}), nil, nil)) + + it, _ = db.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 0, it, verifyStorage) + it.Release() + + // Re-insert the slots of the same account + db.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), 4, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x07", "0x08", "0x09"}}, nil), nil, nil)) + + // The output should be 07,08,09 + it, _ = db.StorageIterator(common.HexToHash("0x05"), common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 3, it, verifyStorage) + it.Release() + + // Destruct the whole storage but re-create the account in the same layer + db.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), 5, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, [][]string{{"0x07", "0x08", "0x09"}}), nil, nil)) + + it, _ = db.StorageIterator(common.HexToHash("0x06"), common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 2, it, verifyStorage) // The output should be 11,12 + it.Release() + + verifyIterator(t, 2, db.tree.get(common.HexToHash("0x06")).(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}), verifyStorage) +} + +// TestStaleIterator tests if the iterator could correctly terminate the iteration +// if the associated layers are outdated. +func TestStaleIterator(t *testing.T) { + testStaleIterator(t, func(db *Database, hash common.Hash) Iterator { + it, _ := db.StorageIterator(hash, common.HexToHash("0xaa"), common.Hash{}) + return it + }) + testStaleIterator(t, func(db *Database, hash common.Hash) Iterator { + head := db.tree.get(hash) + return head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}) + }) +} + +func testStaleIterator(t *testing.T, newIter func(db *Database, hash common.Hash) Iterator) { + config := &Config{ + WriteBufferSize: 16 * 1024 * 1024, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + // [02 (disk), 03] + db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01"}}, nil), nil, nil)) + db.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), 2, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02"}}, nil), nil, nil)) + db.tree.cap(common.HexToHash("0x03"), 1) + + // [02 (disk), 03, 04] + db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 3, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x03"}}, nil), nil, nil)) + iter := newIter(db, common.HexToHash("0x04")) + + // [04 (disk), 05] + db.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), 3, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x04"}}, nil), nil, nil)) + db.tree.cap(common.HexToHash("0x05"), 1) + + // Iterator can't finish the traversal as the layer 02 has becoming stale. + for iter.Next() { + } + err := iter.Error() + t.Log(err) + if err == nil { + t.Fatalf("Expected iterator error is not reported") + } +} + +// BenchmarkAccountIteratorTraversal is a bit notorious -- all layers contain the +// exact same 200 accounts. That means that we need to process 2000 items, but +// only spit out 200 values eventually. +// +// The value-fetching benchmark is easy on the binary iterator, since it never has to reach +// down at any depth for retrieving the values -- all are on the topmost layer +// +// BenchmarkAccountIteratorTraversal/binary_iterator_keys-8 759984 1566 ns/op +// BenchmarkAccountIteratorTraversal/binary_iterator_values-8 150028 7900 ns/op +// BenchmarkAccountIteratorTraversal/fast_iterator_keys-8 172809 7006 ns/op +// BenchmarkAccountIteratorTraversal/fast_iterator_values-8 165112 7658 ns/op +func BenchmarkAccountIteratorTraversal(b *testing.B) { + // Create a custom account factory to recreate the same addresses + makeAccounts := func(num int) map[common.Hash][]byte { + accounts := make(map[common.Hash][]byte) + for i := 0; i < num; i++ { + h := common.Hash{} + binary.BigEndian.PutUint64(h[:], uint64(i+1)) + accounts[h] = randomAccount() + } + return accounts + } + config := &Config{ + WriteBufferSize: 0, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + for i := 1; i <= 100; i++ { + parent := types.EmptyRootHash + if i == 1 { + parent = common.HexToHash(fmt.Sprintf("0x%02x", i)) + } + db.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), parent, uint64(i), trienode.NewMergedNodeSet(), NewStateSetWithOrigin(makeAccounts(200), nil, nil, nil)) + } + // We call this once before the benchmark, so the creation of + // sorted accountlists are not included in the results. + head := db.tree.get(common.HexToHash("0x65")) + head.(*diffLayer).newBinaryAccountIterator(common.Hash{}) + + b.Run("binary iterator keys", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := 0 + it := head.(*diffLayer).newBinaryAccountIterator(common.Hash{}) + for it.Next() { + got++ + } + if exp := 200; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("binary iterator values", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := 0 + it := head.(*diffLayer).newBinaryAccountIterator(common.Hash{}) + for it.Next() { + got++ + head.(*diffLayer).account(it.Hash(), 0) + } + if exp := 200; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("fast iterator keys", func(b *testing.B) { + for i := 0; i < b.N; i++ { + it, _ := db.AccountIterator(common.HexToHash("0x65"), common.Hash{}) + defer it.Release() + + got := 0 + for it.Next() { + got++ + } + if exp := 200; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("fast iterator values", func(b *testing.B) { + for i := 0; i < b.N; i++ { + it, _ := db.AccountIterator(common.HexToHash("0x65"), common.Hash{}) + defer it.Release() + + got := 0 + for it.Next() { + got++ + it.Account() + } + if exp := 200; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) +} + +// BenchmarkAccountIteratorLargeBaselayer is a pretty realistic benchmark, where +// the baselayer is a lot larger than the upper layer. +// +// This is heavy on the binary iterator, which in most cases will have to +// call recursively 100 times for the majority of the values +// +// BenchmarkAccountIteratorLargeBaselayer/binary_iterator_(keys)-6 514 1971999 ns/op +// BenchmarkAccountIteratorLargeBaselayer/binary_iterator_(values)-6 61 18997492 ns/op +// BenchmarkAccountIteratorLargeBaselayer/fast_iterator_(keys)-6 10000 114385 ns/op +// BenchmarkAccountIteratorLargeBaselayer/fast_iterator_(values)-6 4047 296823 ns/op +func BenchmarkAccountIteratorLargeBaselayer(b *testing.B) { + // Create a custom account factory to recreate the same addresses + makeAccounts := func(num int) map[common.Hash][]byte { + accounts := make(map[common.Hash][]byte) + for i := 0; i < num; i++ { + h := common.Hash{} + binary.BigEndian.PutUint64(h[:], uint64(i+1)) + accounts[h] = randomAccount() + } + return accounts + } + config := &Config{ + WriteBufferSize: 0, + } + db := New(rawdb.NewMemoryDatabase(), config, false) + // db.WaitGeneration() + + db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(makeAccounts(2000), nil, nil, nil)) + for i := 2; i <= 100; i++ { + db.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), uint64(i), trienode.NewMergedNodeSet(), NewStateSetWithOrigin(makeAccounts(20), nil, nil, nil)) + } + // We call this once before the benchmark, so the creation of + // sorted accountlists are not included in the results. + head := db.tree.get(common.HexToHash("0x65")) + head.(*diffLayer).newBinaryAccountIterator(common.Hash{}) + + b.Run("binary iterator (keys)", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := 0 + it := head.(*diffLayer).newBinaryAccountIterator(common.Hash{}) + for it.Next() { + got++ + } + if exp := 2000; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("binary iterator (values)", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := 0 + it := head.(*diffLayer).newBinaryAccountIterator(common.Hash{}) + for it.Next() { + got++ + v := it.Hash() + head.(*diffLayer).account(v, 0) + } + if exp := 2000; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("fast iterator (keys)", func(b *testing.B) { + for i := 0; i < b.N; i++ { + it, _ := db.AccountIterator(common.HexToHash("0x65"), common.Hash{}) + defer it.Release() + + got := 0 + for it.Next() { + got++ + } + if exp := 2000; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("fast iterator (values)", func(b *testing.B) { + for i := 0; i < b.N; i++ { + it, _ := db.AccountIterator(common.HexToHash("0x65"), common.Hash{}) + defer it.Release() + + got := 0 + for it.Next() { + it.Account() + got++ + } + if exp := 2000; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) +} + +/* +func BenchmarkBinaryAccountIteration(b *testing.B) { + benchmarkAccountIteration(b, func(snap snapshot) AccountIterator { + return snap.(*diffLayer).newBinaryAccountIterator() + }) +} + +func BenchmarkFastAccountIteration(b *testing.B) { + benchmarkAccountIteration(b, newFastAccountIterator) +} + +func benchmarkAccountIteration(b *testing.B, iterator func(snap snapshot) AccountIterator) { + // Create a diff stack and randomize the accounts across them + layers := make([]map[common.Hash][]byte, 128) + for i := 0; i < len(layers); i++ { + layers[i] = make(map[common.Hash][]byte) + } + for i := 0; i < b.N; i++ { + depth := rand.Intn(len(layers)) + layers[depth][randomHash()] = randomAccount() + } + stack := snapshot(emptyLayer()) + for _, layer := range layers { + stack = stack.Update(common.Hash{}, layer, nil, nil) + } + // Reset the timers and report all the stats + it := iterator(stack) + + b.ResetTimer() + b.ReportAllocs() + + for it.Next() { + } +} +*/ diff --git a/triedb/pathdb/reader.go b/triedb/pathdb/reader.go index 4bba813b14..a404409035 100644 --- a/triedb/pathdb/reader.go +++ b/triedb/pathdb/reader.go @@ -86,6 +86,17 @@ func (r *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, return blob, nil } +// AccountRLP directly retrieves the account associated with a particular hash. +// An error will be returned if the read operation exits abnormally. Specifically, +// if the layer is already stale. +// +// Note: +// - the returned account data is not a copy, please don't modify it +// - no error will be returned if the requested account is not found in database +func (r *reader) AccountRLP(hash common.Hash) ([]byte, error) { + return r.layer.account(hash, 0) +} + // Account directly retrieves the account associated with a particular hash in // the slim data format. An error will be returned if the read operation exits // abnormally. Specifically, if the layer is already stale. diff --git a/triedb/pathdb/states.go b/triedb/pathdb/states.go index 6e633db272..81d34da5df 100644 --- a/triedb/pathdb/states.go +++ b/triedb/pathdb/states.go @@ -98,6 +98,17 @@ func (s *stateSet) account(hash common.Hash) ([]byte, bool) { return nil, false // account is unknown in this set } +// mustAccount returns the account data associated with the specified address +// hash. The difference is this function will return an error if the account +// is not found. +func (s *stateSet) mustAccount(hash common.Hash) ([]byte, error) { + // If the account is known locally, return it + if data, ok := s.accountData[hash]; ok { + return data, nil + } + return nil, fmt.Errorf("account is not found, %x", hash) +} + // storage returns the storage slot associated with the specified address hash // and storage key hash. func (s *stateSet) storage(accountHash, storageHash common.Hash) ([]byte, bool) { @@ -110,6 +121,19 @@ func (s *stateSet) storage(accountHash, storageHash common.Hash) ([]byte, bool) return nil, false // storage is unknown in this set } +// mustStorage returns the storage slot associated with the specified address +// hash and storage key hash. The difference is this function will return an +// error if the storage slot is not found. +func (s *stateSet) mustStorage(accountHash, storageHash common.Hash) ([]byte, error) { + // If the account is known locally, try to resolve the slot locally + if storage, ok := s.storageData[accountHash]; ok { + if data, ok := storage[storageHash]; ok { + return data, nil + } + } + return nil, fmt.Errorf("storage slot is not found, %x %x", accountHash, storageHash) +} + // check sanitizes accounts and storage slots to ensure the data validity. // Additionally, it computes the total memory size occupied by the maps. func (s *stateSet) check() uint64 { @@ -132,8 +156,6 @@ func (s *stateSet) check() uint64 { // the deleted ones. // // Note, the returned slice is not a copy, so do not modify it. -// -// nolint:unused func (s *stateSet) accountList() []common.Hash { // If an old list already exists, return it s.listLock.RLock() @@ -160,8 +182,6 @@ func (s *stateSet) accountList() []common.Hash { // storage slot. // // Note, the returned slice is not a copy, so do not modify it. -// -// nolint:unused func (s *stateSet) storageList(accountHash common.Hash) []common.Hash { s.listLock.RLock() if _, ok := s.storageData[accountHash]; !ok { diff --git a/triedb/pathdb/states_test.go b/triedb/pathdb/states_test.go index 4557fa958d..f097e90e81 100644 --- a/triedb/pathdb/states_test.go +++ b/triedb/pathdb/states_test.go @@ -28,39 +28,39 @@ import ( func TestStatesMerge(t *testing.T) { a := newStates( map[common.Hash][]byte{ - common.Hash{0xa}: {0xa0}, - common.Hash{0xb}: {0xb0}, - common.Hash{0xc}: {0xc0}, + {0xa}: {0xa0}, + {0xb}: {0xb0}, + {0xc}: {0xc0}, }, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0xa}: { + {0xa}: { common.Hash{0x1}: {0x10}, common.Hash{0x2}: {0x20}, }, - common.Hash{0xb}: { + {0xb}: { common.Hash{0x1}: {0x10}, }, - common.Hash{0xc}: { + {0xc}: { common.Hash{0x1}: {0x10}, }, }, ) b := newStates( map[common.Hash][]byte{ - common.Hash{0xa}: {0xa1}, - common.Hash{0xb}: {0xb1}, - common.Hash{0xc}: nil, // delete account + {0xa}: {0xa1}, + {0xb}: {0xb1}, + {0xc}: nil, // delete account }, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0xa}: { + {0xa}: { common.Hash{0x1}: {0x11}, common.Hash{0x2}: nil, // delete slot common.Hash{0x3}: {0x31}, }, - common.Hash{0xb}: { + {0xb}: { common.Hash{0x1}: {0x11}, }, - common.Hash{0xc}: { + {0xc}: { common.Hash{0x1}: nil, // delete slot }, }, @@ -116,39 +116,39 @@ func TestStatesMerge(t *testing.T) { func TestStatesRevert(t *testing.T) { a := newStates( map[common.Hash][]byte{ - common.Hash{0xa}: {0xa0}, - common.Hash{0xb}: {0xb0}, - common.Hash{0xc}: {0xc0}, + {0xa}: {0xa0}, + {0xb}: {0xb0}, + {0xc}: {0xc0}, }, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0xa}: { + {0xa}: { common.Hash{0x1}: {0x10}, common.Hash{0x2}: {0x20}, }, - common.Hash{0xb}: { + {0xb}: { common.Hash{0x1}: {0x10}, }, - common.Hash{0xc}: { + {0xc}: { common.Hash{0x1}: {0x10}, }, }, ) b := newStates( map[common.Hash][]byte{ - common.Hash{0xa}: {0xa1}, - common.Hash{0xb}: {0xb1}, - common.Hash{0xc}: nil, + {0xa}: {0xa1}, + {0xb}: {0xb1}, + {0xc}: nil, }, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0xa}: { + {0xa}: { common.Hash{0x1}: {0x11}, common.Hash{0x2}: nil, common.Hash{0x3}: {0x31}, }, - common.Hash{0xb}: { + {0xb}: { common.Hash{0x1}: {0x11}, }, - common.Hash{0xc}: { + {0xc}: { common.Hash{0x1}: nil, }, }, @@ -156,20 +156,20 @@ func TestStatesRevert(t *testing.T) { a.merge(b) a.revertTo( map[common.Hash][]byte{ - common.Hash{0xa}: {0xa0}, - common.Hash{0xb}: {0xb0}, - common.Hash{0xc}: {0xc0}, + {0xa}: {0xa0}, + {0xb}: {0xb0}, + {0xc}: {0xc0}, }, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0xa}: { + {0xa}: { common.Hash{0x1}: {0x10}, common.Hash{0x2}: {0x20}, common.Hash{0x3}: nil, }, - common.Hash{0xb}: { + {0xb}: { common.Hash{0x1}: {0x10}, }, - common.Hash{0xc}: { + {0xc}: { common.Hash{0x1}: {0x10}, }, }, @@ -227,14 +227,14 @@ func TestStateRevertAccountNullMarker(t *testing.T) { a := newStates(nil, nil) // empty initial state b := newStates( map[common.Hash][]byte{ - common.Hash{0xa}: {0xa}, + {0xa}: {0xa}, }, nil, ) a.merge(b) // create account 0xa a.revertTo( map[common.Hash][]byte{ - common.Hash{0xa}: nil, + {0xa}: nil, }, nil, ) // revert the transition b @@ -253,13 +253,13 @@ func TestStateRevertAccountNullMarker(t *testing.T) { // entry in the set. func TestStateRevertStorageNullMarker(t *testing.T) { a := newStates(map[common.Hash][]byte{ - common.Hash{0xa}: {0xa}, + {0xa}: {0xa}, }, nil) // initial state with account 0xa b := newStates( nil, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0xa}: { + {0xa}: { common.Hash{0x1}: {0x1}, }, }, @@ -268,7 +268,7 @@ func TestStateRevertStorageNullMarker(t *testing.T) { a.revertTo( nil, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0xa}: { + {0xa}: { common.Hash{0x1}: nil, }, }, @@ -286,10 +286,10 @@ func TestStateRevertStorageNullMarker(t *testing.T) { func TestStatesEncode(t *testing.T) { s := newStates( map[common.Hash][]byte{ - common.Hash{0x1}: {0x1}, + {0x1}: {0x1}, }, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0x1}: { + {0x1}: { common.Hash{0x1}: {0x1}, }, }, @@ -313,18 +313,18 @@ func TestStatesEncode(t *testing.T) { func TestStateWithOriginEncode(t *testing.T) { s := NewStateSetWithOrigin( map[common.Hash][]byte{ - common.Hash{0x1}: {0x1}, + {0x1}: {0x1}, }, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0x1}: { + {0x1}: { common.Hash{0x1}: {0x1}, }, }, map[common.Address][]byte{ - common.Address{0x1}: {0x1}, + {0x1}: {0x1}, }, map[common.Address]map[common.Hash][]byte{ - common.Address{0x1}: { + {0x1}: { common.Hash{0x1}: {0x1}, }, }, @@ -359,19 +359,19 @@ func TestStateSizeTracking(t *testing.T) { a := newStates( map[common.Hash][]byte{ - common.Hash{0xa}: {0xa0}, // common.HashLength+1 - common.Hash{0xb}: {0xb0}, // common.HashLength+1 - common.Hash{0xc}: {0xc0}, // common.HashLength+1 + {0xa}: {0xa0}, // common.HashLength+1 + {0xb}: {0xb0}, // common.HashLength+1 + {0xc}: {0xc0}, // common.HashLength+1 }, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0xa}: { + {0xa}: { common.Hash{0x1}: {0x10}, // 2*common.HashLength+1 common.Hash{0x2}: {0x20}, // 2*common.HashLength+1 }, - common.Hash{0xb}: { + {0xb}: { common.Hash{0x1}: {0x10, 0x11, 0x12}, // 2*common.HashLength+3 }, - common.Hash{0xc}: { + {0xc}: { common.Hash{0x1}: {0x10}, // 2*common.HashLength+1 }, }, @@ -386,21 +386,21 @@ func TestStateSizeTracking(t *testing.T) { 3*2*common.HashLength /* storage data of 0xc */ b := newStates( map[common.Hash][]byte{ - common.Hash{0xa}: {0xa1, 0xa1}, // common.HashLength+2 - common.Hash{0xb}: {0xb1, 0xb1, 0xb1}, // common.HashLength+3 - common.Hash{0xc}: nil, // common.HashLength, account deletion + {0xa}: {0xa1, 0xa1}, // common.HashLength+2 + {0xb}: {0xb1, 0xb1, 0xb1}, // common.HashLength+3 + {0xc}: nil, // common.HashLength, account deletion }, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0xa}: { + {0xa}: { common.Hash{0x1}: {0x11, 0x11, 0x11}, // 2*common.HashLength+3 common.Hash{0x3}: {0x31, 0x31}, // 2*common.HashLength+2, slot creation }, - common.Hash{0xb}: { + {0xb}: { common.Hash{0x1}: {0x11, 0x11}, // 2*common.HashLength+2 common.Hash{0x2}: {0x22, 0x22}, // 2*common.HashLength+2, slot creation }, // The storage of 0xc is entirely removed - common.Hash{0xc}: { + {0xc}: { common.Hash{0x1}: nil, // 2*common.HashLength, slot deletion common.Hash{0x2}: nil, // 2*common.HashLength, slot deletion common.Hash{0x3}: nil, // 2*common.HashLength, slot deletion @@ -424,21 +424,21 @@ func TestStateSizeTracking(t *testing.T) { // Revert the set to original status a.revertTo( map[common.Hash][]byte{ - common.Hash{0xa}: {0xa0}, - common.Hash{0xb}: {0xb0}, - common.Hash{0xc}: {0xc0}, + {0xa}: {0xa0}, + {0xb}: {0xb0}, + {0xc}: {0xc0}, }, map[common.Hash]map[common.Hash][]byte{ - common.Hash{0xa}: { + {0xa}: { common.Hash{0x1}: {0x10}, common.Hash{0x2}: {0x20}, common.Hash{0x3}: nil, // revert slot creation }, - common.Hash{0xb}: { + {0xb}: { common.Hash{0x1}: {0x10, 0x11, 0x12}, common.Hash{0x2}: nil, // revert slot creation }, - common.Hash{0xc}: { + {0xc}: { common.Hash{0x1}: {0x10}, common.Hash{0x2}: {0x20}, // resurrected slot common.Hash{0x3}: {0x30}, // resurrected slot From 5c58612e12a5131233be5efc9aa9ff55aac4538a Mon Sep 17 00:00:00 2001 From: Martin HS Date: Tue, 17 Dec 2024 08:58:26 +0100 Subject: [PATCH 4/5] core/vm, go.mod: update uint256 and use faster method to write to memory (#30868) Updates geth to use the latest uint256, and use faster memory-writer --- core/vm/memory.go | 3 +-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/core/vm/memory.go b/core/vm/memory.go index 1ddd8d1ead..5e11e83748 100644 --- a/core/vm/memory.go +++ b/core/vm/memory.go @@ -73,8 +73,7 @@ func (m *Memory) Set32(offset uint64, val *uint256.Int) { panic("invalid memory: store empty") } // Fill in relevant bits - b32 := val.Bytes32() - copy(m.store[offset:], b32[:]) + val.PutUint256(m.store[offset:]) } // Resize resizes the memory to size diff --git a/go.mod b/go.mod index b33c30f41d..0f000636c3 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/hashicorp/go-bexpr v0.1.10 github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 github.com/holiman/bloomfilter/v2 v2.0.3 - github.com/holiman/uint256 v1.3.1 + github.com/holiman/uint256 v1.3.2 github.com/huin/goupnp v1.3.0 github.com/influxdata/influxdb-client-go/v2 v2.4.0 github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c diff --git a/go.sum b/go.sum index b61121664f..e79fc6941c 100644 --- a/go.sum +++ b/go.sum @@ -306,8 +306,8 @@ github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8 github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= -github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= -github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= +github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= From 3c208cdea82acff878c7675312a97a638997e1fc Mon Sep 17 00:00:00 2001 From: maskpp Date: Tue, 17 Dec 2024 16:12:10 +0800 Subject: [PATCH 5/5] accounts/abi/bind: make it possible to wait for tx hash (#30079) This change adds methods which makes it possible for to wait for a transaction with a specific hash when deploying contracts during abi bind interaction. --------- Co-authored-by: Marius van der Wijden --- accounts/abi/bind/util.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/accounts/abi/bind/util.go b/accounts/abi/bind/util.go index b931fbb04d..c83116e9a1 100644 --- a/accounts/abi/bind/util.go +++ b/accounts/abi/bind/util.go @@ -30,12 +30,18 @@ import ( // WaitMined waits for tx to be mined on the blockchain. // It stops waiting when the context is canceled. func WaitMined(ctx context.Context, b DeployBackend, tx *types.Transaction) (*types.Receipt, error) { + return WaitMinedHash(ctx, b, tx.Hash()) +} + +// WaitMinedHash waits for a transaction with the provided hash to be mined on the blockchain. +// It stops waiting when the context is canceled. +func WaitMinedHash(ctx context.Context, b DeployBackend, hash common.Hash) (*types.Receipt, error) { queryTicker := time.NewTicker(time.Second) defer queryTicker.Stop() - logger := log.New("hash", tx.Hash()) + logger := log.New("hash", hash) for { - receipt, err := b.TransactionReceipt(ctx, tx.Hash()) + receipt, err := b.TransactionReceipt(ctx, hash) if err == nil { return receipt, nil } @@ -61,7 +67,13 @@ func WaitDeployed(ctx context.Context, b DeployBackend, tx *types.Transaction) ( if tx.To() != nil { return common.Address{}, errors.New("tx is not contract creation") } - receipt, err := WaitMined(ctx, b, tx) + return WaitDeployedHash(ctx, b, tx.Hash()) +} + +// WaitDeployedHash waits for a contract deployment transaction with the provided hash and returns the on-chain +// contract address when it is mined. It stops waiting when ctx is canceled. +func WaitDeployedHash(ctx context.Context, b DeployBackend, hash common.Hash) (common.Address, error) { + receipt, err := WaitMinedHash(ctx, b, hash) if err != nil { return common.Address{}, err }